Compare commits

..
Author SHA1 Message Date
Andrew BeveridgeandClaude Opus 4.5 275fc909f6 fix: E2E test issues - add Firestore index and fix job endpoint URL
- Add missing Firestore composite index for customer_email + made_for_you queries
  (required for test-webhook made-for-you order processing)
- Fix getJobDetails helper to use /api/jobs/{job_id} instead of /api/admin/jobs/{job_id}
  (the admin route doesn't have a GET method)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-29 21:52:34 -08:00
51 changed files with 446 additions and 5294 deletions

No files matched your search

+17 -710
View File
@@ -20,7 +20,6 @@ from backend.services.user_service import get_user_service, UserService, USERS_C
from backend.services.job_manager import JobManager
from backend.services.flacfetch_client import get_flacfetch_client, FlacfetchServiceError
from backend.services.storage_service import StorageService
from backend.services.audio_search_service import get_audio_search_service, NoResultsError, AudioSearchError
from backend.models.job import JobStatus
from backend.utils.test_data import is_test_email
from karaoke_gen.utils import sanitize_filename
@@ -1013,8 +1012,8 @@ class TriggerWorkerResponse(BaseModel):
ALLOWED_RESET_STATES = {
"pending",
"awaiting_audio_selection",
"awaiting_review", # Combined review (lyrics + instrumental selection)
# Note: awaiting_instrumental_selection removed - deprecated by combined review flow (Jan 2026)
"awaiting_review",
"awaiting_instrumental_selection",
"instrumental_selected", # Re-run video generation with same settings
}
@@ -1035,9 +1034,6 @@ STATE_DATA_CLEAR_KEYS = {
"render_progress",
"screens_progress",
"encoding_progress",
# Parallel processing flags - must clear to allow workers to run again
"audio_complete",
"lyrics_complete",
],
"awaiting_audio_selection": [
"audio_selection",
@@ -1050,7 +1046,6 @@ STATE_DATA_CLEAR_KEYS = {
"encoding_progress",
],
"awaiting_review": [
# Combined review - user reviews lyrics AND selects instrumental in one step
"review_complete",
"corrected_lyrics",
"instrumental_selection",
@@ -1059,7 +1054,13 @@ STATE_DATA_CLEAR_KEYS = {
"screens_progress",
"encoding_progress",
],
# Note: awaiting_instrumental_selection removed - deprecated by combined review flow (Jan 2026)
"awaiting_instrumental_selection": [
"instrumental_selection",
"video_progress",
"render_progress",
"screens_progress",
"encoding_progress",
],
"instrumental_selected": [
# Clear video/encoding/distribution state to allow re-processing
# Preserves: instrumental_selection, lyrics_metadata, review data
@@ -1147,8 +1148,14 @@ async def reset_job(
}
# Perform the update with state_data clearing
# Note: update_job() raises exceptions on failure, returns None on success
job_manager.update_job(job_id, updates)
# We need to set the cleared keys to DELETE_FIELD
success = job_manager.update_job(job_id, updates)
if not success:
raise HTTPException(
status_code=500,
detail="Failed to reset job. Please try again."
)
# Clear the state data keys and error state using direct Firestore update
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
@@ -1856,703 +1863,3 @@ async def impersonate_user(
user_email=target_email,
message=f"Now impersonating {target_email}",
)
# =============================================================================
# Regenerate Screens Endpoint
# =============================================================================
class RegenerateScreensResponse(BaseModel):
"""Response from regenerate screens endpoint."""
status: str
job_id: str
message: str
previous_screens_deleted: bool
worker_triggered: bool
error: Optional[str] = None
# States where regenerate screens is allowed
REGENERATE_SCREENS_ALLOWED_STATES = {
"complete",
"failed",
"awaiting_review",
"awaiting_instrumental_selection",
"instrumental_selected",
"prep_complete",
}
@router.post("/jobs/{job_id}/regenerate-screens", response_model=RegenerateScreensResponse)
async def regenerate_screens(
job_id: str,
auth_data: AuthResult = Depends(require_admin),
):
"""
Regenerate title and end screens with current artist/title metadata (admin only).
Use this when you've edited the artist or title fields and need the
title/end screens to reflect the new metadata.
This endpoint:
1. Validates the job is in an appropriate state
2. Deletes existing screen files from GCS
3. Triggers the screens worker to regenerate with current metadata
4. Returns immediately (does not wait for completion)
Monitor progress via the job detail page or logs.
Allowed states: complete, failed, awaiting_review, awaiting_instrumental_selection,
instrumental_selected, prep_complete
"""
admin_email = auth_data.user_email or "unknown"
job_manager = JobManager()
job = job_manager.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
# Validate job state
if job.status not in REGENERATE_SCREENS_ALLOWED_STATES:
raise HTTPException(
status_code=400,
detail=f"Cannot regenerate screens for job in '{job.status}' state. "
f"Allowed states: {', '.join(sorted(REGENERATE_SCREENS_ALLOWED_STATES))}"
)
# Validate job has artist and title
if not job.artist or not job.title:
raise HTTPException(
status_code=400,
detail="Job must have artist and title set to regenerate screens"
)
# Check that audio and lyrics processing completed at some point
state_data = job.state_data or {}
audio_progress = state_data.get('audio_progress', {})
lyrics_progress = state_data.get('lyrics_progress', {})
if audio_progress.get('stage') != 'audio_complete':
raise HTTPException(
status_code=400,
detail="Audio processing must be complete before regenerating screens"
)
if lyrics_progress.get('stage') != 'lyrics_complete':
raise HTTPException(
status_code=400,
detail="Lyrics processing must be complete before regenerating screens"
)
# Delete existing screen files from GCS
storage = StorageService()
screens_deleted = False
try:
# Delete screen video and image files
screen_paths = [
f"jobs/{job_id}/screens/title.mov",
f"jobs/{job_id}/screens/title.jpg",
f"jobs/{job_id}/screens/title.png",
f"jobs/{job_id}/screens/end.mov",
f"jobs/{job_id}/screens/end.jpg",
f"jobs/{job_id}/screens/end.png",
]
for path in screen_paths:
try:
storage.delete_file(path)
except Exception:
pass # File may not exist
screens_deleted = True
logger.info(f"Admin {admin_email} deleted existing screens for job {job_id}")
except Exception as e:
logger.warning(f"Error deleting screens for job {job_id}: {e}")
# Clear screens_progress to allow worker to run
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
db = job_manager.firestore.db
job_ref = db.collection("jobs").document(job_id)
# Update state to allow screens worker to run
job_ref.update({
"state_data.screens_progress": DELETE_FIELD,
"state_data.audio_complete": True,
"state_data.lyrics_complete": True,
"timeline": ArrayUnion([{
"status": job.status,
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} triggered screen regeneration"
}])
})
# Trigger screens worker
worker_triggered = False
error_msg = None
try:
from backend.services.worker_service import get_worker_service
worker_service = get_worker_service()
worker_triggered = await worker_service.trigger_screens_worker(job_id)
if worker_triggered:
logger.info(f"Admin {admin_email} triggered screens regeneration for job {job_id}")
else:
error_msg = "Worker service returned False - check service logs"
except Exception as e:
error_msg = str(e)
logger.error(f"Error triggering screens worker for job {job_id}: {e}", exc_info=True)
return RegenerateScreensResponse(
status="success" if worker_triggered else "error",
job_id=job_id,
message="Screens regeneration started" if worker_triggered else "Failed to trigger screens worker",
previous_screens_deleted=screens_deleted,
worker_triggered=worker_triggered,
error=error_msg,
)
# =============================================================================
# Restart Job Endpoint
# =============================================================================
class RestartJobRequest(BaseModel):
"""Request model for restarting a job."""
preserve_audio_stems: bool = False # If true, keep existing stems (faster)
delete_outputs: bool = True # Delete existing output files from GCS
class RestartJobResponse(BaseModel):
"""Response from restart job endpoint."""
status: str
job_id: str
message: str
previous_status: str
new_status: str
cleared_data: List[str]
deleted_gcs_paths: List[str]
workers_triggered: List[str]
error: Optional[str] = None
# States where restart is allowed
RESTART_ALLOWED_STATES = {
"pending",
"complete",
"failed",
"awaiting_review",
"awaiting_audio_selection",
"awaiting_instrumental_selection",
"instrumental_selected",
"prep_complete",
}
@router.post("/jobs/{job_id}/restart", response_model=RestartJobResponse)
async def restart_job(
job_id: str,
request: RestartJobRequest,
auth_data: AuthResult = Depends(require_admin),
):
"""
Fully restart a job from the beginning (admin only).
Unlike reset (which just changes state), restart actually triggers
the appropriate workers to begin processing again.
Options:
- preserve_audio_stems: If True, keeps existing audio separation (faster restart).
Useful when you just need to regenerate screens with updated metadata.
- delete_outputs: If True (default), deletes existing output files from GCS.
When preserve_audio_stems=True:
- Keeps stems, lyrics, and transcription data
- Clears screens, video, encoding state
- Triggers screens worker immediately
When preserve_audio_stems=False:
- Clears all processing state
- For YouTube URL jobs: triggers download then audio/lyrics workers
- For audio search jobs: transitions to awaiting_audio_selection
- For file upload jobs: triggers audio/lyrics workers
Allowed states: pending, complete, failed, awaiting_review, awaiting_audio_selection,
awaiting_instrumental_selection, instrumental_selected, prep_complete
"""
admin_email = auth_data.user_email or "unknown"
job_manager = JobManager()
job = job_manager.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
# Validate job state
if job.status not in RESTART_ALLOWED_STATES:
raise HTTPException(
status_code=400,
detail=f"Cannot restart job in '{job.status}' state. "
f"Allowed states: {', '.join(sorted(RESTART_ALLOWED_STATES))}"
)
previous_status = job.status
state_data = job.state_data or {}
cleared_keys = []
deleted_paths = []
workers_triggered = []
error_msg = None
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
db = job_manager.firestore.db
job_ref = db.collection("jobs").document(job_id)
# Delete output files if requested
if request.delete_outputs:
storage = StorageService()
# Delete screens
for screen_file in ["title.mov", "title.jpg", "title.png", "end.mov", "end.jpg", "end.png"]:
try:
path = f"jobs/{job_id}/screens/{screen_file}"
storage.delete_file(path)
deleted_paths.append(path)
except Exception:
pass
# Delete final outputs
for output_file in ["final_4k.mp4", "final_720p.mp4", "with_vocals.mkv", "preview.mp4"]:
try:
path = f"jobs/{job_id}/output/{output_file}"
storage.delete_file(path)
deleted_paths.append(path)
except Exception:
pass
if request.preserve_audio_stems:
# Quick restart: keep stems/lyrics, just regenerate screens and video
# Validate that audio and lyrics completed
audio_progress = state_data.get('audio_progress', {})
lyrics_progress = state_data.get('lyrics_progress', {})
if audio_progress.get('stage') != 'audio_complete':
raise HTTPException(
status_code=400,
detail="Cannot preserve audio stems - audio processing was not complete"
)
if lyrics_progress.get('stage') != 'lyrics_complete':
raise HTTPException(
status_code=400,
detail="Cannot preserve audio stems - lyrics processing was not complete"
)
# Clear only screens/video/encoding state
keys_to_clear = [
"screens_progress",
"video_progress",
"render_progress",
"encoding_progress",
"review_complete",
"instrumental_selection",
]
update_payload = {}
for key in keys_to_clear:
if key in state_data:
update_payload[f"state_data.{key}"] = DELETE_FIELD
cleared_keys.append(key)
# Set coordination flags to trigger screens worker
update_payload["state_data.audio_complete"] = True
update_payload["state_data.lyrics_complete"] = True
update_payload["status"] = "transcribing" # Allows screens worker to run
update_payload["message"] = "Restarting with preserved audio stems"
update_payload["error_message"] = DELETE_FIELD
update_payload["error_details"] = DELETE_FIELD
update_payload["timeline"] = ArrayUnion([{
"status": "transcribing",
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} restarted job (preserving audio stems)"
}])
job_ref.update(update_payload)
new_status = "transcribing"
# Trigger screens worker
try:
from backend.services.worker_service import get_worker_service
worker_service = get_worker_service()
if await worker_service.trigger_screens_worker(job_id):
workers_triggered.append("screens")
logger.info(f"Admin {admin_email} triggered screens worker for restart of job {job_id}")
else:
error_msg = "Failed to trigger screens worker"
except Exception as e:
error_msg = f"Error triggering screens worker: {e}"
logger.error(f"Error triggering screens worker for job {job_id}: {e}", exc_info=True)
else:
# Full restart: clear everything and start from beginning
keys_to_clear = [
"audio_complete",
"lyrics_complete",
"audio_progress",
"lyrics_progress",
"screens_progress",
"video_progress",
"render_progress",
"encoding_progress",
"review_complete",
"corrected_lyrics",
"instrumental_selection",
"instrumental_options",
"lyrics_metadata",
]
update_payload = {}
for key in keys_to_clear:
if key in state_data:
update_payload[f"state_data.{key}"] = DELETE_FIELD
cleared_keys.append(key)
update_payload["error_message"] = DELETE_FIELD
update_payload["error_details"] = DELETE_FIELD
# Determine what workers to trigger based on job input type
has_youtube_url = bool(job.url)
has_uploaded_file = bool(job.input_media_gcs_path)
has_audio_search = bool(state_data.get('audio_search_results') or state_data.get('audio_selection'))
if has_youtube_url and not has_audio_search:
# YouTube URL job - need to re-download and process
# Clear the downloaded file to force re-download
update_payload["input_media_gcs_path"] = DELETE_FIELD
cleared_keys.append("input_media_gcs_path")
update_payload["status"] = "downloading"
update_payload["message"] = "Re-downloading audio from YouTube"
update_payload["timeline"] = ArrayUnion([{
"status": "downloading",
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} restarted job - downloading from YouTube"
}])
job_ref.update(update_payload)
new_status = "downloading"
# Download YouTube audio using YouTubeDownloadService, then trigger processing
try:
from backend.services.youtube_download_service import (
get_youtube_download_service,
YouTubeDownloadError,
)
from backend.services.worker_service import get_worker_service
youtube_service = get_youtube_download_service()
audio_gcs_path = await youtube_service.download(
url=job.url,
job_id=job_id,
artist=job.artist,
title=job.title,
)
# Update job with downloaded audio path
job_manager.update_job(job_id, {'input_media_gcs_path': audio_gcs_path})
workers_triggered.append("download")
logger.info(f"YouTube audio re-downloaded to GCS: {audio_gcs_path}")
# Now trigger audio and lyrics workers
worker_service = get_worker_service()
if await worker_service.trigger_audio_worker(job_id):
workers_triggered.append("audio")
if await worker_service.trigger_lyrics_worker(job_id):
workers_triggered.append("lyrics")
if "audio" not in workers_triggered and "lyrics" not in workers_triggered:
error_msg = "Audio downloaded but failed to trigger processing workers"
except YouTubeDownloadError as e:
error_msg = f"YouTube download failed: {e}"
logger.error(f"YouTube download failed for job {job_id}: {e}")
# Update job to failed state
job_manager.mark_job_failed(job_id, f"YouTube download failed: {e}")
new_status = "failed"
except Exception as e:
error_msg = f"Error during YouTube restart: {e}"
logger.error(f"Error restarting YouTube job {job_id}: {e}", exc_info=True)
elif has_audio_search:
# Audio search job - go back to audio selection
# Clear audio search results to allow re-search
for key in ["audio_search_results", "audio_search_count", "remote_search_id", "audio_selection"]:
if key in state_data:
update_payload[f"state_data.{key}"] = DELETE_FIELD
cleared_keys.append(key)
update_payload["input_media_gcs_path"] = DELETE_FIELD
update_payload["status"] = "awaiting_audio_selection"
update_payload["message"] = "Ready for audio source selection"
update_payload["timeline"] = ArrayUnion([{
"status": "awaiting_audio_selection",
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} restarted job - select audio source"
}])
job_ref.update(update_payload)
new_status = "awaiting_audio_selection"
# No workers to trigger - waiting for admin to select audio
elif has_uploaded_file:
# File upload job - just trigger audio/lyrics workers
update_payload["status"] = "downloading"
update_payload["message"] = "Restarting audio and lyrics processing"
update_payload["timeline"] = ArrayUnion([{
"status": "downloading",
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} restarted job from beginning"
}])
job_ref.update(update_payload)
new_status = "downloading"
try:
from backend.services.worker_service import get_worker_service
worker_service = get_worker_service()
if await worker_service.trigger_audio_worker(job_id):
workers_triggered.append("audio")
if await worker_service.trigger_lyrics_worker(job_id):
workers_triggered.append("lyrics")
if not workers_triggered:
error_msg = "Failed to trigger audio/lyrics workers"
except Exception as e:
error_msg = f"Error triggering workers: {e}"
logger.error(f"Error triggering workers for job {job_id}: {e}", exc_info=True)
else:
# Unknown job type - just set to pending
update_payload["status"] = "pending"
update_payload["message"] = "Job reset - manual intervention may be needed"
update_payload["timeline"] = ArrayUnion([{
"status": "pending",
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} restarted job (no automatic worker trigger)"
}])
job_ref.update(update_payload)
new_status = "pending"
error_msg = "Job type unclear - may need manual worker trigger"
logger.info(
f"Admin {admin_email} restarted job {job_id}: "
f"{previous_status} -> {new_status}, cleared: {cleared_keys}, workers: {workers_triggered}"
)
return RestartJobResponse(
status="success" if workers_triggered or new_status == "awaiting_audio_selection" else "partial",
job_id=job_id,
message=f"Job restarted from {previous_status} to {new_status}",
previous_status=previous_status,
new_status=new_status,
cleared_data=cleared_keys,
deleted_gcs_paths=deleted_paths,
workers_triggered=workers_triggered,
error=error_msg,
)
# =============================================================================
# Override Audio Source Endpoint
# =============================================================================
class OverrideAudioSourceRequest(BaseModel):
"""Request model for overriding audio source."""
source_type: str # "audio_search" - switch to audio search mode
class OverrideAudioSourceResponse(BaseModel):
"""Response from override audio source endpoint."""
status: str
job_id: str
message: str
previous_source: str # "youtube", "audio_search", "file_upload"
new_source: str
cleared_data: List[str]
new_status: str
search_results_count: Optional[int] = None
error: Optional[str] = None
# States where audio source override is allowed
OVERRIDE_AUDIO_ALLOWED_STATES = {
"pending",
"complete",
"failed",
"awaiting_review",
"awaiting_audio_selection",
"instrumental_selected",
"prep_complete",
}
@router.post("/jobs/{job_id}/override-audio-source", response_model=OverrideAudioSourceResponse)
async def override_audio_source(
job_id: str,
request: OverrideAudioSourceRequest,
auth_data: AuthResult = Depends(require_admin),
):
"""
Override the audio source for a job (admin only).
Use this when a Made-For-You order was submitted with a YouTube URL
but you want to use audio search to find higher quality audio instead.
Currently supports switching to audio_search mode, which:
1. Clears existing audio-related state
2. Performs an audio search using the job's artist/title
3. Stores search results in state_data
4. Transitions job to awaiting_audio_selection with results ready
5. Admin can then select an audio source from the search results
Allowed states: pending, complete, failed, awaiting_review, awaiting_audio_selection,
instrumental_selected, prep_complete
"""
admin_email = auth_data.user_email or "unknown"
job_manager = JobManager()
job = job_manager.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
# Validate job state
if job.status not in OVERRIDE_AUDIO_ALLOWED_STATES:
raise HTTPException(
status_code=400,
detail=f"Cannot override audio source for job in '{job.status}' state. "
f"Allowed states: {', '.join(sorted(OVERRIDE_AUDIO_ALLOWED_STATES))}"
)
# Validate source type
if request.source_type != "audio_search":
raise HTTPException(
status_code=400,
detail="Only 'audio_search' source type is currently supported"
)
# Determine current source type
state_data = job.state_data or {}
if state_data.get('audio_selection') or state_data.get('audio_search_results'):
previous_source = "audio_search"
elif job.url:
previous_source = "youtube"
elif job.input_media_gcs_path:
previous_source = "file_upload"
else:
previous_source = "unknown"
previous_status = job.status
cleared_keys = []
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
db = job_manager.firestore.db
job_ref = db.collection("jobs").document(job_id)
# Clear all audio-related state to start fresh with audio search
keys_to_clear = [
"audio_complete",
"lyrics_complete",
"audio_progress",
"lyrics_progress",
"screens_progress",
"video_progress",
"render_progress",
"encoding_progress",
"review_complete",
"corrected_lyrics",
"instrumental_selection",
"instrumental_options",
"lyrics_metadata",
"audio_search_results",
"audio_search_count",
"remote_search_id",
"audio_selection",
]
update_payload = {}
for key in keys_to_clear:
if key in state_data:
update_payload[f"state_data.{key}"] = DELETE_FIELD
cleared_keys.append(key)
# Clear job-level audio fields
update_payload["url"] = DELETE_FIELD # Clear YouTube URL
update_payload["input_media_gcs_path"] = DELETE_FIELD # Clear downloaded file
update_payload["error_message"] = DELETE_FIELD
update_payload["error_details"] = DELETE_FIELD
# Set audio search fields based on current artist/title
search_artist = job.artist
search_title = job.title
update_payload["audio_search_artist"] = search_artist
update_payload["audio_search_title"] = search_title
# First transition to searching_audio
update_payload["status"] = "searching_audio"
update_payload["message"] = f"Searching for: {search_artist} - {search_title}"
update_payload["progress"] = 5
update_payload["timeline"] = ArrayUnion([{
"status": "searching_audio",
"timestamp": datetime.now(timezone.utc).isoformat(),
"message": f"Admin {admin_email} switched audio source from {previous_source} to audio_search"
}])
job_ref.update(update_payload)
# Perform the actual audio search
search_results_count = None
error_msg = None
try:
audio_search_service = get_audio_search_service()
search_results = audio_search_service.search(search_artist, search_title)
search_results_count = len(search_results)
# Store results in job state_data
results_dicts = [r.to_dict() for r in search_results]
state_data_update = {
'audio_search_results': results_dicts,
'audio_search_count': search_results_count,
}
if audio_search_service.last_remote_search_id:
state_data_update['remote_search_id'] = audio_search_service.last_remote_search_id
# Transition to awaiting_audio_selection with results
new_status = "awaiting_audio_selection"
job_ref.update({
"state_data": state_data_update,
"status": new_status,
"message": f"Found {search_results_count} audio sources - select one",
"progress": 10,
})
logger.info(
f"Admin {admin_email} overrode audio source for job {job_id}: "
f"{previous_source} -> audio_search, found {search_results_count} results"
)
except NoResultsError as e:
new_status = "failed"
error_msg = f"No audio sources found for: {search_artist} - {search_title}"
job_ref.update({
"status": new_status,
"message": error_msg,
"error_message": error_msg,
})
logger.warning(f"Audio search failed for job {job_id}: {error_msg}")
except AudioSearchError as e:
new_status = "failed"
error_msg = f"Audio search failed: {e}"
job_ref.update({
"status": new_status,
"message": error_msg,
"error_message": error_msg,
})
logger.error(f"Audio search error for job {job_id}: {e}")
return OverrideAudioSourceResponse(
status="success" if not error_msg else "error",
job_id=job_id,
message=f"Found {search_results_count} audio sources - select one in the admin panel." if not error_msg
else error_msg,
previous_source=previous_source,
new_source="audio_search",
cleared_data=cleared_keys,
new_status=new_status,
search_results_count=search_results_count,
error=error_msg,
)
+13 -15
View File
@@ -236,6 +236,14 @@ def extract_request_metadata(request: Request, created_from: str = "audio_search
}
async def _trigger_workers_parallel(job_id: str) -> None:
"""Trigger both audio and lyrics workers in parallel."""
await asyncio.gather(
worker_service.trigger_audio_worker(job_id),
worker_service.trigger_lyrics_worker(job_id)
)
def _extract_gcs_path(filepath: str) -> str:
"""
Extract the path portion from a GCS path or gs:// URL.
@@ -426,31 +434,21 @@ async def _download_and_start_processing(
except Exception as e:
logger.warning(f"Failed to clean up temp files: {e}")
# Update job with GCS path
# Update job with GCS path and transition to DOWNLOADING
job_manager.update_job(job_id, {
'input_media_gcs_path': audio_gcs_path,
'filename': filename,
})
# Transition from DOWNLOADING_AUDIO to DOWNLOADING and trigger workers
# Note: Can't use start_job_processing() here because that expects PENDING status,
# but we're in DOWNLOADING_AUDIO state from the audio search flow.
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=15,
message="Audio downloaded, starting processing"
)
# Trigger workers in background
async def trigger_workers():
worker_service = get_worker_service()
await asyncio.gather(
worker_service.trigger_audio_worker(job_id),
worker_service.trigger_lyrics_worker(job_id)
)
background_tasks.add_task(trigger_workers)
# Trigger workers
background_tasks.add_task(_trigger_workers_parallel, job_id)
return {
'selected_index': selection_index,
+48 -34
View File
@@ -256,6 +256,19 @@ ALLOWED_IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp'}
ALLOWED_FONT_EXTENSIONS = {'.ttf', '.otf', '.woff', '.woff2'}
async def _trigger_workers_parallel(job_id: str) -> None:
"""
Trigger both audio and lyrics workers in parallel.
FastAPI's BackgroundTasks runs async tasks sequentially, so we use
asyncio.gather to ensure both workers start at the same time.
"""
await asyncio.gather(
worker_service.trigger_audio_worker(job_id),
worker_service.trigger_lyrics_worker(job_id)
)
async def _trigger_audio_worker_only(job_id: str) -> None:
"""
Trigger only the audio worker.
@@ -742,17 +755,18 @@ async def upload_and_create_job(
if style_assets:
logger.info(f"Style assets: {list(style_assets.keys())}")
# Use centralized start_job_processing which handles:
# 1. Validates job exists and has input_media_gcs_path
# 2. Transitions to DOWNLOADING (raises InvalidStateTransitionError if invalid)
# 3. Triggers audio + lyrics workers in parallel
background_tasks.add_task(
job_manager.start_job_processing,
job_id,
5, # progress
"Files uploaded, preparing to process"
# Transition job to DOWNLOADING state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=5,
message="Files uploaded, preparing to process"
)
# Trigger workers in parallel using asyncio.gather
# (FastAPI's BackgroundTasks runs async tasks sequentially)
background_tasks.add_task(_trigger_workers_parallel, job_id)
# Build distribution services info for response
distribution_services: Dict[str, Any] = {}
@@ -1291,17 +1305,17 @@ async def mark_uploads_complete(
logger.info(f"Validated uploads for job {job_id}: {body.uploaded_files}")
# Use centralized start_job_processing which handles:
# 1. Validates job exists and has input_media_gcs_path
# 2. Transitions to DOWNLOADING (raises InvalidStateTransitionError if invalid)
# 3. Triggers audio + lyrics workers in parallel
background_tasks.add_task(
job_manager.start_job_processing,
job_id,
5, # progress
"Files uploaded, preparing to process"
# Transition job to DOWNLOADING state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=5,
message="Files uploaded, preparing to process"
)
# Trigger workers in parallel
background_tasks.add_task(_trigger_workers_parallel, job_id)
# Get distribution services info for response
credential_manager = get_credential_manager()
distribution_services: Dict[str, Any] = {}
@@ -1574,6 +1588,14 @@ async def create_job_from_url(
if title:
logger.info(f" Title: {title}")
# Transition job to DOWNLOADING state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=5,
message="Starting audio download from URL"
)
# For YouTube URLs, download audio NOW using YouTubeDownloadService
# This uses remote flacfetch (if configured) to avoid bot detection on Cloud Run
if _is_youtube_url(body.url):
@@ -1595,16 +1617,14 @@ async def create_job_from_url(
logger.info(f"YouTube audio downloaded to GCS: {audio_gcs_path}")
# Use centralized start_job_processing which handles:
# 1. Validates job exists and has input_media_gcs_path
# 2. Transitions PENDING → DOWNLOADING (raises if invalid)
# 3. Triggers audio + lyrics workers in parallel
background_tasks.add_task(
job_manager.start_job_processing,
job_id,
10, # progress
"Audio downloaded, starting processing"
# Now trigger both workers in parallel (audio already downloaded)
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=10,
message="Audio downloaded, starting processing"
)
background_tasks.add_task(_trigger_workers_parallel, job_id)
except YouTubeDownloadError as e:
logger.error(f"YouTube download failed: {e}")
@@ -1614,14 +1634,8 @@ async def create_job_from_url(
detail=f"YouTube download failed: {e}"
)
else:
# For non-YouTube URLs, transition to DOWNLOADING and trigger audio worker
# For non-YouTube URLs, trigger ONLY audio worker first
# The audio worker will download the URL and trigger lyrics worker after
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=5,
message="Starting audio download from URL"
)
background_tasks.add_task(_trigger_audio_worker_only, job_id)
return CreateJobFromUrlResponse(
-66
View File
@@ -16,9 +16,6 @@ from backend.services.encoding_service import get_encoding_service
from backend.services.spacy_preloader import get_preloaded_model, is_model_preloaded
from backend.services.nltk_preloader import get_preloaded_cmudict, is_cmudict_preloaded
from backend.services.langfuse_preloader import get_preloaded_langfuse_handler, is_langfuse_preloaded, is_langfuse_configured
from backend.services.job_health_service import check_job_consistency_detailed
from backend.services.job_manager import JobManager
from backend.models.job import JobStatus
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -433,69 +430,6 @@ async def preload_status() -> Dict[str, Any]:
}
@router.get("/health/job-consistency")
async def job_consistency_check(
status: Optional[str] = None,
limit: int = 50
) -> Dict[str, Any]:
"""
Check job consistency across the system.
This endpoint identifies jobs with state inconsistencies that may indicate
bugs in the state machine or trigger logic. Use this to find stuck jobs
or jobs in invalid states.
Args:
status: Optional filter by job status (e.g., "pending", "downloading")
limit: Maximum number of jobs to check (default 50)
Returns:
Dictionary with:
- total_checked: Number of jobs checked
- inconsistent_count: Number of jobs with issues
- inconsistent_jobs: List of job details with their issues
- summary: Summary of issue types found
"""
job_manager = JobManager()
# Query jobs to check
query_params = {}
if status:
try:
job_status = JobStatus(status)
query_params['status'] = job_status
except ValueError:
return {
"error": f"Invalid status: {status}",
"valid_statuses": [s.value for s in JobStatus]
}
# Get jobs from Firestore
jobs = job_manager.list_jobs(limit=limit, **query_params)
# Check each job for consistency
inconsistent_jobs = []
issue_summary = {}
for job in jobs:
result = check_job_consistency_detailed(job)
if not result['is_healthy']:
inconsistent_jobs.append(result)
# Count issue types
for issue in result['issues']:
issue_type = issue.split(':')[0] if ':' in issue else issue
issue_summary[issue_type] = issue_summary.get(issue_type, 0) + 1
return {
"total_checked": len(jobs),
"inconsistent_count": len(inconsistent_jobs),
"inconsistent_jobs": inconsistent_jobs,
"summary": issue_summary,
"status_filter": status,
"limit": limit,
}
@router.get("/readiness")
async def readiness_check() -> Dict[str, str]:
"""Readiness check endpoint for Cloud Run."""
+9 -16
View File
@@ -65,7 +65,6 @@ from backend.services.youtube_download_service import (
get_youtube_download_service,
YouTubeDownloadError,
)
from backend.exceptions import InvalidStateTransitionError
logger = logging.getLogger(__name__)
@@ -477,12 +476,14 @@ async def _handle_made_for_you_order(
"""
from backend.models.job import JobCreate, JobStatus
from backend.services.job_manager import JobManager
from backend.services.worker_service import get_worker_service
from backend.services.audio_search_service import (
get_audio_search_service,
NoResultsError,
AudioSearchError,
)
from backend.services.storage_service import StorageService
import asyncio
import tempfile
import os
@@ -500,6 +501,7 @@ async def _handle_made_for_you_order(
try:
job_manager = JobManager()
worker_service = get_worker_service()
storage_service = StorageService()
# Apply default theme (Nomad) - same as audio_search endpoint
@@ -600,22 +602,13 @@ async def _handle_made_for_you_order(
'filename': os.path.basename(audio_gcs_path),
})
logger.info(f"Job {job_id}: YouTube audio downloaded to {audio_gcs_path}")
logger.info(f"Job {job_id}: YouTube audio downloaded to {audio_gcs_path}, triggering workers")
# Use centralized helper that handles transition + worker triggers
# This ensures consistent state machine flow and raises on invalid transitions
try:
await job_manager.start_job_processing(job_id)
except InvalidStateTransitionError as e:
logger.error(f"Job {job_id}: Failed to start processing - invalid state transition: {e}")
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.FAILED,
progress=0,
message=f"Failed to start processing: {str(e)}",
raise_on_invalid=False # Don't raise if already failed
)
raise
# Now trigger both workers in parallel (audio already downloaded)
await asyncio.gather(
worker_service.trigger_audio_worker(job_id),
worker_service.trigger_lyrics_worker(job_id)
)
except YouTubeDownloadError as e:
logger.error(f"Job {job_id}: YouTube download failed: {e}")
-36
View File
@@ -64,39 +64,3 @@ class IPBlockedError(Exception):
self.message = message
self.ip_address = ip_address
super().__init__(message)
class InvalidStateTransitionError(Exception):
"""
Raised when an invalid job state transition is attempted.
This exception is raised by JobManager.transition_to_state() when the
requested transition is not allowed by the state machine defined in
STATE_TRANSITIONS.
Example: Trying to transition from PENDING to GENERATING_SCREENS is invalid
because PENDING can only transition to DOWNLOADING, SEARCHING_AUDIO, FAILED,
or CANCELLED.
Attributes:
message: Human-readable error message
job_id: The job ID that failed to transition
from_status: Current status of the job
to_status: Attempted target status
valid_transitions: List of valid transitions from current status
"""
def __init__(
self,
message: str,
job_id: str = "",
from_status: str = "",
to_status: str = "",
valid_transitions: list = None
):
self.message = message
self.job_id = job_id
self.from_status = from_status
self.to_status = to_status
self.valid_transitions = valid_transitions or []
super().__init__(message)
-265
View File
@@ -1,265 +0,0 @@
#!/usr/bin/env python3
"""
Generate a Mermaid state diagram from the job state machine.
This script reads the STATE_TRANSITIONS dictionary from models/job.py
and generates a Mermaid diagram that can be embedded in documentation.
Usage:
python scripts/generate_state_diagram.py > docs/state-machine.mmd
# Or to update ARCHITECTURE.md directly:
python scripts/generate_state_diagram.py --update-docs
Created as part of state machine robustness improvements (2026-02-02).
"""
import sys
import argparse
from pathlib import Path
# Add backend to path for imports
backend_dir = Path(__file__).parent.parent
sys.path.insert(0, str(backend_dir))
# Import the state machine data
# Note: Must be run from an environment where karaoke_gen is installed
try:
from models.job import STATE_TRANSITIONS, JobStatus
except ImportError:
# Fallback: define the state machine inline for documentation purposes
from enum import Enum
class JobStatus(str, Enum):
PENDING = "pending"
SEARCHING_AUDIO = "searching_audio"
AWAITING_AUDIO_SELECTION = "awaiting_audio_selection"
DOWNLOADING_AUDIO = "downloading_audio"
DOWNLOADING = "downloading"
SEPARATING_STAGE1 = "separating_stage1"
SEPARATING_STAGE2 = "separating_stage2"
AUDIO_COMPLETE = "audio_complete"
TRANSCRIBING = "transcribing"
CORRECTING = "correcting"
LYRICS_COMPLETE = "lyrics_complete"
GENERATING_SCREENS = "generating_screens"
APPLYING_PADDING = "applying_padding"
AWAITING_REVIEW = "awaiting_review"
IN_REVIEW = "in_review"
REVIEW_COMPLETE = "review_complete"
RENDERING_VIDEO = "rendering_video"
AWAITING_INSTRUMENTAL_SELECTION = "awaiting_instrumental_selection"
INSTRUMENTAL_SELECTED = "instrumental_selected"
GENERATING_VIDEO = "generating_video"
ENCODING = "encoding"
PACKAGING = "packaging"
UPLOADING = "uploading"
NOTIFYING = "notifying"
COMPLETE = "complete"
PREP_COMPLETE = "prep_complete"
FAILED = "failed"
CANCELLED = "cancelled"
QUEUED = "queued"
PROCESSING = "processing"
ERROR = "error"
READY_FOR_FINALIZATION = "ready_for_finalization"
FINALIZING = "finalizing"
STATE_TRANSITIONS = {
JobStatus.PENDING: [JobStatus.DOWNLOADING, JobStatus.SEARCHING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
JobStatus.SEARCHING_AUDIO: [JobStatus.AWAITING_AUDIO_SELECTION, JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED],
JobStatus.AWAITING_AUDIO_SELECTION: [JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
JobStatus.DOWNLOADING_AUDIO: [JobStatus.DOWNLOADING, JobStatus.FAILED],
JobStatus.DOWNLOADING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING, JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
JobStatus.SEPARATING_STAGE1: [JobStatus.SEPARATING_STAGE2, JobStatus.FAILED],
JobStatus.SEPARATING_STAGE2: [JobStatus.AUDIO_COMPLETE, JobStatus.FAILED],
JobStatus.AUDIO_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
JobStatus.TRANSCRIBING: [JobStatus.CORRECTING, JobStatus.FAILED],
JobStatus.CORRECTING: [JobStatus.LYRICS_COMPLETE, JobStatus.FAILED],
JobStatus.LYRICS_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
JobStatus.GENERATING_SCREENS: [JobStatus.APPLYING_PADDING, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
JobStatus.APPLYING_PADDING: [JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
JobStatus.AWAITING_REVIEW: [JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
JobStatus.IN_REVIEW: [JobStatus.REVIEW_COMPLETE, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
JobStatus.REVIEW_COMPLETE: [JobStatus.RENDERING_VIDEO, JobStatus.FAILED],
JobStatus.RENDERING_VIDEO: [JobStatus.INSTRUMENTAL_SELECTED, JobStatus.FAILED],
JobStatus.INSTRUMENTAL_SELECTED: [JobStatus.GENERATING_VIDEO, JobStatus.FAILED],
JobStatus.GENERATING_VIDEO: [JobStatus.ENCODING, JobStatus.FAILED],
JobStatus.ENCODING: [JobStatus.PACKAGING, JobStatus.COMPLETE, JobStatus.PREP_COMPLETE, JobStatus.FAILED],
JobStatus.PACKAGING: [JobStatus.UPLOADING, JobStatus.COMPLETE, JobStatus.PREP_COMPLETE, JobStatus.FAILED],
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
}
def generate_mermaid() -> str:
"""
Generate a Mermaid state diagram from STATE_TRANSITIONS.
Returns:
Mermaid diagram as a string
"""
lines = ["stateDiagram-v2"]
lines.append(" direction TB")
lines.append("")
# Group states by category for better visualization
categories = {
"Initial": [JobStatus.PENDING],
"Audio Search": [
JobStatus.SEARCHING_AUDIO,
JobStatus.AWAITING_AUDIO_SELECTION,
JobStatus.DOWNLOADING_AUDIO,
],
"Processing": [
JobStatus.DOWNLOADING,
JobStatus.SEPARATING_STAGE1,
JobStatus.SEPARATING_STAGE2,
JobStatus.AUDIO_COMPLETE,
JobStatus.TRANSCRIBING,
JobStatus.CORRECTING,
JobStatus.LYRICS_COMPLETE,
],
"Screens": [
JobStatus.GENERATING_SCREENS,
JobStatus.APPLYING_PADDING,
],
"Review": [
JobStatus.AWAITING_REVIEW,
JobStatus.IN_REVIEW,
JobStatus.REVIEW_COMPLETE,
],
"Video": [
JobStatus.RENDERING_VIDEO,
JobStatus.INSTRUMENTAL_SELECTED,
JobStatus.GENERATING_VIDEO,
JobStatus.ENCODING,
JobStatus.PACKAGING,
],
"Distribution": [
JobStatus.UPLOADING,
JobStatus.NOTIFYING,
],
"Terminal": [
JobStatus.COMPLETE,
JobStatus.PREP_COMPLETE,
JobStatus.FAILED,
JobStatus.CANCELLED,
],
}
# Add state definitions with notes for key states
lines.append(" %% State definitions")
key_states = {
JobStatus.PENDING: "Job created",
JobStatus.AWAITING_AUDIO_SELECTION: "⚠️ USER ACTION",
JobStatus.AWAITING_REVIEW: "⚠️ USER ACTION",
JobStatus.COMPLETE: "✅ Success",
JobStatus.FAILED: "❌ Error",
}
for status, note in key_states.items():
lines.append(f" {status.value}: {status.value}")
lines.append(f" note right of {status.value}: {note}")
lines.append("")
lines.append(" %% Transitions")
# Generate transitions
for from_status, to_statuses in STATE_TRANSITIONS.items():
for to_status in to_statuses:
# Skip deprecated statuses
if to_status in [JobStatus.QUEUED, JobStatus.PROCESSING, JobStatus.ERROR,
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING]:
continue
if from_status in [JobStatus.QUEUED, JobStatus.PROCESSING, JobStatus.ERROR,
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING]:
continue
lines.append(f" {from_status.value} --> {to_status.value}")
# Add start and end markers
lines.append("")
lines.append(" %% Start and end")
lines.append(" [*] --> pending")
lines.append(" complete --> [*]")
lines.append(" prep_complete --> [*]")
lines.append(" failed --> [*]")
lines.append(" cancelled --> [*]")
return "\n".join(lines)
def generate_markdown_section() -> str:
"""
Generate a markdown section with the state diagram.
Returns:
Markdown string with the diagram
"""
diagram = generate_mermaid()
return f"""## Job State Machine
The following diagram shows all valid state transitions for a karaoke generation job.
Key states:
- **PENDING**: Initial state when job is created
- **AWAITING_AUDIO_SELECTION**: Human selects audio source (for artist+title search)
- **AWAITING_REVIEW**: Human reviews lyrics and selects instrumental
- **COMPLETE**: All processing finished successfully
```mermaid
{diagram}
```
### Parallel Processing
Jobs use parallel processing for audio separation and lyrics transcription:
1. **DOWNLOADING** triggers both:
- Audio worker → SEPARATING_STAGE1 → SEPARATING_STAGE2 → AUDIO_COMPLETE
- Lyrics worker → TRANSCRIBING → CORRECTING → LYRICS_COMPLETE
2. When BOTH complete, the screens worker is triggered automatically.
### Human Review Points
Jobs wait for human input at these points:
- **AWAITING_AUDIO_SELECTION**: User selects audio source from search results
- **AWAITING_REVIEW**: User reviews/corrects lyrics AND selects instrumental
### Error Handling
- Any state can transition to **FAILED** on unrecoverable errors
- **CANCELLED** is set when user cancels the job
"""
def main():
parser = argparse.ArgumentParser(description="Generate state machine diagram")
parser.add_argument(
"--update-docs",
action="store_true",
help="Update ARCHITECTURE.md with the diagram"
)
parser.add_argument(
"--markdown",
action="store_true",
help="Output markdown section instead of raw Mermaid"
)
args = parser.parse_args()
if args.update_docs:
# TODO: Implement auto-update of ARCHITECTURE.md
print("Auto-update of ARCHITECTURE.md not yet implemented")
print("Please copy the output below to the appropriate section:")
print()
print(generate_markdown_section())
elif args.markdown:
print(generate_markdown_section())
else:
print(generate_mermaid())
if __name__ == "__main__":
main()
-221
View File
@@ -1,221 +0,0 @@
"""
Job health and consistency checking service.
Provides functions to detect state inconsistencies in jobs that might indicate
bugs in the state machine or trigger logic.
Created as part of state machine robustness improvements (2026-02-02).
"""
import logging
from typing import List, Dict, Any, Optional
from backend.models.job import Job, JobStatus
logger = logging.getLogger(__name__)
# Jobs in these statuses should NOT have audio_complete flag set
# (unless they're actively processing or beyond)
STATUSES_BEFORE_AUDIO_START = {
JobStatus.PENDING,
JobStatus.SEARCHING_AUDIO,
JobStatus.AWAITING_AUDIO_SELECTION,
}
# Jobs in these statuses should NOT have lyrics_complete flag set
STATUSES_BEFORE_LYRICS_START = {
JobStatus.PENDING,
JobStatus.SEARCHING_AUDIO,
JobStatus.AWAITING_AUDIO_SELECTION,
}
# Statuses where screens_progress.stage='complete' is valid
STATUSES_AFTER_SCREENS_COMPLETE = {
JobStatus.AWAITING_REVIEW,
JobStatus.IN_REVIEW,
JobStatus.REVIEW_COMPLETE,
JobStatus.RENDERING_VIDEO,
JobStatus.AWAITING_INSTRUMENTAL_SELECTION, # Legacy
JobStatus.INSTRUMENTAL_SELECTED,
JobStatus.GENERATING_VIDEO,
JobStatus.ENCODING,
JobStatus.PACKAGING,
JobStatus.UPLOADING,
JobStatus.NOTIFYING,
JobStatus.COMPLETE,
JobStatus.PREP_COMPLETE,
}
def check_job_consistency(job: Job) -> List[str]:
"""
Check for state inconsistencies in a job.
This function detects situations where state_data flags indicate
progress that doesn't match the job status - which usually indicates
a bug in the state machine or trigger logic.
Args:
job: Job object to check
Returns:
List of issue descriptions found. Empty list means no issues.
"""
issues = []
state_data = job.state_data or {}
# Normalize status to JobStatus enum if it's a string
status = job.status
if isinstance(status, str):
try:
status = JobStatus(status)
except ValueError:
issues.append(f"invalid_status: '{job.status}' is not a valid JobStatus")
return issues
# Check: audio_complete=True but status still in pre-audio states
if state_data.get('audio_complete') and status in STATUSES_BEFORE_AUDIO_START:
issues.append(
f"audio_complete_status_mismatch: audio_complete=True but status={status.value}"
)
# Check: lyrics_complete=True but status still in pre-lyrics states
if state_data.get('lyrics_complete') and status in STATUSES_BEFORE_LYRICS_START:
issues.append(
f"lyrics_complete_status_mismatch: lyrics_complete=True but status={status.value}"
)
# Check: screens_progress.stage='complete' but status not in post-screens states
screens_progress = state_data.get('screens_progress', {})
if isinstance(screens_progress, dict) and screens_progress.get('stage') == 'complete':
if status not in STATUSES_AFTER_SCREENS_COMPLETE:
issues.append(
f"screens_complete_status_mismatch: screens_progress.stage=complete but status={status.value}"
)
# Check: job has input_media_gcs_path but is stuck at pending
# This was the exact bug that got jobs 06cfea29 and 984da08b stuck
if job.input_media_gcs_path and status == JobStatus.PENDING:
# This could be legitimate if job was just created and workers haven't started
# But if it's been more than a few minutes, it's likely stuck
# We flag it as a warning, not an error
issues.append(
f"pending_with_input_media: has input_media_gcs_path but status=pending (may be stuck)"
)
return issues
def check_job_consistency_detailed(job: Job) -> Dict[str, Any]:
"""
Check job consistency with detailed information for debugging.
Returns a dictionary with:
- job_id: The job ID
- status: Current job status
- issues: List of issue descriptions
- state_data_summary: Summary of relevant state_data fields
- is_healthy: Boolean indicating if no issues were found
Args:
job: Job object to check
Returns:
Dictionary with detailed consistency information
"""
issues = check_job_consistency(job)
state_data = job.state_data or {}
return {
"job_id": job.job_id,
"status": job.status.value if hasattr(job.status, 'value') else str(job.status),
"issues": issues,
"state_data_summary": {
"audio_complete": state_data.get('audio_complete', False),
"lyrics_complete": state_data.get('lyrics_complete', False),
"screens_progress_stage": state_data.get('screens_progress', {}).get('stage'),
"has_input_media_gcs_path": bool(job.input_media_gcs_path),
},
"is_healthy": len(issues) == 0,
}
def get_worker_valid_statuses() -> Dict[str, List[JobStatus]]:
"""
Get mapping of worker names to their valid job statuses.
This is used by workers to validate they should run for a given job.
Returns:
Dictionary mapping worker name to list of valid JobStatus values
"""
return {
"audio_worker": [
JobStatus.DOWNLOADING,
JobStatus.DOWNLOADING_AUDIO,
JobStatus.SEPARATING_STAGE1,
JobStatus.SEPARATING_STAGE2,
],
"lyrics_worker": [
JobStatus.DOWNLOADING,
JobStatus.TRANSCRIBING,
JobStatus.CORRECTING,
],
"screens_worker": [
# Screens worker can be triggered when both audio+lyrics complete
# which can happen in any of these states
JobStatus.DOWNLOADING,
JobStatus.AUDIO_COMPLETE,
JobStatus.LYRICS_COMPLETE,
# Also valid if re-running after a failure
JobStatus.GENERATING_SCREENS,
],
"video_worker": [
JobStatus.INSTRUMENTAL_SELECTED,
JobStatus.REVIEW_COMPLETE,
JobStatus.RENDERING_VIDEO,
JobStatus.GENERATING_VIDEO,
],
"render_video_worker": [
JobStatus.REVIEW_COMPLETE,
JobStatus.RENDERING_VIDEO,
],
}
def validate_worker_can_run(worker_name: str, job: Job) -> Optional[str]:
"""
Validate that a worker should run for the given job.
Args:
worker_name: Name of the worker (e.g., "screens_worker")
job: Job object to validate
Returns:
None if valid, or an error message string if invalid
"""
valid_statuses_map = get_worker_valid_statuses()
if worker_name not in valid_statuses_map:
# Unknown worker, can't validate
return None
valid_statuses = valid_statuses_map[worker_name]
# Normalize status
status = job.status
if isinstance(status, str):
try:
status = JobStatus(status)
except ValueError:
return f"Invalid job status: {job.status}"
if status not in valid_statuses:
valid_names = [s.value for s in valid_statuses]
return (
f"{worker_name} called but job status is {status.value}, "
f"expected one of {valid_names}. This may indicate a bug in the trigger logic."
)
return None
+17 -113
View File
@@ -14,7 +14,7 @@ from datetime import datetime
from typing import Optional, Dict, Any, List
from backend.config import settings
from backend.exceptions import RateLimitExceededError, InvalidStateTransitionError
from backend.exceptions import RateLimitExceededError
from backend.models.job import Job, JobStatus, JobCreate, STATE_TRANSITIONS
from backend.models.worker_log import WorkerLogEntry
from backend.services.firestore_service import FirestoreService
@@ -316,87 +316,53 @@ class JobManager:
self.firestore.delete_job(job_id)
logger.info(f"Deleted job {job_id}")
def validate_state_transition(
self,
job_id: str,
new_status: JobStatus,
raise_on_invalid: bool = True
) -> bool:
def validate_state_transition(self, job_id: str, new_status: JobStatus) -> bool:
"""
Validate that a state transition is legal.
Args:
job_id: Job ID
new_status: Target state
raise_on_invalid: If True, raise InvalidStateTransitionError on invalid
transitions. If False, return False (legacy behavior).
Returns:
True if transition is valid
Raises:
InvalidStateTransitionError: If transition is invalid and raise_on_invalid=True
ValueError: If job not found and raise_on_invalid=True
True if transition is valid, False otherwise
"""
job = self.get_job(job_id)
if not job:
error_msg = f"Job {job_id} not found"
logger.error(error_msg)
if raise_on_invalid:
raise ValueError(error_msg)
logger.error(f"Job {job_id} not found")
return False
current_status = job.status
valid_transitions = STATE_TRANSITIONS.get(current_status, [])
if new_status not in valid_transitions:
error_msg = (
logger.error(
f"Invalid state transition for job {job_id}: "
f"{current_status} -> {new_status}. "
f"Valid transitions: {[s.value for s in valid_transitions]}"
f"Valid transitions: {valid_transitions}"
)
logger.error(error_msg)
if raise_on_invalid:
raise InvalidStateTransitionError(
message=error_msg,
job_id=job_id,
from_status=current_status,
to_status=new_status.value if hasattr(new_status, 'value') else str(new_status),
valid_transitions=[s.value for s in valid_transitions]
)
return False
return True
def transition_to_state(
self,
job_id: str,
new_status: JobStatus,
progress: Optional[int] = None,
message: Optional[str] = None,
state_data_updates: Optional[Dict[str, Any]] = None,
raise_on_invalid: bool = True
state_data_updates: Optional[Dict[str, Any]] = None
) -> bool:
"""
Transition job to new state with validation.
Args:
job_id: Job ID
new_status: Target state
progress: Progress percentage (0-100)
message: Timeline message
state_data_updates: Updates to state_data field
raise_on_invalid: If True (default), raise InvalidStateTransitionError
on invalid transitions. If False, return False silently.
Returns:
True if transition succeeded, False if failed (only when raise_on_invalid=False)
Raises:
InvalidStateTransitionError: If transition is invalid and raise_on_invalid=True
True if transition succeeded, False otherwise
"""
if not self.validate_state_transition(job_id, new_status, raise_on_invalid=raise_on_invalid):
if not self.validate_state_transition(job_id, new_status):
return False
updates = {
@@ -802,69 +768,7 @@ class JobManager:
lyrics_complete = job.state_data.get('lyrics_complete', False)
return audio_complete and lyrics_complete
async def start_job_processing(
self,
job_id: str,
progress: int = 15,
message: str = "Starting audio and lyrics processing"
) -> None:
"""
Transition job to DOWNLOADING and trigger audio + lyrics workers.
This is the single entry point for starting job processing after audio
is available. All handlers (file_upload, webhook, audio_search) should
use this method to ensure consistent state transitions.
The method:
1. Validates job exists and has input_media_gcs_path
2. Transitions to DOWNLOADING status (raises if invalid)
3. Triggers audio and lyrics workers in parallel
Args:
job_id: Job ID to start processing
progress: Progress percentage to set (default 15)
message: Timeline message (default "Starting audio and lyrics processing")
Raises:
InvalidStateTransitionError: If job can't transition to DOWNLOADING
ValueError: If job not found or missing input_media_gcs_path
Example:
# After downloading audio to GCS:
job_manager.update_job(job_id, {'input_media_gcs_path': audio_path})
await job_manager.start_job_processing(job_id)
"""
import asyncio
from backend.services.worker_service import get_worker_service
job = self.get_job(job_id)
if not job:
raise ValueError(f"Job {job_id} not found")
if not job.input_media_gcs_path:
raise ValueError(
f"Job {job_id} missing input_media_gcs_path. "
"Audio must be downloaded to GCS before starting processing."
)
# Transition to DOWNLOADING - this will raise if invalid
self.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=progress,
message=message
)
# Trigger both workers in parallel
worker_service = get_worker_service()
await asyncio.gather(
worker_service.trigger_audio_worker(job_id),
worker_service.trigger_lyrics_worker(job_id)
)
logger.info(f"Job {job_id}: Started processing (audio + lyrics workers triggered)")
def mark_audio_complete(self, job_id: str) -> None:
"""
Mark audio processing as complete and check if can proceed.
@@ -1,122 +0,0 @@
"""
Integration tests for admin job reset endpoint.
These tests use the real Firestore emulator and real JobManager,
catching bugs that unit tests with mocks might miss (e.g., incorrect
assumptions about return values).
Run with: scripts/run-emulator-tests.sh
"""
import pytest
from datetime import datetime, UTC
from backend.tests.emulator.conftest import emulators_running
# Skip all tests if emulators aren't running
pytestmark = pytest.mark.skipif(
not emulators_running(),
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
)
# Only import if emulators are running
if emulators_running():
from backend.models.job import Job, JobStatus
from backend.services.job_manager import JobManager
from backend.services.firestore_service import FirestoreService
class TestUpdateJobReturnValue:
"""
Critical tests documenting that update_job() returns None, not boolean.
This test class exists because a production bug (PR #371) was caused by
assuming update_job() returns a boolean success indicator. The real API
returns None on success and raises on failure. Mocks that return True
hide this behavior.
"""
@pytest.fixture
def job_manager(self):
"""Create a real JobManager using emulator."""
return JobManager()
@pytest.fixture
def firestore_service(self):
"""Create a real FirestoreService using emulator."""
return FirestoreService()
@pytest.fixture
def test_job(self, firestore_service):
"""Create a test job for verification."""
job_id = f"update-return-test-{datetime.now(UTC).timestamp()}"
job = Job(
job_id=job_id,
status=JobStatus.COMPLETE,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
artist="Test Artist",
title="Test Song",
)
firestore_service.create_job(job)
yield job
# Cleanup
try:
firestore_service.delete_job(job_id)
except Exception:
pass
def test_update_job_returns_none_not_boolean(self, job_manager, test_job):
"""
Verify update_job() returns None, not a boolean.
This test exists because a production bug was introduced when code
assumed update_job() returns a boolean. The method actually returns
None on success and raises an exception on failure.
Bug pattern that this test prevents:
```python
success = job_manager.update_job(job_id, updates)
if not success: # BUG: None is falsy, so this is always True!
raise HTTPException(status_code=500, ...)
```
Correct pattern:
```python
# update_job() raises on failure, returns None on success
job_manager.update_job(job_id, updates)
```
"""
# update_job() should return None, not True/False
result = job_manager.update_job(
test_job.job_id,
{"status": JobStatus.PENDING.value}
)
# The actual return value is None, not True
assert result is None, (
"update_job() should return None, not a boolean. "
"Code checking 'if not update_job(...)' will always be True!"
)
def test_update_job_on_nonexistent_job(self, job_manager):
"""
Document update_job() behavior for non-existent jobs.
In production Firestore, update() on non-existent doc raises NotFound.
The key insight for preventing the admin reset bug is understanding
that update_job() returns None (not boolean) on success.
"""
try:
result = job_manager.update_job(
"nonexistent-job-id-12345",
{"status": JobStatus.PENDING.value}
)
# If no exception raised (emulator quirk), verify return is still None
assert result is None, "update_job should return None even on non-existent doc"
except Exception:
# Exception is the expected behavior in production Firestore
pass
@@ -1,69 +0,0 @@
"""
Integration tests for admin job reset using REAL job fixture from production.
This test validates that update_job() returns None, not a boolean
(the root cause of PR #371 bug).
Uses actual data from production job 984da08b which needed admin reset.
Run with: scripts/run-emulator-tests.sh
"""
import pytest
from datetime import datetime, UTC
from backend.tests.emulator.conftest import emulators_running
pytestmark = pytest.mark.skipif(
not emulators_running(),
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
)
if emulators_running():
from backend.models.job import Job, JobStatus
from backend.services.job_manager import JobManager
class TestAdminResetBugFix:
"""
Tests that prove the PR #371 bug fix works using real emulator.
The bug: admin reset endpoint returned 500 because:
success = job_manager.update_job(job_id, updates)
if not success: # BUG: update_job returns None, not True!
raise HTTPException(500)
"""
def test_update_job_returns_none_not_boolean(self):
"""
Prove that update_job() returns None, not a boolean.
This is the root cause of the bug. Any code that does
'if not update_job(...)' will always be True.
"""
jm = JobManager()
job_id = f"test-{datetime.now(UTC).timestamp()}"
job = Job(
job_id=job_id,
status=JobStatus.COMPLETE,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
artist="Mark Mallman",
title="Minneapolis",
)
try:
jm.firestore.create_job(job)
# THE KEY TEST: update_job returns None
result = jm.update_job(job_id, {"status": "pending"})
assert result is None, (
"update_job() must return None. "
"Code checking 'if not success:' would always fail!"
)
finally:
try:
jm.firestore.delete_job(job_id)
except Exception:
pass
@@ -218,16 +218,16 @@ class TestCountdownStateTransitions:
"""
Countdown state should survive job state transitions.
When job moves from GENERATING_VIDEO to ENCODING,
When job moves from RENDERING_VIDEO to ENCODING,
the lyrics_metadata should remain intact.
"""
job_id = f"transition-test-{datetime.now(UTC).timestamp()}"
try:
# Create job in GENERATING_VIDEO state (valid predecessor to ENCODING)
# Create job in RENDERING_VIDEO state
job = Job(
job_id=job_id,
status=JobStatus.GENERATING_VIDEO,
status=JobStatus.RENDERING_VIDEO,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
artist="Test",
@@ -241,7 +241,7 @@ class TestCountdownStateTransitions:
)
firestore_service.create_job(job)
# Transition to ENCODING (valid transition from GENERATING_VIDEO)
# Transition to ENCODING
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.ENCODING,
@@ -229,12 +229,6 @@ class TestMadeForYouYouTubeOrderFlow:
mock_job.job_id = "test-job-workers"
mock_job_manager.create_job.return_value = mock_job
# start_job_processing is async - mock it to call workers directly
async def mock_start_job_processing(job_id, progress=15, message=""):
await mock_worker_service_local.trigger_audio_worker(job_id)
await mock_worker_service_local.trigger_lyrics_worker(job_id)
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
patch('backend.services.storage_service.StorageService'), \
@@ -251,7 +245,7 @@ class TestMadeForYouYouTubeOrderFlow:
email_service=mock_email_service,
)
# Verify workers were triggered via start_job_processing
# Verify workers were triggered
mock_worker_service_local.trigger_audio_worker.assert_called_once_with(mock_job.job_id)
mock_worker_service_local.trigger_lyrics_worker.assert_called_once_with(mock_job.job_id)
@@ -275,9 +269,6 @@ class TestMadeForYouYouTubeOrderFlow:
mock_job.job_id = "test-job-notifications"
mock_job_manager.create_job.return_value = mock_job
# start_job_processing is async - mock it as simple AsyncMock
mock_job_manager.start_job_processing = AsyncMock()
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
patch('backend.services.storage_service.StorageService'), \
@@ -398,12 +389,6 @@ class TestMadeForYouYouTubeOrderFlow:
mock_job.job_id = "test-job-no-search"
mock_job_manager.create_job.return_value = mock_job
# start_job_processing is async - mock it to call workers directly
async def mock_start_job_processing(job_id, progress=15, message=""):
await mock_worker_service_local.trigger_audio_worker(job_id)
await mock_worker_service_local.trigger_lyrics_worker(job_id)
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
patch('backend.services.audio_search_service.get_audio_search_service') as mock_get_audio, \
@@ -427,7 +412,7 @@ class TestMadeForYouYouTubeOrderFlow:
# Audio search should NOT be called for YouTube URL orders
mock_audio_service.search.assert_not_called()
# Workers should be triggered directly via start_job_processing
# Workers should be triggered directly
mock_worker_service_local.trigger_audio_worker.assert_called_once()
@@ -563,12 +548,6 @@ class TestMadeForYouWebhookIntegration:
mock_job.job_id = "test-job-webhook-flow"
mock_job_manager.create_job.return_value = mock_job
# start_job_processing is async - mock it to call workers directly
async def mock_start_job_processing(job_id, progress=15, message=""):
await mock_worker_service_local.trigger_audio_worker(job_id)
await mock_worker_service_local.trigger_lyrics_worker(job_id)
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
patch('backend.services.storage_service.StorageService'), \
@@ -594,7 +573,7 @@ class TestMadeForYouWebhookIntegration:
# 2. Verify YouTube download was called
mock_youtube_download_service.download.assert_called_once()
# 3. Verify workers were triggered via start_job_processing
# 3. Verify workers were triggered
mock_worker_service_local.trigger_audio_worker.assert_called_once()
mock_worker_service_local.trigger_lyrics_worker.assert_called_once()
@@ -1,348 +0,0 @@
"""
Integration tests for the job state machine flow.
These tests verify complete state transitions without mocking the state machine,
ensuring the flow works correctly end-to-end (with mocked external services).
Created as part of state machine robustness improvements (2026-02-02).
"""
import pytest
from datetime import datetime, UTC
from unittest.mock import Mock, AsyncMock, patch, MagicMock
# Mock Firestore before importing JobManager
import sys
sys.modules['google.cloud.firestore'] = MagicMock()
from backend.models.job import Job, JobStatus
from backend.services.job_manager import JobManager, InvalidStateTransitionError
@pytest.fixture
def mock_firestore_service():
"""Mock FirestoreService."""
with patch('backend.services.job_manager.FirestoreService') as mock:
service = Mock()
mock.return_value = service
yield service
@pytest.fixture
def job_manager(mock_firestore_service):
"""Create JobManager instance with mocked Firestore."""
return JobManager()
class TestMadeForYouYouTubeFlow:
"""
Integration tests for Made-For-You YouTube order flow.
This was the flow that caused jobs 06cfea29 and 984da08b to get stuck.
The fix ensures proper state transitions from PENDING -> DOWNLOADING.
"""
@pytest.mark.asyncio
async def test_complete_flow_pending_to_downloading(self, job_manager, mock_firestore_service):
"""
Test the flow from PENDING to DOWNLOADING for YouTube orders.
This verifies the fix for the bug where jobs got stuck at PENDING
because start_job_processing wasn't called properly.
"""
# Create initial job in PENDING state
job = Job(
job_id="test-mfy-001",
artist="Test Artist",
title="Test Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path="gs://bucket/test.mp3",
state_data={},
)
# Access the firestore through the job_manager's instance
job_manager.firestore.get_job.return_value = job
job_manager.firestore.update_job_status.return_value = True
# Mock worker service
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
mock_worker = Mock()
mock_worker.trigger_audio_worker = AsyncMock()
mock_worker.trigger_lyrics_worker = AsyncMock()
mock_get_worker.return_value = mock_worker
# Call start_job_processing - this is what _handle_made_for_you_order now uses
await job_manager.start_job_processing("test-mfy-001")
# Verify transition to DOWNLOADING was called
job_manager.firestore.update_job_status.assert_called()
call_kwargs = job_manager.firestore.update_job_status.call_args.kwargs
assert call_kwargs['status'] == JobStatus.DOWNLOADING
# Verify both workers were triggered
mock_worker.trigger_audio_worker.assert_called_once_with("test-mfy-001")
mock_worker.trigger_lyrics_worker.assert_called_once_with("test-mfy-001")
@pytest.mark.asyncio
async def test_fails_if_job_stuck_at_wrong_state(self, job_manager, mock_firestore_service):
"""
Test that starting processing fails if job is in wrong state.
Jobs that are already past PENDING (e.g., DOWNLOADING) shouldn't
be able to start processing again - this would indicate a bug.
"""
# Create job already in DOWNLOADING state
job = Job(
job_id="test-mfy-002",
artist="Test Artist",
title="Test Song",
status=JobStatus.DOWNLOADING, # Already past PENDING
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path="gs://bucket/test.mp3",
state_data={},
)
job_manager.firestore.get_job.return_value = job
# Attempting to start processing should raise InvalidStateTransitionError
# because DOWNLOADING -> DOWNLOADING is not a valid transition
with pytest.raises(InvalidStateTransitionError):
await job_manager.start_job_processing("test-mfy-002")
class TestParallelProcessingCoordination:
"""
Integration tests for parallel audio + lyrics processing coordination.
Both workers run in parallel and set flags when complete.
Screens worker is triggered when both are done.
"""
def test_audio_complete_updates_state_data(self, job_manager, mock_firestore_service):
"""Test that mark_audio_complete updates state_data correctly."""
job = Job(
job_id="test-parallel-001",
artist="Test Artist",
title="Test Song",
status=JobStatus.DOWNLOADING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={},
)
job_manager.firestore.get_job.return_value = job
job_manager.firestore.update_job.return_value = True
# Call mark_audio_complete
job_manager.mark_audio_complete("test-parallel-001")
# Verify update_job was called to set the audio_complete flag
job_manager.firestore.update_job.assert_called()
def test_lyrics_complete_updates_state_data(self, job_manager, mock_firestore_service):
"""Test that mark_lyrics_complete updates state_data correctly."""
job = Job(
job_id="test-parallel-002",
artist="Test Artist",
title="Test Song",
status=JobStatus.DOWNLOADING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={},
)
job_manager.firestore.get_job.return_value = job
job_manager.firestore.update_job.return_value = True
# Call mark_lyrics_complete
job_manager.mark_lyrics_complete("test-parallel-002")
# Verify update_job was called to set the lyrics_complete flag
job_manager.firestore.update_job.assert_called()
def test_both_complete_triggers_screens(self, job_manager, mock_firestore_service):
"""Test that screens worker is triggered when both complete."""
# First call: Job with audio already complete (for update_state_data)
# Second call: Job with both complete (for check_parallel_processing_complete)
job_audio_only = Job(
job_id="test-parallel-003",
artist="Test Artist",
title="Test Song",
status=JobStatus.DOWNLOADING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={'audio_complete': True},
)
job_both_complete = Job(
job_id="test-parallel-003",
artist="Test Artist",
title="Test Song",
status=JobStatus.DOWNLOADING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={'audio_complete': True, 'lyrics_complete': True},
)
# Return different jobs on subsequent calls
job_manager.firestore.get_job.side_effect = [job_audio_only, job_both_complete]
job_manager.firestore.update_job.return_value = True
# Mock the screens worker trigger
with patch.object(job_manager, '_trigger_screens_worker') as mock_trigger:
# Now lyrics completes
job_manager.mark_lyrics_complete("test-parallel-003")
# Verify screens worker was triggered (both complete)
mock_trigger.assert_called_once_with("test-parallel-003")
class TestStateTransitionValidation:
"""
Integration tests for state transition validation.
Ensures invalid transitions are rejected with clear errors.
"""
def test_valid_transition_succeeds(self, job_manager, mock_firestore_service):
"""Test that valid transitions succeed."""
job = Job(
job_id="test-transition-001",
artist="Test Artist",
title="Test Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={},
)
job_manager.firestore.get_job.return_value = job
job_manager.firestore.update_job_status.return_value = True
# PENDING -> DOWNLOADING is valid
result = job_manager.transition_to_state(
job_id="test-transition-001",
new_status=JobStatus.DOWNLOADING,
progress=10,
message="Starting processing"
)
assert result is True
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
"""Test that invalid transitions raise exception by default."""
job = Job(
job_id="test-transition-002",
artist="Test Artist",
title="Test Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={},
)
job_manager.firestore.get_job.return_value = job
# PENDING -> COMPLETE is invalid (skips many steps)
with pytest.raises(InvalidStateTransitionError) as exc_info:
job_manager.transition_to_state(
job_id="test-transition-002",
new_status=JobStatus.COMPLETE,
)
assert "pending" in str(exc_info.value).lower()
assert "complete" in str(exc_info.value).lower()
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
"""Test that invalid transitions return False when raise_on_invalid=False."""
job = Job(
job_id="test-transition-003",
artist="Test Artist",
title="Test Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={},
)
job_manager.firestore.get_job.return_value = job
# PENDING -> COMPLETE is invalid, but with raise_on_invalid=False
result = job_manager.transition_to_state(
job_id="test-transition-003",
new_status=JobStatus.COMPLETE,
raise_on_invalid=False
)
assert result is False
class TestAdminResetFlow:
"""
Integration tests for admin reset functionality.
Admin resets bypass normal state machine validation but must
properly clear state_data flags to avoid inconsistent state.
"""
def test_reset_to_pending_clears_parallel_flags(self, job_manager, mock_firestore_service):
"""
Test that resetting to pending clears audio_complete and lyrics_complete.
This was a bug where the Start button would reset to pending but
leave these flags set, causing inconsistent state.
"""
# This is tested via the admin endpoint, not directly in JobManager
# The test in test_admin_job_reset.py covers this
pass
class TestConsistencyWithHealthService:
"""
Integration tests that verify the job health service correctly
identifies inconsistencies in real job flows.
"""
def test_detects_stuck_job_scenario(self, job_manager, mock_firestore_service):
"""
Test that the health service would detect the exact bug that
caused jobs 06cfea29 and 984da08b to get stuck.
"""
from backend.services.job_health_service import check_job_consistency
# Simulate the stuck job scenario:
# - Job has input_media_gcs_path (YouTube download completed)
# - But status is still PENDING (transition never happened)
stuck_job = Job(
job_id="test-stuck-001",
artist="Test Artist",
title="Test Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path="gs://bucket/downloaded.mp3",
state_data={},
)
issues = check_job_consistency(stuck_job)
# Should detect the pending_with_input_media issue
assert len(issues) > 0
assert any("pending_with_input_media" in issue for issue in issues)
def test_detects_flag_status_mismatch(self, job_manager, mock_firestore_service):
"""
Test that the health service detects when flags don't match status.
"""
from backend.services.job_health_service import check_job_consistency
# Simulate: audio_complete=True but status still PENDING
# (would happen if mark_audio_complete ran but transition didn't)
inconsistent_job = Job(
job_id="test-inconsistent-001",
artist="Test Artist",
title="Test Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={'audio_complete': True},
)
issues = check_job_consistency(inconsistent_job)
# Should detect the audio_complete_status_mismatch issue
assert len(issues) > 0
assert any("audio_complete_status_mismatch" in issue for issue in issues)
+43 -126
View File
@@ -84,7 +84,7 @@ class TestResetJobToPending:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -104,7 +104,7 @@ class TestResetJobToPending:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -128,7 +128,7 @@ class TestResetJobToAwaitingAudioSelection:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -149,7 +149,7 @@ class TestResetJobToAwaitingReview:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -162,6 +162,27 @@ class TestResetJobToAwaitingReview:
assert data["new_status"] == "awaiting_review"
class TestResetJobToAwaitingInstrumentalSelection:
"""Tests for resetting job to AWAITING_INSTRUMENTAL_SELECTION state."""
def test_reset_to_awaiting_instrumental_selection(self, client, mock_job):
"""Test resetting a job to AWAITING_INSTRUMENTAL_SELECTION state."""
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
"/api/admin/jobs/test-job-123/reset",
json={"target_state": "awaiting_instrumental_selection"},
)
assert response.status_code == 200
data = response.json()
assert data["new_status"] == "awaiting_instrumental_selection"
class TestResetJobValidation:
"""Tests for reset endpoint validation."""
@@ -247,7 +268,7 @@ class TestResetJobLogging:
patch('backend.api.routes.admin.logger') as mock_logger:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -270,7 +291,7 @@ class TestResetJobTimeline:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -319,7 +340,7 @@ class TestResetJobClearsStateData:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -332,57 +353,12 @@ class TestResetJobClearsStateData:
data = response.json()
assert "cleared_data" in data or "state_data" in str(data)
def test_reset_to_pending_clears_parallel_processing_flags(self, client):
"""Test that resetting to PENDING clears audio_complete and lyrics_complete.
This is critical for proper re-processing - without clearing these flags,
workers may think audio/lyrics processing is already done.
"""
from backend.models.job import Job, JobStatus
from datetime import datetime, UTC
# Create a job with audio_complete and lyrics_complete set
mock_job = Job(
job_id="test-job-123",
status=JobStatus.COMPLETE,
artist="Test Artist",
title="Test Title",
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={
"audio_complete": True,
"lyrics_complete": True,
"audio_search_results": [{"source": "test"}],
}
)
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
"/api/admin/jobs/test-job-123/reset",
json={"target_state": "pending"},
)
assert response.status_code == 200
data = response.json()
# Verify the cleared_data includes the parallel processing flags
cleared_keys = data.get("cleared_data", [])
assert "audio_complete" in cleared_keys, \
f"audio_complete should be cleared. Cleared: {cleared_keys}"
assert "lyrics_complete" in cleared_keys, \
f"lyrics_complete should be cleared. Cleared: {cleared_keys}"
def test_reset_to_awaiting_review_preserves_audio_data(self, client, mock_job):
"""Test that resetting to AWAITING_REVIEW preserves audio stems."""
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -392,15 +368,12 @@ class TestResetJobClearsStateData:
assert response.status_code == 200
def test_rejects_deprecated_awaiting_instrumental_selection(self, client, mock_job):
"""Test that awaiting_instrumental_selection is no longer a valid reset target.
This state was deprecated in Jan 2026 when lyrics review and instrumental
selection were combined into a single review step.
"""
def test_reset_to_instrumental_preserves_review_data(self, client, mock_job):
"""Test that resetting to AWAITING_INSTRUMENTAL_SELECTION preserves review data."""
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -408,9 +381,7 @@ class TestResetJobClearsStateData:
json={"target_state": "awaiting_instrumental_selection"},
)
# Should be rejected as invalid
assert response.status_code == 400
assert "invalid" in response.json()["detail"].lower()
assert response.status_code == 200
class TestResetJobToInstrumentalSelected:
@@ -425,7 +396,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
# Mock worker service with async method
@@ -450,7 +421,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
# Mock worker service
@@ -478,7 +449,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
# Mock worker service failure
@@ -515,7 +486,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -550,7 +521,7 @@ class TestResetJobClearsErrorState:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -654,7 +625,7 @@ class TestResetIncludesEncodingProgress:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -679,7 +650,7 @@ class TestResetWorkerTriggerResponseFields:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -704,7 +675,7 @@ class TestResetWorkerTriggerResponseFields:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -730,7 +701,7 @@ class TestResetWorkerTriggerResponseFields:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -754,7 +725,7 @@ class TestResetWorkerTriggerResponseFields:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm.update_job.return_value = True
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -962,57 +933,3 @@ class TestTriggerWorkerEndpoint:
app.dependency_overrides[require_admin] = original_override
else:
app.dependency_overrides[require_admin] = get_mock_admin
class TestResetEndpointWithRealJobData:
"""
Tests using real production job data (job 984da08b) as fixture.
This validates the admin reset endpoint works correctly with realistic
state_data from a production job that needed manual admin reset.
"""
# Real state_data from production job 984da08b
REAL_JOB_STATE_DATA = {
"backing_vocals_analysis": {"has_audible_content": True},
"video_progress": {"stage": "running"}, # Stale progress
"lyrics_complete": True,
"audio_complete": True
}
def test_reset_endpoint_returns_200_with_none_return_value(self, client):
"""
Test that reset endpoint returns 200 (not 500) when update_job
returns None, matching the real API behavior.
Before fix: endpoint checked 'if not success:' which was always True
After fix: endpoint doesn't check return value
Uses real production job 984da08b data as fixture.
"""
# Mock job with real production data
mock_job = Mock(spec=Job)
mock_job.job_id = "984da08b"
mock_job.status = JobStatus.COMPLETE
mock_job.artist = "Mark Mallman"
mock_job.title = "Minneapolis"
mock_job.theme_id = "nomad"
mock_job.user_email = "madeforyou@nomadkaraoke.com"
mock_job.file_urls = {"stems": {}, "lyrics": {}}
mock_job.state_data = self.REAL_JOB_STATE_DATA.copy()
mock_job.timeline = []
with patch('backend.api.routes.admin.JobManager') as mock_jm:
mock_jm.return_value.get_job.return_value = mock_job
# KEY: return None (matching real API contract - not boolean!)
mock_jm.return_value.update_job.return_value = None
response = client.post(
"/api/admin/jobs/984da08b/reset",
json={"target_state": "pending"},
)
# This is the bug fix - before it was 500, now 200
assert response.status_code == 200
assert response.json()["status"] == "success"
assert response.json()["new_status"] == "pending"
+3 -81
View File
@@ -7,7 +7,7 @@ route logic without hitting real cloud services.
import pytest
import json
from datetime import datetime, UTC
from unittest.mock import MagicMock, AsyncMock, patch, Mock
from unittest.mock import MagicMock, AsyncMock, patch
from fastapi.testclient import TestClient
from io import BytesIO
@@ -131,91 +131,13 @@ class TestHealthRoutes:
assert data["queue_length"] == 5
class TestJobConsistencyEndpoint:
"""Tests for the /health/job-consistency endpoint."""
@pytest.fixture
def client(self):
"""Create test client."""
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
from backend.main import app
return TestClient(app)
def test_job_consistency_returns_empty_when_no_jobs(self, client):
"""Test endpoint returns empty results when no jobs exist."""
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.list_jobs.return_value = []
mock_jm_class.return_value = mock_jm
response = client.get("/api/health/job-consistency")
assert response.status_code == 200
data = response.json()
assert data["total_checked"] == 0
assert data["inconsistent_count"] == 0
assert data["inconsistent_jobs"] == []
def test_job_consistency_detects_issues(self, client):
"""Test endpoint detects inconsistent jobs."""
from backend.models.job import Job, JobStatus
from datetime import datetime, UTC
stuck_job = Job(
job_id="stuck-123",
artist="Test",
title="Song",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data={'audio_complete': True},
input_media_gcs_path="gs://bucket/audio.mp3",
)
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.list_jobs.return_value = [stuck_job]
mock_jm_class.return_value = mock_jm
response = client.get("/api/health/job-consistency")
assert response.status_code == 200
data = response.json()
assert data["total_checked"] == 1
assert data["inconsistent_count"] == 1
assert len(data["inconsistent_jobs"]) == 1
assert data["inconsistent_jobs"][0]["job_id"] == "stuck-123"
def test_job_consistency_filters_by_status(self, client):
"""Test endpoint filters by status parameter."""
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.list_jobs.return_value = []
mock_jm_class.return_value = mock_jm
response = client.get("/api/health/job-consistency?status=pending")
assert response.status_code == 200
data = response.json()
assert data["status_filter"] == "pending"
def test_job_consistency_rejects_invalid_status(self, client):
"""Test endpoint rejects invalid status parameter."""
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm_class.return_value = mock_jm
response = client.get("/api/health/job-consistency?status=invalid_status")
assert response.status_code == 200
data = response.json()
assert "error" in data
assert "Invalid status" in data["error"]
class TestJobRoutes:
"""Tests for jobs.py routes.
Note: These tests verify the route module structure.
Full integration tests are in test_api_integration.py.
"""
def test_jobs_router_exists(self):
"""Test jobs router can be imported."""
from backend.api.routes import jobs
@@ -269,12 +269,10 @@ class TestUrlJobWorkerSequencing:
async def test_parallel_worker_triggers_both_workers(self):
"""
_trigger_workers_parallel should trigger both workers (for uploaded files).
Note: This function now lives in jobs.py (direct job creation endpoint).
Most code paths use job_manager.start_job_processing() instead.
"""
from backend.api.routes.jobs import _trigger_workers_parallel
from backend.api.routes.file_upload import _trigger_workers_parallel
with patch('backend.api.routes.jobs.worker_service') as mock_ws:
with patch('backend.api.routes.file_upload.worker_service') as mock_ws:
mock_ws.trigger_audio_worker = AsyncMock()
mock_ws.trigger_lyrics_worker = AsyncMock()
-262
View File
@@ -1,262 +0,0 @@
"""
Tests for the job health service.
Tests consistency checks and worker validation functions.
"""
import pytest
from datetime import datetime, UTC
from backend.models.job import Job, JobStatus
from backend.services.job_health_service import (
check_job_consistency,
check_job_consistency_detailed,
validate_worker_can_run,
get_worker_valid_statuses,
)
def create_test_job(
job_id: str = "test-job-123",
status: JobStatus = JobStatus.PENDING,
state_data: dict = None,
input_media_gcs_path: str = None,
) -> Job:
"""Helper to create test Job objects."""
return Job(
job_id=job_id,
artist="Test Artist",
title="Test Song",
status=status,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
state_data=state_data or {},
input_media_gcs_path=input_media_gcs_path,
)
class TestCheckJobConsistency:
"""Tests for check_job_consistency function."""
def test_healthy_job_has_no_issues(self):
"""Test that a properly configured job has no issues."""
job = create_test_job(
status=JobStatus.DOWNLOADING,
state_data={},
)
issues = check_job_consistency(job)
assert issues == []
def test_detects_audio_complete_with_pending_status(self):
"""Test detection of audio_complete flag with pending status."""
job = create_test_job(
status=JobStatus.PENDING,
state_data={'audio_complete': True},
)
issues = check_job_consistency(job)
assert len(issues) == 1
assert "audio_complete_status_mismatch" in issues[0]
assert "audio_complete=True but status=pending" in issues[0]
def test_detects_lyrics_complete_with_pending_status(self):
"""Test detection of lyrics_complete flag with pending status."""
job = create_test_job(
status=JobStatus.PENDING,
state_data={'lyrics_complete': True},
)
issues = check_job_consistency(job)
assert len(issues) == 1
assert "lyrics_complete_status_mismatch" in issues[0]
def test_detects_screens_complete_with_wrong_status(self):
"""Test detection of screens complete but status not in post-screens states."""
job = create_test_job(
status=JobStatus.DOWNLOADING,
state_data={'screens_progress': {'stage': 'complete'}},
)
issues = check_job_consistency(job)
assert len(issues) == 1
assert "screens_complete_status_mismatch" in issues[0]
def test_screens_complete_valid_with_awaiting_review(self):
"""Test that screens complete is valid with AWAITING_REVIEW status."""
job = create_test_job(
status=JobStatus.AWAITING_REVIEW,
state_data={'screens_progress': {'stage': 'complete'}},
)
issues = check_job_consistency(job)
assert issues == []
def test_screens_complete_valid_with_complete_status(self):
"""Test that screens complete is valid with COMPLETE status."""
job = create_test_job(
status=JobStatus.COMPLETE,
state_data={'screens_progress': {'stage': 'complete'}},
)
issues = check_job_consistency(job)
assert issues == []
def test_detects_pending_with_input_media_path(self):
"""Test detection of job stuck at pending with input_media_gcs_path."""
job = create_test_job(
status=JobStatus.PENDING,
input_media_gcs_path="gs://bucket/audio.mp3",
)
issues = check_job_consistency(job)
assert len(issues) == 1
assert "pending_with_input_media" in issues[0]
def test_multiple_issues_detected(self):
"""Test that multiple issues are detected simultaneously."""
job = create_test_job(
status=JobStatus.PENDING,
state_data={
'audio_complete': True,
'lyrics_complete': True,
},
input_media_gcs_path="gs://bucket/audio.mp3",
)
issues = check_job_consistency(job)
assert len(issues) == 3
def test_handles_invalid_status_string(self):
"""Test handling of invalid status string."""
job = create_test_job(status=JobStatus.PENDING)
job.status = "invalid_status" # type: ignore
issues = check_job_consistency(job)
assert len(issues) == 1
assert "invalid_status" in issues[0]
def test_handles_empty_state_data(self):
"""Test handling of empty or None state_data."""
job = create_test_job(
status=JobStatus.DOWNLOADING,
state_data=None,
)
issues = check_job_consistency(job)
assert issues == []
class TestCheckJobConsistencyDetailed:
"""Tests for check_job_consistency_detailed function."""
def test_returns_detailed_info(self):
"""Test that detailed info is returned."""
job = create_test_job(
status=JobStatus.PENDING,
state_data={'audio_complete': True},
input_media_gcs_path="gs://bucket/audio.mp3",
)
result = check_job_consistency_detailed(job)
assert result['job_id'] == "test-job-123"
assert result['status'] == "pending"
assert len(result['issues']) == 2
assert result['state_data_summary']['audio_complete'] is True
assert result['state_data_summary']['lyrics_complete'] is False
assert result['state_data_summary']['has_input_media_gcs_path'] is True
assert result['is_healthy'] is False
def test_healthy_job_returns_is_healthy_true(self):
"""Test that healthy job returns is_healthy=True."""
job = create_test_job(status=JobStatus.DOWNLOADING)
result = check_job_consistency_detailed(job)
assert result['is_healthy'] is True
assert result['issues'] == []
class TestValidateWorkerCanRun:
"""Tests for validate_worker_can_run function."""
def test_audio_worker_valid_with_downloading_status(self):
"""Test audio worker is valid with DOWNLOADING status."""
job = create_test_job(status=JobStatus.DOWNLOADING)
error = validate_worker_can_run("audio_worker", job)
assert error is None
def test_audio_worker_invalid_with_pending_status(self):
"""Test audio worker is invalid with PENDING status."""
job = create_test_job(status=JobStatus.PENDING)
error = validate_worker_can_run("audio_worker", job)
assert error is not None
assert "audio_worker called but job status is pending" in error
def test_lyrics_worker_valid_with_downloading_status(self):
"""Test lyrics worker is valid with DOWNLOADING status."""
job = create_test_job(status=JobStatus.DOWNLOADING)
error = validate_worker_can_run("lyrics_worker", job)
assert error is None
def test_screens_worker_valid_with_audio_complete_status(self):
"""Test screens worker is valid with AUDIO_COMPLETE status."""
job = create_test_job(status=JobStatus.AUDIO_COMPLETE)
error = validate_worker_can_run("screens_worker", job)
assert error is None
def test_screens_worker_valid_with_lyrics_complete_status(self):
"""Test screens worker is valid with LYRICS_COMPLETE status."""
job = create_test_job(status=JobStatus.LYRICS_COMPLETE)
error = validate_worker_can_run("screens_worker", job)
assert error is None
def test_screens_worker_invalid_with_awaiting_review_status(self):
"""Test screens worker is invalid with AWAITING_REVIEW status (too late)."""
job = create_test_job(status=JobStatus.AWAITING_REVIEW)
error = validate_worker_can_run("screens_worker", job)
assert error is not None
assert "screens_worker called but job status is awaiting_review" in error
def test_video_worker_valid_with_instrumental_selected_status(self):
"""Test video worker is valid with INSTRUMENTAL_SELECTED status."""
job = create_test_job(status=JobStatus.INSTRUMENTAL_SELECTED)
error = validate_worker_can_run("video_worker", job)
assert error is None
def test_video_worker_invalid_with_pending_status(self):
"""Test video worker is invalid with PENDING status."""
job = create_test_job(status=JobStatus.PENDING)
error = validate_worker_can_run("video_worker", job)
assert error is not None
assert "video_worker called but job status is pending" in error
def test_render_video_worker_valid_with_review_complete_status(self):
"""Test render video worker is valid with REVIEW_COMPLETE status."""
job = create_test_job(status=JobStatus.REVIEW_COMPLETE)
error = validate_worker_can_run("render_video_worker", job)
assert error is None
def test_unknown_worker_returns_none(self):
"""Test that unknown worker names return None (no validation)."""
job = create_test_job(status=JobStatus.PENDING)
error = validate_worker_can_run("unknown_worker", job)
assert error is None
def test_handles_invalid_status_string(self):
"""Test handling of invalid status string."""
job = create_test_job(status=JobStatus.PENDING)
job.status = "invalid_status" # type: ignore
error = validate_worker_can_run("audio_worker", job)
assert error is not None
assert "Invalid job status" in error
class TestGetWorkerValidStatuses:
"""Tests for get_worker_valid_statuses function."""
def test_returns_dict_with_all_workers(self):
"""Test that all known workers are included."""
statuses = get_worker_valid_statuses()
assert "audio_worker" in statuses
assert "lyrics_worker" in statuses
assert "screens_worker" in statuses
assert "video_worker" in statuses
assert "render_video_worker" in statuses
def test_audio_worker_includes_downloading(self):
"""Test that audio worker includes DOWNLOADING status."""
statuses = get_worker_valid_statuses()
assert JobStatus.DOWNLOADING in statuses["audio_worker"]
def test_lyrics_worker_includes_downloading(self):
"""Test that lyrics worker includes DOWNLOADING status."""
statuses = get_worker_valid_statuses()
assert JobStatus.DOWNLOADING in statuses["lyrics_worker"]
+1 -284
View File
@@ -5,7 +5,7 @@ Tests the job lifecycle management without requiring actual Firestore connection
Uses mocking to isolate the business logic.
"""
import pytest
from unittest.mock import Mock, MagicMock, patch, AsyncMock
from unittest.mock import Mock, MagicMock, patch
from datetime import datetime, UTC
# Mock Firestore before importing JobManager
@@ -14,7 +14,6 @@ sys.modules['google.cloud.firestore'] = MagicMock()
from backend.services.job_manager import JobManager
from backend.models.job import Job, JobCreate, JobStatus
from backend.exceptions import InvalidStateTransitionError
@pytest.fixture
@@ -569,288 +568,6 @@ class TestDeleteStateDataKeys:
assert result == []
# =============================================================================
# InvalidStateTransitionError Tests
# =============================================================================
class TestInvalidStateTransitionError:
"""Tests for InvalidStateTransitionError exception."""
def test_exception_stores_all_attributes(self):
"""Test exception stores job_id, from_status, to_status, and valid_transitions."""
error = InvalidStateTransitionError(
message="Test error",
job_id="test-123",
from_status="pending",
to_status="generating_screens",
valid_transitions=["downloading", "searching_audio", "failed"]
)
assert error.message == "Test error"
assert error.job_id == "test-123"
assert error.from_status == "pending"
assert error.to_status == "generating_screens"
assert error.valid_transitions == ["downloading", "searching_audio", "failed"]
assert str(error) == "Test error"
def test_exception_defaults(self):
"""Test exception has sensible defaults."""
error = InvalidStateTransitionError(message="Minimal error")
assert error.message == "Minimal error"
assert error.job_id == ""
assert error.from_status == ""
assert error.to_status == ""
assert error.valid_transitions == []
# =============================================================================
# State Transition Validation Tests (raise_on_invalid behavior)
# =============================================================================
class TestValidateStateTransitionRaisesBehavior:
"""Tests for validate_state_transition with raise_on_invalid parameter."""
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
"""Test that invalid transition raises InvalidStateTransitionError by default."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC)
)
# PENDING cannot transition directly to GENERATING_SCREENS
with pytest.raises(InvalidStateTransitionError) as exc_info:
job_manager.validate_state_transition("test123", JobStatus.GENERATING_SCREENS)
error = exc_info.value
assert error.job_id == "test123"
assert error.from_status == JobStatus.PENDING
assert error.to_status == "generating_screens"
assert "downloading" in error.valid_transitions
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
"""Test that invalid transition returns False when raise_on_invalid=False."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC)
)
# PENDING cannot transition directly to GENERATING_SCREENS
result = job_manager.validate_state_transition(
"test123",
JobStatus.GENERATING_SCREENS,
raise_on_invalid=False
)
assert result is False
def test_valid_transition_returns_true(self, job_manager, mock_firestore_service):
"""Test that valid transition returns True."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC)
)
# PENDING can transition to DOWNLOADING
result = job_manager.validate_state_transition("test123", JobStatus.DOWNLOADING)
assert result is True
def test_job_not_found_raises_value_error(self, job_manager, mock_firestore_service):
"""Test that missing job raises ValueError when raise_on_invalid=True."""
mock_firestore_service.get_job.return_value = None
with pytest.raises(ValueError, match="Job nonexistent not found"):
job_manager.validate_state_transition("nonexistent", JobStatus.DOWNLOADING)
def test_job_not_found_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
"""Test that missing job returns False when raise_on_invalid=False."""
mock_firestore_service.get_job.return_value = None
result = job_manager.validate_state_transition(
"nonexistent",
JobStatus.DOWNLOADING,
raise_on_invalid=False
)
assert result is False
class TestTransitionToStateRaisesBehavior:
"""Tests for transition_to_state with raise_on_invalid parameter."""
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
"""Test that invalid transition raises InvalidStateTransitionError by default."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC)
)
# PENDING cannot transition directly to AWAITING_REVIEW
with pytest.raises(InvalidStateTransitionError) as exc_info:
job_manager.transition_to_state("test123", JobStatus.AWAITING_REVIEW)
error = exc_info.value
assert error.job_id == "test123"
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
"""Test that invalid transition returns False when raise_on_invalid=False."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC)
)
result = job_manager.transition_to_state(
"test123",
JobStatus.AWAITING_REVIEW,
raise_on_invalid=False
)
assert result is False
# Verify update was NOT called since transition was invalid
mock_firestore_service.update_job.assert_not_called()
def test_valid_transition_updates_job(self, job_manager, mock_firestore_service):
"""Test that valid transition updates the job."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC)
)
result = job_manager.transition_to_state(
"test123",
JobStatus.DOWNLOADING,
progress=15,
message="Starting processing"
)
assert result is True
# transition_to_state uses update_job_status internally
mock_firestore_service.update_job_status.assert_called()
call_kwargs = mock_firestore_service.update_job_status.call_args.kwargs
assert call_kwargs['job_id'] == "test123"
assert call_kwargs['status'] == JobStatus.DOWNLOADING
# =============================================================================
# start_job_processing Tests
# =============================================================================
class TestStartJobProcessing:
"""Tests for start_job_processing method."""
@pytest.mark.asyncio
async def test_raises_if_job_not_found(self, job_manager, mock_firestore_service):
"""Test raises ValueError if job doesn't exist."""
mock_firestore_service.get_job.return_value = None
with pytest.raises(ValueError, match="Job missing-job not found"):
await job_manager.start_job_processing("missing-job")
@pytest.mark.asyncio
async def test_raises_if_missing_input_media_path(self, job_manager, mock_firestore_service):
"""Test raises ValueError if job has no input_media_gcs_path."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path=None # Missing
)
with pytest.raises(ValueError, match="missing input_media_gcs_path"):
await job_manager.start_job_processing("test123")
@pytest.mark.asyncio
async def test_raises_if_invalid_state_transition(self, job_manager, mock_firestore_service):
"""Test raises InvalidStateTransitionError if job can't transition to DOWNLOADING."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.AWAITING_REVIEW, # Can't go back to DOWNLOADING
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path="gs://bucket/audio.mp3"
)
with pytest.raises(InvalidStateTransitionError):
await job_manager.start_job_processing("test123")
@pytest.mark.asyncio
async def test_transitions_and_triggers_workers(self, job_manager, mock_firestore_service):
"""Test successful flow: transitions to DOWNLOADING and triggers workers."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path="gs://bucket/audio.mp3"
)
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
mock_worker = Mock()
mock_worker.trigger_audio_worker = AsyncMock()
mock_worker.trigger_lyrics_worker = AsyncMock()
mock_get_worker.return_value = mock_worker
await job_manager.start_job_processing("test123")
# Verify transition was called (uses update_job_status internally)
mock_firestore_service.update_job_status.assert_called()
call_kwargs = mock_firestore_service.update_job_status.call_args.kwargs
assert call_kwargs['status'] == JobStatus.DOWNLOADING
# Verify both workers were triggered
mock_worker.trigger_audio_worker.assert_called_once_with("test123")
mock_worker.trigger_lyrics_worker.assert_called_once_with("test123")
@pytest.mark.asyncio
async def test_workers_triggered_in_parallel(self, job_manager, mock_firestore_service):
"""Test that audio and lyrics workers are triggered concurrently."""
mock_firestore_service.get_job.return_value = Job(
job_id="test123",
status=JobStatus.PENDING,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
input_media_gcs_path="gs://bucket/audio.mp3"
)
call_order = []
async def track_audio(*args):
call_order.append(('audio_start', datetime.now()))
# Simulate some work
call_order.append(('audio_end', datetime.now()))
async def track_lyrics(*args):
call_order.append(('lyrics_start', datetime.now()))
# Simulate some work
call_order.append(('lyrics_end', datetime.now()))
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
mock_worker = Mock()
mock_worker.trigger_audio_worker = AsyncMock(side_effect=track_audio)
mock_worker.trigger_lyrics_worker = AsyncMock(side_effect=track_lyrics)
mock_get_worker.return_value = mock_worker
await job_manager.start_job_processing("test123")
# Both workers should have been called
assert any('audio_start' in str(c) for c in call_order)
assert any('lyrics_start' in str(c) for c in call_order)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+9 -72
View File
@@ -1332,13 +1332,10 @@ class TestHandleMadeForYouOrder:
mock_theme_service
):
"""
Orders with YouTube URL should download audio then start processing (no search).
Orders with YouTube URL should download audio then trigger workers (no search).
Note: The download step was added in Jan 2026 to fix job 811ec4e5 failure.
See docs/LESSONS-LEARNED.md "YouTube URL Downloads Must Happen Before Workers".
The handler now uses job_manager.start_job_processing() which handles
transitioning to DOWNLOADING and triggering workers.
"""
from backend.api.routes.users import _handle_made_for_you_order
@@ -1351,16 +1348,18 @@ class TestHandleMadeForYouOrder:
"youtube_url": "https://youtube.com/watch?v=abc123",
}
mock_worker_service = MagicMock()
mock_worker_service.trigger_audio_worker = AsyncMock()
mock_worker_service.trigger_lyrics_worker = AsyncMock()
# Mock YouTube download service to return a GCS path
mock_youtube_service = MagicMock()
mock_youtube_service.download = AsyncMock(
return_value="uploads/test-job-123/audio/test.webm"
)
# Mock start_job_processing as an async method
mock_job_manager.start_job_processing = AsyncMock()
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service), \
patch('backend.services.audio_search_service.get_audio_search_service') as mock_get_audio, \
patch('backend.services.storage_service.StorageService'), \
patch('backend.api.routes.users.get_theme_service', return_value=mock_theme_service), \
@@ -1381,8 +1380,9 @@ class TestHandleMadeForYouOrder:
# Audio should be downloaded first
mock_youtube_service.download.assert_called_once()
# start_job_processing should be called (handles transition + workers)
mock_job_manager.start_job_processing.assert_called_once_with("test-job-123")
# Workers should be triggered after download
mock_worker_service.trigger_audio_worker.assert_called_once()
mock_worker_service.trigger_lyrics_worker.assert_called_once()
@pytest.mark.asyncio
async def test_stripe_session_marked_processed(
@@ -1588,69 +1588,6 @@ class TestMadeForYouYouTubeDownloadContract:
f"input_media_gcs_path must be set BEFORE workers are triggered. " \
f"Call order was: {call_order}"
@pytest.mark.asyncio
async def test_youtube_url_calls_start_job_processing_after_download(
self, mock_job_manager, mock_email_service, mock_user_service,
mock_theme_service, youtube_order_metadata
):
"""
CRITICAL: start_job_processing must be called AFTER download completes.
The handler uses job_manager.start_job_processing() which:
1. Transitions job to DOWNLOADING
2. Triggers audio and lyrics workers
Without start_job_processing, workers would run but all status updates
would fail silently because transitions from PENDING to GENERATING_SCREENS
or AWAITING_REVIEW are invalid.
This bug caused made-for-you orders to get stuck at PENDING status even though
all processing completed successfully. See job IDs 06cfea29 and 984da08b.
"""
from backend.api.routes.users import _handle_made_for_you_order
# Track call order to verify sequence
call_order = []
# Mock YouTubeDownloadService to return a GCS path
mock_youtube_service = MagicMock()
mock_youtube_service.download = AsyncMock(
return_value="uploads/test-job-123/audio/test.webm",
side_effect=lambda **kwargs: (
call_order.append('youtube_download'),
"uploads/test-job-123/audio/test.webm"
)[1]
)
# Track when start_job_processing is called
mock_job_manager.start_job_processing = AsyncMock(
side_effect=lambda job_id: call_order.append('start_job_processing')
)
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
patch('backend.services.storage_service.StorageService'), \
patch('backend.api.routes.users.get_theme_service', return_value=mock_theme_service), \
patch('backend.api.routes.users.get_youtube_download_service', return_value=mock_youtube_service):
await _handle_made_for_you_order(
session_id="sess_123",
metadata=youtube_order_metadata,
user_service=mock_user_service,
email_service=mock_email_service,
)
# CRITICAL: start_job_processing must be called
assert 'start_job_processing' in call_order, \
f"start_job_processing must be called. Call order was: {call_order}"
# CRITICAL: start_job_processing must happen AFTER download
download_index = call_order.index('youtube_download')
processing_index = call_order.index('start_job_processing')
assert download_index < processing_index, \
f"YouTube download must complete BEFORE start_job_processing. " \
f"Call order was: {call_order}"
@pytest.mark.asyncio
async def test_youtube_download_called_with_correct_params(
self, mock_job_manager, mock_email_service, mock_user_service,
+1 -10
View File
@@ -24,7 +24,6 @@ from pathlib import Path
from backend.models.job import JobStatus
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.services.job_health_service import validate_worker_can_run
from backend.config import get_settings
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
from backend.workers.registry import worker_registry
@@ -289,15 +288,7 @@ async def process_audio_separation(job_id: str) -> bool:
logger.error(f"[job:{job_id}] Job not found in Firestore")
job_log.error(f"Job {job_id} not found in Firestore!")
return False
# Validate job status is appropriate for audio worker
# This helps catch bugs where the worker is triggered incorrectly
status_error = validate_worker_can_run("audio_worker", job)
if status_error:
logger.warning(f"[job:{job_id}] {status_error}")
job_log.warning(status_error)
# Continue anyway - this is a safety net warning, not a hard failure
# Create temporary working directory
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_{job_id}_")
job_log.info(f"Created temp directory: {temp_dir}")
+1 -10
View File
@@ -32,7 +32,6 @@ from backend.models.job import JobStatus
from karaoke_gen.utils import sanitize_filename
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.services.job_health_service import validate_worker_can_run
from backend.services.lyrics_cache_service import LyricsCacheService
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
from backend.workers.style_helper import load_style_config
@@ -191,15 +190,7 @@ async def process_lyrics_transcription(job_id: str) -> bool:
logger.error(f"[job:{job_id}] Job not found in Firestore")
job_log.error(f"Job {job_id} not found in Firestore!")
return False
# Validate job status is appropriate for lyrics worker
# This helps catch bugs where the worker is triggered incorrectly
status_error = validate_worker_can_run("lyrics_worker", job)
if status_error:
logger.warning(f"[job:{job_id}] {status_error}")
job_log.warning(status_error)
# Continue anyway - this is a safety net warning, not a hard failure
add_span_attribute("artist", job.artist)
add_span_attribute("title", job.title)
+1 -10
View File
@@ -36,7 +36,6 @@ from dataclasses import dataclass
from backend.models.job import JobStatus
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.services.job_health_service import validate_worker_can_run
from backend.config import get_settings
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
from backend.services.tracing import job_span, add_span_event, add_span_attribute
@@ -99,15 +98,7 @@ async def process_render_video(job_id: str) -> bool:
logger.error(f"[job:{job_id}] Job not found in Firestore")
job_log.error(f"Job {job_id} not found in Firestore!")
return False
# Validate job status is appropriate for render video worker
# This helps catch bugs where the worker is triggered incorrectly
status_error = validate_worker_can_run("render_video_worker", job)
if status_error:
logger.warning(f"[job:{job_id}] {status_error}")
job_log.warning(status_error)
# Continue anyway - this is a safety net warning, not a hard failure
job_log.info(f"Starting video render for {job.artist} - {job.title}")
logger.info(f"[job:{job_id}] Starting video render (post-review)")
+1 -10
View File
@@ -33,7 +33,6 @@ from pathlib import Path
from backend.models.job import JobStatus
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.services.job_health_service import validate_worker_can_run
from backend.config import get_settings
from backend.workers.style_helper import load_style_config, StyleConfig
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
@@ -239,14 +238,6 @@ def _validate_prerequisites(job) -> bool:
Returns:
True if prerequisites met, False otherwise
"""
# Validate job status is appropriate for screens worker
# This helps catch bugs where the worker is triggered incorrectly
status_error = validate_worker_can_run("screens_worker", job)
if status_error:
logger.warning(f"Job {job.job_id}: {status_error}")
# Log warning but don't fail - status validation is a safety net,
# the state_data flags below are the authoritative check
# SAFETY NET: Enforce theme requirement at processing time
# This catches any jobs that somehow bypassed JobManager.create_job() validation
if not job.theme_id:
@@ -271,7 +262,7 @@ def _validate_prerequisites(job) -> bool:
if not job.artist or not job.title:
logger.error(f"Job {job.job_id}: Missing artist or title")
return False
return True
-8
View File
@@ -34,7 +34,6 @@ from pathlib import Path
from backend.models.job import JobStatus
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.services.job_health_service import validate_worker_can_run
from backend.services.rclone_service import get_rclone_service
from backend.services.youtube_service import get_youtube_service
from backend.services.encoding_service import get_encoding_service
@@ -895,13 +894,6 @@ async def _handle_native_distribution(
def _validate_prerequisites(job) -> bool:
"""Validate that all prerequisites are met for video generation."""
# Validate job status is appropriate for video worker
# This helps catch bugs where the worker is triggered incorrectly
status_error = validate_worker_can_run("video_worker", job)
if status_error:
logger.warning(f"Job {job.job_id}: {status_error}")
# Continue anyway - this is a safety net warning, not a hard failure
# Check instrumental selection
instrumental_selection = job.state_data.get('instrumental_selection')
if not instrumental_selection:
+5 -124
View File
@@ -958,134 +958,15 @@ Response:
}
```
#### Regenerate Screens
```http
POST /api/admin/jobs/{job_id}/regenerate-screens
Authorization: Bearer ADMIN_TOKEN
```
Regenerates title and end screen videos using the **current** artist/title metadata. Use this after editing artist or title fields to update the screens without full reprocessing.
Requirements:
- Job must have audio and lyrics processing complete
- Job must be in an allowed state: `complete`, `failed`, `awaiting_review`, `awaiting_instrumental_selection`, `instrumental_selected`, `prep_complete`
The endpoint:
1. Deletes existing screen files from GCS
2. Triggers the screens worker to regenerate with current metadata
3. Returns immediately (does not wait for completion)
Response:
```json
{
"status": "success",
"job_id": "abc123",
"message": "Screens regeneration started",
"previous_screens_deleted": true,
"worker_triggered": true,
"error": null
}
```
Screen generation takes 30-60 seconds. Monitor progress via job timeline or logs.
#### Restart Job
```http
POST /api/admin/jobs/{job_id}/restart
Authorization: Bearer ADMIN_TOKEN
Content-Type: application/json
{
"preserve_audio_stems": true,
"delete_outputs": true
}
```
Fully restarts a job from the beginning. Unlike the reset endpoint (which just changes state), restart actually **triggers workers** to begin processing.
Options:
- `preserve_audio_stems` (default: false) - If true, keeps existing audio separation and lyrics. Only regenerates screens and video with current metadata. Good for fixing title/artist typos.
- `delete_outputs` (default: true) - Delete existing output files from GCS.
Behavior by job type:
- **YouTube URL jobs**: Re-downloads audio, then triggers audio/lyrics workers
- **Audio search jobs**: Transitions to `awaiting_audio_selection` for admin to re-select
- **File upload jobs**: Triggers audio/lyrics workers directly
When `preserve_audio_stems=true`:
- Validates audio and lyrics processing completed previously
- Clears screens/video/encoding state only
- Triggers screens worker immediately
Allowed states: `pending`, `complete`, `failed`, `awaiting_review`, `awaiting_audio_selection`, `awaiting_instrumental_selection`, `instrumental_selected`, `prep_complete`
Response:
```json
{
"status": "success",
"job_id": "abc123",
"message": "Job restarted from complete to downloading",
"previous_status": "complete",
"new_status": "downloading",
"cleared_data": ["audio_complete", "lyrics_complete", "screens_progress", "..."],
"deleted_gcs_paths": ["jobs/abc123/screens/title.mov", "..."],
"workers_triggered": ["download", "audio", "lyrics"],
"error": null
}
```
#### Override Audio Source
```http
POST /api/admin/jobs/{job_id}/override-audio-source
Authorization: Bearer ADMIN_TOKEN
Content-Type: application/json
{
"source_type": "audio_search"
}
```
Switches a job's audio source from YouTube URL to audio search mode. Use this when a Made-For-You order was submitted with a YouTube URL but you want to find higher quality audio.
Currently only supports switching to `audio_search` mode, which:
1. Clears existing audio-related state (URL, downloaded file, stems, transcription)
2. Performs an audio search using the job's artist/title
3. Stores search results in state_data
4. Transitions job to `awaiting_audio_selection` with results ready
5. Admin can then select an audio source from the search results in the UI
Allowed states: `pending`, `complete`, `failed`, `awaiting_review`, `awaiting_audio_selection`, `instrumental_selected`, `prep_complete`
Response:
```json
{
"status": "success",
"job_id": "abc123",
"message": "Found 5 audio sources - select one in the admin panel.",
"previous_source": "youtube",
"new_source": "audio_search",
"cleared_data": ["audio_complete", "lyrics_complete", "url", "input_media_gcs_path", "..."],
"new_status": "awaiting_audio_selection",
"search_results_count": 5,
"error": null
}
```
If no audio sources found:
If triggering fails:
```json
{
"status": "error",
"job_id": "abc123",
"message": "No audio sources found for: Artist - Title",
"previous_source": "youtube",
"new_source": "audio_search",
"cleared_data": ["..."],
"new_status": "failed",
"search_results_count": null,
"error": "No audio sources found for: Artist - Title"
"worker_type": "video",
"triggered": false,
"message": "Failed to trigger video worker for job abc123",
"error": "Job is in wrong state for video processing"
}
```
-2
View File
@@ -102,8 +102,6 @@
**Note**: `awaiting_instrumental_selection` exists for backwards compatibility with historical jobs but is no longer used. Instrumental selection is now part of the combined review (`awaiting_review` → `in_review` → `review_complete`).
**State Machine Robustness (Feb 2026)**: State transitions are enforced by `STATE_TRANSITIONS` in `models/job.py`. Invalid transitions raise `InvalidStateTransitionError` by default. Use `scripts/generate_state_diagram.py` to generate a Mermaid diagram of valid transitions. The `/health/job-consistency` endpoint detects jobs stuck in invalid states. See `docs/archive/2026-02-02-state-machine-robustness-plan.md` for implementation details.
## GCS File Structure
```text
-92
View File
@@ -45,19 +45,6 @@ When UI removes options, backend must apply sensible defaults. Don't rely on eac
### Centralize Job Creation Logic
When multiple code paths create jobs (file upload, audio search, webhooks), use a shared service for default resolution. The made-for-you webhook handler diverged from regular job creation, missing CDG/TXT defaults because it didn't call the shared `resolve_cdg_txt_defaults()` function. Fix: Create `job_defaults_service.py` with centralized helpers used by ALL job creation paths.
### Make State Transitions Loud, Not Silent (Feb 2026)
**What happened**: Jobs 06cfea29 and 984da08b got stuck at `pending` forever. The Made-For-You webhook handler triggered workers without first transitioning from PENDING to DOWNLOADING. The state machine returned `False` silently, workers ran, but job status never advanced.
**Root cause**: `transition_to_state()` returned `False` on invalid transitions, but callers didn't check the return value. Silent failures let bugs hide in production.
**Fix implemented**:
1. **Make failures loud**: `transition_to_state()` now raises `InvalidStateTransitionError` by default (use `raise_on_invalid=False` for soft failures)
2. **Centralize the pattern**: Created `start_job_processing()` helper that atomically transitions + triggers workers - all handlers use this instead of implementing independently
3. **Runtime consistency checks**: Added `job_health_service.py` with `check_job_consistency()` to detect stuck jobs (audio_complete but status=pending)
4. **Worker validation**: Workers now log warnings if triggered when job is in unexpected status
**Pattern**: When a function can fail, prefer exceptions over boolean returns. Silent failures accumulate into production bugs. Use centralized helpers for multi-step operations (state change + side effects) to prevent divergent implementations.
### Fix Both Sides of Dual Code Paths
When fixing a bug in a system with multiple code paths (e.g., legacy vs orchestrator, local vs cloud), verify ALL paths are fixed. PR #271 fixed the GCE worker to READ `instrumental_selection` but only checked the legacy path which was already SENDING it. The orchestrator path (production default) wasn't sending it. **Pattern**: If a component receives config from multiple callers, check ALL callers when fixing the receiving side. Write integration tests that cover each path.
@@ -138,22 +125,6 @@ For subprocess without shell, FFmpeg filter paths need: apostrophes escaped as `
### Unicode in HTTP Headers
Sanitize user input (artist/title) to ASCII for HTTP headers. Smart quotes, em dashes from copy/paste cause encoding errors.
### Unicode in File Paths (Feb 2026)
**What happened**: Job `1eab1172` with Russian title "я куплю тебе дом" (Cyrillic) failed with FFmpeg exit code 183. The temp subtitle file path contained Unicode characters: `temp_subtitles_zivert___я_куплю_тебе_дом_1770150285852.ass`.
**Root cause**: Python's `str.isalnum()` returns `True` for ALL Unicode alphanumeric characters (Cyrillic, Arabic, Chinese, etc.), not just ASCII. The code used `"".join(c if c.isalnum() else "_" for c in output_prefix)` to sanitize filenames, which allowed Cyrillic letters through because they're "alphanumeric" in Unicode. FFmpeg cannot handle Unicode characters in filter argument paths, causing file I/O errors.
**Fix**: Replace `c.isalnum()` with `c.isascii() and c.isalnum()` to only allow ASCII alphanumeric characters (a-z, A-Z, 0-9). This ensures temp file paths are safe for FFmpeg and other external tools.
**Locations fixed** (v0.115.2):
- `karaoke_gen/lyrics_transcriber/output/video.py:222` - `generate_video()` temp subtitle path
- `karaoke_gen/lyrics_transcriber/output/video.py:282` - `generate_preview_video()` temp subtitle path
- `karaoke_gen/utils/remote_cli.py:1970` - output folder name sanitization
**Pattern**: When sanitizing filenames or paths for external tools, always use ASCII-only checks. Don't assume `isalnum()`, `isalpha()`, or `isdigit()` are ASCII-only - they match all Unicode categories.
**Test coverage**: Added 10 comprehensive tests covering Cyrillic, Chinese, Arabic, emoji, and accented characters to prevent regression.
### Fonts in Docker
Base Docker images have no fonts. Install `fonts-noto-core`, `fonts-noto-cjk` for video rendering.
@@ -243,37 +214,6 @@ worker_index = call_order.index('audio_worker')
assert download_index < worker_index, "Download must happen BEFORE workers"
```
### Made-For-You Jobs Must Transition to DOWNLOADING Before Workers (Feb 2026)
**Problem**: Made-for-you orders with YouTube URLs (jobs 06cfea29, 984da08b) got stuck at `pending` status even though all processing completed successfully. Workers ran, `state_data` showed `audio_complete: true`, `lyrics_complete: true`, `screens_progress.stage: complete`, but the `status` field remained `pending`.
**Root cause**: The `_handle_made_for_you_order` function in `users.py` triggered workers for YouTube URL orders **without first transitioning the job from PENDING to DOWNLOADING**. The job state machine defines strict valid transitions:
- `PENDING` → `DOWNLOADING`, `SEARCHING_AUDIO`, `FAILED`, `CANCELLED` (only these are valid)
- Workers try: `PENDING` → `GENERATING_SCREENS` → `AWAITING_REVIEW` (INVALID!)
All status transitions after workers complete fail silently because they're invalid from `PENDING`.
**Why it was subtle**:
1. Workers completed successfully (no errors in logs)
2. `state_data` updates worked (they bypass state machine validation)
3. Only the `status` field failed to update (uses `transition_to_state` which validates)
**Fix**:
1. Immediate: Add `transition_to_state(job_id, JobStatus.DOWNLOADING)` before triggering workers
2. Robustness: Make `transition_to_state()` raise `InvalidStateTransitionError` by default instead of returning `False` silently
3. Pattern: Create centralized `job_manager.start_job_processing(job_id)` helper that handles transition + worker triggers atomically
**Robustness changes** (see `docs/archive/2026-02-02-state-machine-robustness-plan.md`):
- `InvalidStateTransitionError` exception with full context (job_id, from_status, to_status, valid_transitions)
- `validate_state_transition()` and `transition_to_state()` now have `raise_on_invalid=True` default
- `start_job_processing()` centralizes the pattern: validate prerequisites → transition → trigger workers
**Pattern**: Always verify state machine flow in webhook handlers. Use centralized helpers like `start_job_processing()` instead of duplicating transition + worker logic.
**Tests added**:
- `test_youtube_url_calls_start_job_processing_after_download` - verifies proper sequence
- `TestInvalidStateTransitionError`, `TestValidateStateTransitionRaisesBehavior`, `TestStartJobProcessing` in `test_job_manager.py`
### Use data-testid for E2E
Prefer `data-testid` over label/text selectors. They're immune to label changes and won't break when similar fields are added.
@@ -328,38 +268,6 @@ const debouncedUpdate = useCallback((val: string) => {
**Testing**: Verify (1) typing multi-digit values doesn't jump cursor, (2) debounce fires after delay, (3) blur syncs immediately, (4) external updates respected when not focused. See `frontend/components/lyrics-review/__tests__/TimeInput.test.tsx` for comprehensive test suite.
### Mocks Must Match Real Return Values (Feb 2026)
**Problem**: PR #371 introduced a bug where admin reset always returned 500. The code checked `if not success:` after calling `update_job()`, but `update_job()` returns `None`, not a boolean. Since `not None` is `True`, the error was always raised.
**Why tests didn't catch it**: All 22 test mocks had:
```python
mock_jm.update_job.return_value = True # WRONG - real method returns None
```
The mock returned `True`, so `if not True:` was `False` and tests passed. But in production, `if not None:` is `True`, causing 500 errors.
**Root cause**: Mocks were written based on an assumed API contract (boolean return) rather than the actual contract (void return, raises on error).
**Fix**:
1. Updated all mocks to `return_value = None` with comment explaining why
2. Added emulator integration test (`test_admin_reset_integration.py`) that uses real `JobManager`
3. Added explicit test `test_update_job_returns_none_not_boolean` to document the API contract
**Pattern for preventing this**:
1. **Mocks should match real signatures**: Check the actual method before setting `return_value`. Void methods should mock `return_value = None`.
2. **Add emulator tests for critical paths**: Integration tests with real services catch contract mismatches mocks hide.
3. **Document API contracts in tests**: Add explicit tests for non-obvious behaviors (`update_job` returns None, raises on failure).
**Code pattern**:
```python
# ❌ WRONG - assumes boolean return
mock_jm.update_job.return_value = True
# ✅ CORRECT - matches real API
mock_jm.update_job.return_value = None # Real API: returns None, raises on error
```
### Emulator Tests Catch Real Bugs
Firestore emulator tests catch issues (like missing indexes) that unit tests with mocks miss.
-4
View File
@@ -41,10 +41,6 @@
## Recent Changes
- **Unicode Filename Fix** (2026-02-04): Fixed FFmpeg failures (exit code 183) when processing songs with non-ASCII characters in titles. Job `1eab1172` with Russian title "я куплю тебе дом" failed because Python's `str.isalnum()` returns True for ALL Unicode alphanumeric characters (Cyrillic, Arabic, Chinese, emoji, etc.), not just ASCII. These Unicode characters in temp subtitle file paths (`temp_subtitles_zivert___я_куплю_тебе_дом_1770150285852.ass`) caused FFmpeg to fail. **Fix**: Changed filename sanitization from `c.isalnum()` to `c.isascii() and c.isalnum()` in three locations: `video.py:222` (generate_video), `video.py:282` (generate_preview_video), and `remote_cli.py:1970` (output folders). Added comprehensive test suite covering Cyrillic, Chinese, Arabic, emoji, and accented characters. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#unicode-in-file-paths-feb-2026).
- **Admin MFY Restart & Override Tools** (2026-02-04): Added three new admin endpoints for better Made-For-You order management: (1) **Regenerate Screens** (`POST /api/admin/jobs/{job_id}/regenerate-screens`) - regenerates title/end screens with current artist/title metadata without full reprocessing; (2) **Full Restart** (`POST /api/admin/jobs/{job_id}/restart`) - unlike "Reset to Start", this actually triggers workers to run, with option to preserve audio stems for faster restart; (3) **Override Audio Source** (`POST /api/admin/jobs/{job_id}/override-audio-source`) - switch from YouTube URL to audio search mode for higher quality audio. Frontend admin toolbar updated with new buttons (Regen Screens, Full Restart, Audio Search) with confirmation dialogs. **Use case**: Fix MFY orders where customer provided wrong artist/title or low-quality YouTube URL. See [API.md](API.md#regenerate-screens).
- **Made-For-You YouTube URL Download Fix** (2026-01-28): Fixed bug where made-for-you orders with YouTube URLs failed with "Failed to download audio file". **Root cause**: The webhook handler triggered workers directly without downloading the audio first via `YouTubeDownloadService`. Workers require `input_media_gcs_path` to be set, which only happens after download completes. **Fix**: Handler now downloads YouTube audio to GCS before triggering workers, matching the pattern in `file_upload.py` and `audio_search.py`. Added 4 new contract tests with call-order tracking and AST import validation to prevent regression. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#youtube-url-downloads-must-happen-before-workers-jan-2026).
- **Lyrics Corrections Now Applied to Video** (2026-01-26): Fixed critical bug where user's lyrics corrections made during human review were being ignored. Videos were rendering with raw transcription instead of corrected lyrics. **Root cause**: The `GET /api/review/{job_id}/correction-data` endpoint was returning the original `corrections.json` instead of `corrections_updated.json` (where user edits are saved). The combined review flow would then overwrite the user's edits when completing the review. **Fix**: The endpoint now checks for `corrections_updated.json` first, matching the behavior of `render_video_worker.py`. Added 35 new tests (backend unit + integration + frontend E2E) to prevent regression. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#always-test-the-full-data-flow-not-just-individual-components).
-24
View File
@@ -158,30 +158,6 @@ def test_create_job_applies_default_theme(mock_firestore, mock_gcs):
assert job.theme_id == "nomad" # Default theme applied
```
**Critical: Mocks Must Match Real API Contracts**
When mocking methods, ensure the mock's `return_value` matches what the real method returns. A common bug pattern:
```python
# ❌ BAD: Assumes method returns boolean
mock_service.update_job.return_value = True
# Code under test:
success = job_manager.update_job(job_id, updates)
if not success: # BUG: if real method returns None, this is always True!
raise HTTPException(500)
```
The fix:
```python
# ✅ GOOD: Check real method signature first
# def update_job(self, job_id: str, updates: Dict) -> None:
mock_service.update_job.return_value = None # Match real API
```
**Before writing a mock**, check the actual method's return type. Void methods should mock `return_value = None`. Methods that raise on error (rather than returning False) should have mocks that raise exceptions for error cases. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#mocks-must-match-real-return-values-feb-2026) for a production bug caused by this.
### Integration Tests (`tests/integration/`)
**Purpose:** Test CLI workflows and component interactions.
@@ -1,307 +0,0 @@
# State Machine Robustness Plan
**Date**: 2026-02-02
**Status**: ✓ Complete (Weeks 1-3 implemented)
**Context**: Jobs 06cfea29 and 984da08b got stuck at `pending` because state transitions failed silently.
## Problem Analysis
The root issues that allowed this bug:
1. **Silent failures**: `transition_to_state()` returns `False` on invalid transitions, but callers don't check it
2. **No centralized worker trigger pattern**: Each handler (file_upload.py, users.py, audio_search.py) implements "transition + trigger workers" independently
3. **No runtime consistency checks**: Nothing detects when `state_data` says complete but `status` is wrong
4. **Workers don't validate prerequisites**: Workers run regardless of job status
## Proposed Improvements
### 1. Make State Transition Failures Loud (High Priority)
**Option A**: Raise exception by default (breaking change, but safest)
```python
class InvalidStateTransitionError(Exception):
"""Raised when an invalid state transition is attempted."""
pass
def transition_to_state(self, job_id: str, new_status: JobStatus, ...,
raise_on_invalid: bool = True) -> bool:
if not self.validate_state_transition(job_id, new_status):
if raise_on_invalid:
raise InvalidStateTransitionError(
f"Invalid transition for job {job_id}: {current} -> {new_status}"
)
return False
...
```
**Option B**: Always log + metric, return bool (non-breaking)
```python
def transition_to_state(...) -> bool:
if not self.validate_state_transition(job_id, new_status):
# Log with structured data for alerting
logger.error(
f"STATE_TRANSITION_FAILED job={job_id} from={current} to={new_status}",
extra={"job_id": job_id, "from_status": current, "to_status": new_status}
)
# Emit metric for monitoring
emit_metric("state_transition_failed", 1, {"from": current, "to": new_status})
return False
...
```
**Recommendation**: Option A with `raise_on_invalid=True` default. Forces callers to handle failures explicitly.
### 2. Centralize "Start Processing" Pattern (High Priority)
Create a single function that handles the transition + worker trigger pattern:
```python
# backend/services/job_manager.py
async def start_job_processing(self, job_id: str, worker_service) -> None:
"""
Transition job to DOWNLOADING and trigger workers.
This is the single entry point for starting job processing.
All handlers (file_upload, webhook, audio_search) should use this.
Raises:
InvalidStateTransitionError: If job can't transition to DOWNLOADING
ValueError: If job not found or missing required fields
"""
job = self.get_job(job_id)
if not job:
raise ValueError(f"Job {job_id} not found")
if not job.input_media_gcs_path:
raise ValueError(f"Job {job_id} missing input_media_gcs_path")
# This will raise if transition is invalid
self.transition_to_state(
job_id=job_id,
new_status=JobStatus.DOWNLOADING,
progress=15,
message="Starting audio and lyrics processing"
)
# Only trigger workers if transition succeeded
await asyncio.gather(
worker_service.trigger_audio_worker(job_id),
worker_service.trigger_lyrics_worker(job_id)
)
```
Then update all handlers to use this:
```python
# In _handle_made_for_you_order, file_upload, audio_search:
await job_manager.start_job_processing(job_id, worker_service)
```
### 3. Add State Consistency Checks (Medium Priority)
Add a health check that detects inconsistent state:
```python
# backend/services/job_health_service.py
def check_job_consistency(job: Job) -> List[str]:
"""
Check for state inconsistencies in a job.
Returns list of issues found.
"""
issues = []
state_data = job.state_data or {}
# Check: audio_complete but status still pending
if state_data.get('audio_complete') and job.status == JobStatus.PENDING:
issues.append(f"audio_complete=True but status=pending")
# Check: lyrics_complete but status still pending
if state_data.get('lyrics_complete') and job.status == JobStatus.PENDING:
issues.append(f"lyrics_complete=True but status=pending")
# Check: screens_progress.stage=complete but status not awaiting_review
screens = state_data.get('screens_progress', {})
if screens.get('stage') == 'complete' and job.status not in [
JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE,
JobStatus.RENDERING_VIDEO, JobStatus.INSTRUMENTAL_SELECTED, ...
]:
issues.append(f"screens complete but status={job.status}")
return issues
```
Run this:
- In a daily Cloud Scheduler job
- After each worker completes
- Expose via `/health/job-consistency` endpoint
### 4. Workers Validate Prerequisites (Medium Priority)
Workers should verify job is in expected state before processing:
```python
# backend/workers/screens_worker.py
async def generate_screens(job_id: str) -> bool:
job = job_manager.get_job(job_id)
# Validate job is in expected state for this worker
valid_statuses = [JobStatus.DOWNLOADING, JobStatus.AUDIO_COMPLETE, JobStatus.LYRICS_COMPLETE]
if job.status not in [s.value for s in valid_statuses]:
logger.error(
f"[job:{job_id}] screens_worker called but job status is {job.status}, "
f"expected one of {valid_statuses}. This indicates a bug in the trigger logic."
)
# Option: return False, or try to recover, or alert
return False
# ... rest of worker
```
### 5. Add Integration Tests for State Flow (Medium Priority)
Add tests that verify the complete state machine flow without mocking transitions:
```python
# backend/tests/integration/test_state_machine_flow.py
@pytest.mark.integration
async def test_made_for_you_youtube_flow_state_transitions():
"""
Verify complete state flow for made-for-you YouTube orders.
Uses real JobManager (mocked Firestore) to validate state machine.
"""
# Create job
job = job_manager.create_job(...)
assert job.status == JobStatus.PENDING
# Simulate YouTube download complete
job_manager.update_job(job_id, {'input_media_gcs_path': 'test/path'})
# Start processing (this should transition to DOWNLOADING)
await job_manager.start_job_processing(job_id, mock_worker_service)
job = job_manager.get_job(job_id)
assert job.status == JobStatus.DOWNLOADING, \
f"Expected DOWNLOADING after start_job_processing, got {job.status}"
# ... continue through full flow
```
### 6. State Machine Diagram Generation (Low Priority)
Auto-generate a Mermaid diagram from `STATE_TRANSITIONS`:
```python
# scripts/generate_state_diagram.py
def generate_mermaid():
lines = ["stateDiagram-v2"]
for from_state, to_states in STATE_TRANSITIONS.items():
for to_state in to_states:
lines.append(f" {from_state} --> {to_state}")
return "\n".join(lines)
```
Add to `docs/ARCHITECTURE.md` and regenerate on changes.
### 7. Fix Admin Reset Buttons (High Priority)
The admin dashboard has reset buttons that bypass state machine validation and have several issues:
**Current buttons:**
- "Start" → reset to `pending`
- "Audio" → reset to `awaiting_audio_selection`
- "Lyrics" → reset to `awaiting_review`
- "Inst." → reset to `awaiting_instrumental_selection`
- "Reprocess" → reset to `instrumental_selected`
**Issues identified:**
1. **"Start" button leaves job stuck**: Resets to `pending` but doesn't trigger workers or clear `audio_complete`/`lyrics_complete` flags. Job sits indefinitely.
2. **"Lyrics" and "Inst." are obsolete**: Combined review flow (Jan 2026) merged these into a single step. Should be replaced with single "Review" button that resets to `awaiting_review`.
3. **Missing state_data keys in clear list for `pending`**:
```python
# Should also clear:
"audio_complete", "lyrics_complete"
```
4. **State machine bypass**: Admin resets use `update_job()` directly, not `transition_to_state()`. This is intentional for admin overrides but should be documented.
**Fixes needed:**
```python
# backend/api/routes/admin.py
# 1. Update STATE_DATA_CLEAR_KEYS for "pending"
STATE_DATA_CLEAR_KEYS = {
"pending": [
# ... existing keys ...
"audio_complete", # NEW - clear parallel processing flag
"lyrics_complete", # NEW - clear parallel processing flag
],
# ...
}
# 2. Remove awaiting_instrumental_selection from ALLOWED_RESET_STATES
ALLOWED_RESET_STATES = {
"pending",
"awaiting_audio_selection",
"awaiting_review", # Combined review (lyrics + instrumental)
# "awaiting_instrumental_selection", # REMOVED - deprecated
"instrumental_selected",
}
# 3. After reset to pending, trigger audio search if job has artist/title
# or provide clear guidance to admin about next steps
```
```tsx
// frontend/app/admin/jobs/page.tsx
// Replace "Lyrics" and "Inst." buttons with single "Review" button:
{[
{ state: "pending", icon: RotateCcw, label: "Start" },
{ state: "awaiting_audio_selection", icon: Music, label: "Audio" },
{ state: "awaiting_review", icon: Mic, label: "Review" }, // Combined
// REMOVED: { state: "awaiting_instrumental_selection", ... }
{ state: "instrumental_selected", icon: RefreshCw, label: "Reprocess" },
]}
```
## Implementation Priority
1. **Week 1** ✓ DONE:
- ✓ Make `transition_to_state` raise by default (`InvalidStateTransitionError`, `raise_on_invalid=True`)
- ✓ Create `start_job_processing()` helper in `job_manager.py`
- ✓ Update `_handle_made_for_you_order` to use it
- ✓ Added 15 unit tests covering exception behavior and new helper
2. **Week 1.5** ✓ DONE (Admin Button Fixes):
- ✓ Add `audio_complete`, `lyrics_complete` to STATE_DATA_CLEAR_KEYS for "pending"
- ✓ Remove `awaiting_instrumental_selection` from ALLOWED_RESET_STATES
- ✓ Update frontend to replace "Lyrics"/"Inst." with single "Review" button
- ✓ Add tests for admin reset endpoint (`test_reset_to_pending_clears_parallel_processing_flags`, `test_rejects_deprecated_awaiting_instrumental_selection`)
3. **Week 2** ✓ DONE:
- ✓ Update `file_upload.py` and `audio_search.py` to use `start_job_processing()`
- ✓ Fixed double-transition bug in YouTube URL upload flow
- ✓ Removed unused `_trigger_workers_parallel` helper from file_upload.py and audio_search.py
- ✓ Add worker prerequisite validation (`validate_worker_can_run()` in all workers)
- ✓ Add state consistency health check (`job_health_service.py` with `check_job_consistency()`)
- ✓ Added 26 tests for job_health_service
4. **Week 3** ✓ DONE:
- ✓ Add integration tests for state flow (`tests/integration/test_state_machine_flow.py` - 11 tests)
- ✓ Generate state machine diagram script (`scripts/generate_state_diagram.py`)
- ✓ Expose `/health/job-consistency` endpoint for admin dashboard
- TODO: Add monitoring/alerting on `STATE_TRANSITION_FAILED` logs (requires Cloud Logging alert setup)
## Quick Wins (Can Do Now)
1. Add explicit error logging when transition fails (already exists but add job_id to structured logging)
2. Create Cloud Logging alert for "Invalid state transition" errors
3. Add consistency check to admin dashboard "stuck jobs" view
+78 -283
View File
@@ -2,7 +2,7 @@
import { useEffect, useState, useCallback, Suspense } from "react"
import { useRouter, useSearchParams } from "next/navigation"
import { adminApi, api, Job, FileInfo, JobFilesResponse, JobUpdateRequest, JobResetResponse, DeleteOutputsResponse, RegenerateScreensResponse, RestartJobResponse, OverrideAudioSourceResponse } from "@/lib/api"
import { adminApi, api, Job, FileInfo, JobFilesResponse, JobUpdateRequest, JobResetResponse, DeleteOutputsResponse, ClearWorkersResponse, TriggerWorkerResponse } from "@/lib/api"
import { useAdminSettings } from "@/lib/admin-settings"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
@@ -182,21 +182,11 @@ function AdminJobsPageContent() {
const [deleteOutputsResult, setDeleteOutputsResult] = useState<DeleteOutputsResponse | null>(null)
const [deleteOutputsResultOpen, setDeleteOutputsResultOpen] = useState(false)
// Note: clearingWorkers and triggeringWorker states removed
// These low-level escape hatches are replaced by Full Restart and Regen Screens
// Clear workers state
const [clearingWorkers, setClearingWorkers] = useState(false)
// Regenerate screens state
const [regeneratingScreens, setRegeneratingScreens] = useState(false)
const [regenerateScreensDialogOpen, setRegenerateScreensDialogOpen] = useState(false)
// Restart job state
const [restartDialogOpen, setRestartDialogOpen] = useState(false)
const [restarting, setRestarting] = useState(false)
const [restartPreserveStems, setRestartPreserveStems] = useState(true)
// Override audio source state
const [overrideAudioDialogOpen, setOverrideAudioDialogOpen] = useState(false)
const [overridingAudio, setOverridingAudio] = useState(false)
// Trigger worker state
const [triggeringWorker, setTriggeringWorker] = useState(false)
const loadJobs = useCallback(async () => {
try {
@@ -430,113 +420,74 @@ function AdminJobsPageContent() {
}
}
// Note: handleClearWorkers and handleTriggerWorker removed
// These low-level escape hatches are replaced by Full Restart and Regen Screens
// Handle regenerate screens - regenerates title/end screens with current metadata
const handleRegenerateScreens = async () => {
// Handle clear workers - clears worker progress to allow re-execution
const handleClearWorkers = async () => {
if (!selectedJobId) return
try {
setRegeneratingScreens(true)
const result = await adminApi.regenerateScreens(selectedJobId)
setClearingWorkers(true)
const result = await adminApi.clearWorkers(selectedJobId)
if (result.worker_triggered) {
if (result.cleared_keys.length > 0) {
toast({
title: "Screen Regeneration Started",
description: "Title and end screens are being regenerated with current artist/title. This may take 30-60 seconds.",
title: "Workers Cleared",
description: `${result.message}: ${result.cleared_keys.join(", ")}`,
})
} else {
toast({
title: "Regeneration Failed",
description: result.error || result.message,
title: "No Progress Keys",
description: result.message,
})
}
// Refresh job details
loadJobDetail(selectedJobId)
loadLogs(selectedJobId)
} catch (err: any) {
toast({
title: "Error",
description: err.message || "Failed to clear workers",
variant: "destructive",
})
} finally {
setClearingWorkers(false)
}
}
// Handle trigger worker - manually triggers a worker for processing
const handleTriggerWorker = async (workerType: string = "video") => {
if (!selectedJobId) return
try {
setTriggeringWorker(true)
const result = await adminApi.triggerWorker(selectedJobId, workerType)
if (result.triggered) {
toast({
title: `${workerType.charAt(0).toUpperCase() + workerType.slice(1)} Worker Triggered`,
description: `Job ${selectedJobId} is now processing. The ${workerType} worker will begin generating the video.`,
})
} else {
toast({
title: "Trigger Failed",
description: result.error
? `${result.message}: ${result.error}`
: result.message,
variant: "destructive",
})
}
setRegenerateScreensDialogOpen(false)
// Refresh job details
loadJobDetail(selectedJobId)
loadLogs(selectedJobId)
} catch (err: any) {
toast({
title: "Error",
description: err.message || "Failed to regenerate screens",
description: err.message || "Failed to trigger worker",
variant: "destructive",
})
} finally {
setRegeneratingScreens(false)
}
}
// Handle restart job - fully restarts a job with worker triggering
const handleRestartJob = async () => {
if (!selectedJobId) return
try {
setRestarting(true)
const result = await adminApi.restartJob(selectedJobId, {
preserve_audio_stems: restartPreserveStems,
delete_outputs: true,
})
if (result.status === "success" || result.status === "partial") {
toast({
title: "Job Restarted",
description: `${result.message}. Workers triggered: ${result.workers_triggered.join(", ") || "none (waiting for action)"}`,
})
}
if (result.error) {
toast({
title: "Warning",
description: result.error,
variant: "destructive",
})
}
setRestartDialogOpen(false)
// Refresh job details
loadJobDetail(selectedJobId)
loadLogs(selectedJobId)
} catch (err: any) {
toast({
title: "Error",
description: err.message || "Failed to restart job",
variant: "destructive",
})
} finally {
setRestarting(false)
}
}
// Handle override audio source - switches to audio search mode
const handleOverrideAudioSource = async () => {
if (!selectedJobId) return
try {
setOverridingAudio(true)
const result = await adminApi.overrideAudioSource(selectedJobId, {
source_type: "audio_search",
})
toast({
title: "Audio Source Changed",
description: result.message,
})
setOverrideAudioDialogOpen(false)
// Refresh job details
loadJobDetail(selectedJobId)
loadLogs(selectedJobId)
} catch (err: any) {
toast({
title: "Error",
description: err.message || "Failed to override audio source",
variant: "destructive",
})
} finally {
setOverridingAudio(false)
setTriggeringWorker(false)
}
}
@@ -545,7 +496,7 @@ function AdminJobsPageContent() {
const info: Record<string, { label: string; description: string; icon: any }> = {
pending: {
label: "Pending",
description: "Restart from the beginning. Clears all processing data including audio/lyrics progress.",
description: "Restart from the beginning. Clears all processing data.",
icon: RotateCcw,
},
awaiting_audio_selection: {
@@ -554,10 +505,15 @@ function AdminJobsPageContent() {
icon: Music,
},
awaiting_review: {
label: "Combined Review",
description: "Re-review lyrics AND re-select instrumental. Preserves audio stems.",
label: "Lyrics Review",
description: "Re-review lyrics. Preserves audio stems.",
icon: Mic,
},
awaiting_instrumental_selection: {
label: "Instrumental",
description: "Re-select instrumental. Preserves lyrics review.",
icon: Sliders,
},
instrumental_selected: {
label: "Reprocess Video",
description: "Re-encode and re-distribute with current settings. Triggers video worker automatically.",
@@ -858,9 +814,10 @@ function AdminJobsPageContent() {
<div className="flex items-center gap-2 flex-wrap">
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wide mr-1">Reset to:</span>
{[
// Note: "pending" (Start) removed - use "Full Restart" button instead
{ state: "pending", icon: RotateCcw, label: "Start" },
{ state: "awaiting_audio_selection", icon: Music, label: "Audio" },
{ state: "awaiting_review", icon: Mic, label: "Review" }, // Combined lyrics + instrumental review
{ state: "awaiting_review", icon: Mic, label: "Lyrics" },
{ state: "awaiting_instrumental_selection", icon: Sliders, label: "Inst." },
{ state: "instrumental_selected", icon: RefreshCw, label: "Reprocess" },
].map(({ state, icon: Icon, label }) => (
<Tooltip key={state}>
@@ -907,27 +864,21 @@ function AdminJobsPageContent() {
</TooltipContent>
</Tooltip>
{/* Admin Actions */}
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="outline"
size="sm"
className="h-7 px-2 text-xs border-blue-500 text-blue-600 hover:bg-blue-50 dark:hover:bg-blue-950"
onClick={() => setRegenerateScreensDialogOpen(true)}
disabled={regeneratingScreens}
className="h-7 px-2 text-xs"
onClick={handleClearWorkers}
disabled={clearingWorkers}
>
{regeneratingScreens ? (
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
) : (
<RefreshCw className="w-3 h-3 mr-1" />
)}
Regen Screens
<RefreshCw className={`w-3 h-3 mr-1 ${clearingWorkers ? "animate-spin" : ""}`} />
Clear Workers
</Button>
</TooltipTrigger>
<TooltipContent>
<p className="font-medium">Regenerate Title/End Screens</p>
<p className="text-xs text-muted-foreground">Use after editing artist/title to update screens</p>
Clear worker progress markers to allow re-execution
</TooltipContent>
</Tooltip>
@@ -936,52 +887,23 @@ function AdminJobsPageContent() {
<Button
variant="outline"
size="sm"
className="h-7 px-2 text-xs border-purple-500 text-purple-600 hover:bg-purple-50 dark:hover:bg-purple-950"
onClick={() => {
setRestartPreserveStems(true)
setRestartDialogOpen(true)
}}
disabled={restarting}
className={`h-7 px-2 text-xs ${selectedJob.status === "instrumental_selected" ? "border-green-500 text-green-600 hover:bg-green-50 dark:hover:bg-green-950" : ""}`}
onClick={() => handleTriggerWorker("video")}
disabled={triggeringWorker}
>
{restarting ? (
{triggeringWorker ? (
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
) : (
<RotateCcw className="w-3 h-3 mr-1" />
<Play className="w-3 h-3 mr-1" />
)}
Full Restart
Trigger
</Button>
</TooltipTrigger>
<TooltipContent>
<p className="font-medium">Full Restart with Worker Triggering</p>
<p className="text-xs text-muted-foreground">Unlike Reset, this actually triggers workers to run</p>
Manually trigger video worker processing
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="outline"
size="sm"
className="h-7 px-2 text-xs border-amber-500 text-amber-600 hover:bg-amber-50 dark:hover:bg-amber-950"
onClick={() => setOverrideAudioDialogOpen(true)}
disabled={overridingAudio}
>
{overridingAudio ? (
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
) : (
<Music className="w-3 h-3 mr-1" />
)}
Audio Search
</Button>
</TooltipTrigger>
<TooltipContent>
<p className="font-medium">Switch to Audio Search</p>
<p className="text-xs text-muted-foreground">Override YouTube URL with audio search for better quality</p>
</TooltipContent>
</Tooltip>
<Separator orientation="vertical" className="h-5 mx-1" />
<Tooltip>
<TooltipTrigger asChild>
<Button
@@ -1716,7 +1638,8 @@ function AdminJobsPageContent() {
<span className="font-medium">What happens next:</span>{" "}
{resetResult.new_status === "pending" && "Job will need to go through audio search, download, and all processing steps."}
{resetResult.new_status === "awaiting_audio_selection" && "User can select from the cached audio search results."}
{resetResult.new_status === "awaiting_review" && "User can review lyrics and select instrumental track."}
{resetResult.new_status === "awaiting_review" && "User can review and correct the lyrics."}
{resetResult.new_status === "awaiting_instrumental_selection" && "User can select the instrumental track."}
{resetResult.new_status === "instrumental_selected" && (
resetResult.worker_triggered
? "Video generation is now in progress."
@@ -1744,134 +1667,6 @@ function AdminJobsPageContent() {
</div>
</DialogContent>
</Dialog>
{/* Regenerate Screens Confirmation Dialog */}
<AlertDialog open={regenerateScreensDialogOpen} onOpenChange={setRegenerateScreensDialogOpen}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle className="flex items-center gap-2">
<RefreshCw className="w-5 h-5 text-blue-500" />
Regenerate Title/End Screens
</AlertDialogTitle>
<AlertDialogDescription>
This will regenerate the title and end screen videos using the <strong>current</strong> artist and title metadata.
<br /><br />
<strong>Current values:</strong>
<ul className="list-disc list-inside mt-2 space-y-1">
<li>Artist: <code className="bg-muted px-1 rounded">{selectedJob?.artist || "(not set)"}</code></li>
<li>Title: <code className="bg-muted px-1 rounded">{selectedJob?.title || "(not set)"}</code></li>
</ul>
<br />
Screen generation takes 30-60 seconds. Progress will be visible in the job timeline.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleRegenerateScreens}
disabled={regeneratingScreens}
className="bg-blue-500 hover:bg-blue-600"
>
{regeneratingScreens && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
Regenerate Screens
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
{/* Restart Job Confirmation Dialog */}
<AlertDialog open={restartDialogOpen} onOpenChange={setRestartDialogOpen}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle className="flex items-center gap-2">
<RotateCcw className="w-5 h-5 text-purple-500" />
Full Restart Job
</AlertDialogTitle>
<AlertDialogDescription asChild>
<div>
<p>Unlike &quot;Reset to Start&quot;, this will actually <strong>trigger workers</strong> to begin processing.</p>
<br />
<div className="flex items-center space-x-2 p-3 bg-muted rounded-lg">
<input
type="checkbox"
id="preserveStems"
checked={restartPreserveStems}
onChange={(e) => setRestartPreserveStems(e.target.checked)}
className="h-4 w-4 rounded border-gray-300"
/>
<label htmlFor="preserveStems" className="text-sm font-medium">
Preserve audio stems (faster)
</label>
</div>
<br />
{restartPreserveStems ? (
<p className="text-sm text-muted-foreground">
<strong>Quick restart:</strong> Keeps existing audio separation and lyrics. Only regenerates screens and video with current metadata. Good for fixing title/artist typos.
</p>
) : (
<p className="text-sm text-muted-foreground">
<strong>Full restart:</strong> Re-downloads audio (if YouTube), re-separates stems, re-transcribes lyrics. Use when audio source needs to change.
</p>
)}
</div>
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleRestartJob}
disabled={restarting}
className="bg-purple-500 hover:bg-purple-600"
>
{restarting && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
Restart Job
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
{/* Override Audio Source Confirmation Dialog */}
<AlertDialog open={overrideAudioDialogOpen} onOpenChange={setOverrideAudioDialogOpen}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle className="flex items-center gap-2">
<Music className="w-5 h-5 text-amber-500" />
Switch to Audio Search
</AlertDialogTitle>
<AlertDialogDescription>
This will clear the current audio source and switch to audio search mode.
<br /><br />
<strong>What happens:</strong>
<ul className="list-disc list-inside mt-2 space-y-1">
<li>YouTube URL and downloaded audio will be cleared</li>
<li>All processing state will be reset</li>
<li>Job will move to &quot;Awaiting Audio Selection&quot;</li>
<li>You can then search for and select higher quality audio</li>
</ul>
<br />
<strong>Current source:</strong>{" "}
{selectedJob?.url ? (
<span className="text-muted-foreground">YouTube URL</span>
) : selectedJob?.input_media_gcs_path ? (
<span className="text-muted-foreground">Uploaded file</span>
) : (
<span className="text-muted-foreground">Audio search</span>
)}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleOverrideAudioSource}
disabled={overridingAudio}
className="bg-amber-500 hover:bg-amber-600"
>
{overridingAudio && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
Switch to Audio Search
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
</TooltipProvider>
)
@@ -16,8 +16,8 @@ import { createEmailHelper, isEmailTestingAvailable } from '../helpers/email-tes
* 4. Create Job - Search for test song via UI
* 5. Audio Selection - Select audio source via UI dialog
* 6. Wait for Processing - Monitor via UI status updates
* 7. Combined Review - Open review UI, preview video, proceed to instrumental
* 8. Instrumental Selection - Select instrumental and submit (same page via hash nav)
* 7. Lyrics Review - Open review UI, interact, submit via UI
* 8. Instrumental Selection - Open selection UI, interact, submit via UI
* 9. Wait for Completion - Monitor via UI
* 10. Verify Downloads - Check download links work
* 11. Verify Distribution - Check YouTube/Dropbox/GDrive indicators
@@ -540,15 +540,11 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
console.log('STEP 6 COMPLETE: Lyrics processing finished');
// =========================================================================
// STEP 7: Combined Lyrics Review & Instrumental Selection (via UI - REAL USER FLOW)
// After the UI redesign, lyrics review and instrumental selection are combined:
// 1. User opens lyrics review page
// 2. User clicks "Preview Video" to open modal
// 3. User clicks "Proceed to Instrumental Review" which navigates to instrumental selection
// 4. User selects instrumental and clicks "Confirm & Continue"
// STEP 7: Lyrics Review (via UI - REAL USER FLOW)
// The minimal path: scroll to bottom -> click "Preview Video" -> wait for video -> click "Complete Review"
// =========================================================================
console.log('\n========================================');
console.log('STEP 7: Combined Lyrics Review & Instrumental Selection');
console.log('STEP 7: Lyrics Review (UI Interaction)');
console.log('========================================');
// Click the Review Lyrics button/link
@@ -625,7 +621,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
const errorText = await errorAlert.textContent();
console.log(` WARNING: Preview error: ${errorText}`);
await reviewPage.screenshot({ path: 'test-results/07e-preview-error.png', fullPage: true });
// Continue anyway - we can still proceed to instrumental even if preview failed
// Continue anyway - we can still click Complete Review even if preview failed
} else if (await videoElement.isVisible({ timeout: 10000 }).catch(() => false)) {
console.log(' Video element visible in modal');
// Give the video a moment to buffer/load
@@ -636,93 +632,31 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
await reviewPage.screenshot({ path: 'test-results/07f-video-state.png', fullPage: true });
// Click "Proceed to Instrumental Review" button in the modal
// This button saves corrections and navigates to the instrumental selection UI
const proceedBtn = reviewPage.getByRole('button', { name: /proceed to instrumental/i });
await expect(proceedBtn).toBeVisible({ timeout: TIMEOUTS.action });
console.log(' Found "Proceed to Instrumental Review" button');
// Click "Complete Review" button in the modal
// This button has variant="contained", color="success", and endIcon={<CloudUpload />}
const completeReviewBtn = reviewPage.getByRole('button', { name: /complete review/i });
await expect(completeReviewBtn).toBeVisible({ timeout: TIMEOUTS.action });
console.log(' Found "Complete Review" button');
await proceedBtn.click();
console.log(' Clicked "Proceed to Instrumental Review" button');
await completeReviewBtn.click();
console.log(' Clicked "Complete Review" button');
// Wait for the navigation to instrumental selection (happens via hash change)
// The page navigates to #/{jobId}/instrumental
await reviewPage.waitForTimeout(3000);
await reviewPage.screenshot({ path: 'test-results/07g-after-proceed.png', fullPage: true });
// =========================================================================
// STEP 8: Instrumental Selection (continuation of combined flow)
// The page has navigated to the instrumental selection UI via hash routing
// =========================================================================
console.log('\n========================================');
console.log('STEP 8: Instrumental Selection');
console.log('========================================');
// Wait for the instrumental selection UI to load
// The InstrumentalSelector component renders selection options
console.log(' Waiting for instrumental selection UI to load...');
// Wait for either the selection options or the loading indicator to appear
try {
await reviewPage.waitForSelector('.selection-option, .selection-panel, [class*="selection"]', { timeout: 30000 });
console.log(' Instrumental selection UI loaded');
} catch {
// If selector not found, check for loading state
const loadingState = reviewPage.getByText(/loading instrumental/i);
if (await loadingState.isVisible({ timeout: 5000 }).catch(() => false)) {
console.log(' Waiting for instrumental analysis to complete...');
await expect(loadingState).not.toBeVisible({ timeout: 60000 });
}
}
await reviewPage.screenshot({ path: 'test-results/08a-instrumental-opened.png', fullPage: true });
console.log(' Instrumental selection UI opened');
// Select "Clean" instrumental option
// The options have class "selection-option" and contain labels with "Clean" or "With Backing"
const cleanOption = reviewPage.locator('.selection-option:has-text("Clean")').first();
if (await cleanOption.isVisible({ timeout: 5000 }).catch(() => false)) {
await cleanOption.click();
console.log(' Selected "Clean" instrumental option');
} else {
// If we can't find "Clean" specifically, the first option might already be selected
console.log(' Clean option not found - using default selection');
}
await reviewPage.screenshot({ path: 'test-results/08b-clean-selected.png', fullPage: true });
// Click the submit button: "✓ Confirm & Continue" (id="submit-btn")
const submitBtn = reviewPage.locator('#submit-btn');
await expect(submitBtn).toBeVisible({ timeout: TIMEOUTS.action });
console.log(' Found submit button');
await submitBtn.click();
console.log(' Clicked "Confirm & Continue" button');
// Wait for submission to complete - the page shows a success screen then redirects
// In cloud mode, after success it redirects to /app after a countdown
// Wait for the submission to process
// The handleSubmitToServer function calls apiClient.submitCorrections() then window.close()
await reviewPage.waitForTimeout(5000);
await reviewPage.screenshot({ path: 'test-results/08c-instrumental-submitted.png', fullPage: true });
await reviewPage.screenshot({ path: 'test-results/07g-after-complete-review.png', fullPage: true });
// The page redirects to /app after completion, or may close
// Wait a bit for the redirect/close to happen
// The review page may close automatically after submission (window.close())
// If it's still open, close it manually
if (!reviewPage.isClosed()) {
// Wait for potential redirect or success screen
try {
await reviewPage.waitForURL(/\/app/, { timeout: 10000 });
console.log(' Redirected to /app after instrumental selection');
} catch {
// If no redirect, close manually
await reviewPage.close();
console.log(' Closed review UI');
}
await reviewPage.close();
console.log(' Closed review UI');
} else {
console.log(' Review UI closed automatically after submission');
}
// Refresh main app
// Refresh main app to see updated status
await page.bringToFront();
const refreshBtn = page.getByRole('button', { name: /refresh/i });
if (await refreshBtn.isVisible().catch(() => false)) {
@@ -730,10 +664,121 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
await page.waitForTimeout(3000);
}
await reviewPage.screenshot({ path: 'test-results/08d-after-instrumental.png', fullPage: true }).catch(() => {
// reviewPage may be closed already, that's OK
});
await page.screenshot({ path: 'test-results/08d-main-after-instrumental.png' });
await page.screenshot({ path: 'test-results/07h-after-review-refresh.png' });
console.log('STEP 7 COMPLETE: Lyrics review submitted via UI');
// =========================================================================
// STEP 8: Wait for Video Rendering & Instrumental Selection (via UI - REAL USER FLOW)
// The instrumental UI has: selection-option elements, submit button with id="submit-btn"
// =========================================================================
console.log('\n========================================');
console.log('STEP 8: Wait for Video Rendering & Instrumental Selection');
console.log('========================================');
// Wait for instrumental selection to become available
console.log(' Waiting for video rendering and instrumental selection...');
let foundInstrumentalStatus = false;
const renderStartTime = Date.now();
while (Date.now() - renderStartTime < TIMEOUTS.videoRendering) {
// Refresh
if (await refreshBtn.isVisible().catch(() => false)) {
await refreshBtn.click();
await page.waitForTimeout(2000);
}
// Check job card status
const statusText = await jobCard.textContent() || '';
console.log(` Status: ${statusText.substring(0, 80)}...`);
// Look for instrumental selection UI availability
// Users ALWAYS have to go through the instrumental review UI - there's no skip
if (statusText.toLowerCase().includes('select instrumental') ||
statusText.toLowerCase().includes('awaiting_instrumental') ||
(statusText.toLowerCase().includes('action needed') && statusText.toLowerCase().includes('instrumental'))) {
foundInstrumentalStatus = true;
console.log(' Job ready for instrumental selection');
break;
}
if (statusText.toLowerCase().includes('failed')) {
throw new Error('Job failed during video rendering');
}
await page.waitForTimeout(10000);
}
if (!foundInstrumentalStatus) {
throw new Error('Timeout waiting for instrumental selection - users must always complete this step');
}
{
// Click the Select Instrumental button/link
const instrumentalLink = jobCard.getByRole('link', { name: /select instrumental/i });
await expect(instrumentalLink).toBeVisible({ timeout: TIMEOUTS.action });
const instrumentalUrl = await instrumentalLink.getAttribute('href');
console.log(` Instrumental URL: ${instrumentalUrl}`);
// Open instrumental UI in new page
const instrumentalPage = await context.newPage();
// Handle both absolute and relative URLs
const fullUrl = instrumentalUrl?.startsWith('http') ? instrumentalUrl : `${PROD_URL}${instrumentalUrl}`;
// Use 'load' instead of 'networkidle' - instrumental page has ongoing audio requests
// that prevent networkidle from ever being reached
await instrumentalPage.goto(fullUrl, { waitUntil: 'load' });
await instrumentalPage.waitForTimeout(3000);
await instrumentalPage.screenshot({ path: 'test-results/08a-instrumental-opened.png', fullPage: true });
console.log(' Instrumental selection UI opened');
// Wait for the page to fully load - the instrumental UI renders selection options
await instrumentalPage.waitForSelector('.selection-option, .selection-panel', { timeout: 30000 });
// Select "Clean" instrumental option
// The options have class "selection-option" and contain labels with "Clean" or "With Backing"
const cleanOption = instrumentalPage.locator('.selection-option:has-text("Clean")').first();
if (await cleanOption.isVisible({ timeout: 5000 }).catch(() => false)) {
await cleanOption.click();
console.log(' Selected "Clean" instrumental option');
} else {
// If we can't find "Clean" specifically, the first option might already be selected
console.log(' Clean option not found - using default selection');
}
await instrumentalPage.screenshot({ path: 'test-results/08b-clean-selected.png', fullPage: true });
// Click the submit button: "✓ Confirm & Continue" (id="submit-btn")
const submitBtn = instrumentalPage.locator('#submit-btn');
await expect(submitBtn).toBeVisible({ timeout: TIMEOUTS.action });
console.log(' Found submit button');
await submitBtn.click();
console.log(' Clicked "Confirm & Continue" button');
// Wait for submission to complete - the page shows a success screen then auto-closes
await instrumentalPage.waitForTimeout(5000);
await instrumentalPage.screenshot({ path: 'test-results/08c-instrumental-submitted.png', fullPage: true });
// The page may auto-close after submission
if (!instrumentalPage.isClosed()) {
await instrumentalPage.close();
console.log(' Closed instrumental UI');
} else {
console.log(' Instrumental UI closed automatically after submission');
}
}
// Refresh main app
await page.bringToFront();
if (await refreshBtn.isVisible().catch(() => false)) {
await refreshBtn.click();
await page.waitForTimeout(3000);
}
await page.screenshot({ path: 'test-results/08d-after-instrumental.png' });
console.log('STEP 8 COMPLETE: Instrumental selection handled via UI');
// =========================================================================
@@ -190,51 +190,43 @@ test.describe('Lyrics Review Only - Focused Test', () => {
await page.screenshot({ path: 'test-results/focused-06-video-state.png', fullPage: true });
// Click "Proceed to Instrumental Review" button in the modal
// (Previously "Complete Review" - changed with combined review UI)
console.log('6. Looking for "Proceed to Instrumental Review" button...');
// Click "Complete Review" button in the modal
console.log('6. Looking for "Complete Review" button...');
// List all buttons in the modal
const modalButtons = await previewModal.getByRole('button').allTextContents();
console.log(` Modal buttons: ${modalButtons.join(', ')}`);
const proceedBtn = page.getByRole('button', { name: /proceed to instrumental/i });
const completeReviewBtn = page.getByRole('button', { name: /complete review/i });
if (await proceedBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
console.log(' Found "Proceed to Instrumental Review" button - clicking...');
await proceedBtn.click();
console.log(' Clicked "Proceed to Instrumental Review" button');
if (await completeReviewBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
console.log(' Found "Complete Review" button - clicking...');
await completeReviewBtn.click();
console.log(' Clicked "Complete Review" button');
} else {
// Try to find any submit-like button (for backwards compatibility)
const submitBtn = previewModal.getByRole('button', { name: /submit|confirm|done|save|complete/i });
// Try to find any submit-like button
const submitBtn = previewModal.getByRole('button', { name: /submit|confirm|done|save/i });
if (await submitBtn.isVisible().catch(() => false)) {
const btnText = await submitBtn.textContent();
console.log(` Found alternative submit button: "${btnText}" - clicking...`);
await submitBtn.click();
} else {
console.log(' Could not find Proceed to Instrumental or similar button');
await page.screenshot({ path: 'test-results/focused-07-no-proceed-btn.png', fullPage: true });
throw new Error('Could not find Proceed to Instrumental Review button');
console.log(' Could not find Complete Review or similar button');
await page.screenshot({ path: 'test-results/focused-07-no-complete-btn.png', fullPage: true });
throw new Error('Could not find Complete Review button');
}
}
// Wait for navigation to instrumental selection
console.log('7. Waiting for navigation to instrumental selection...');
// Wait for the submission to process
console.log('7. Waiting for submission to complete...');
await page.waitForTimeout(5000);
await page.screenshot({ path: 'test-results/focused-08-after-proceed.png', fullPage: true });
// Note: This focused test only covers the lyrics review portion.
// The instrumental selection happens next on the same page via hash navigation.
// For full flow testing, use happy-path-real-user.spec.ts
await page.screenshot({ path: 'test-results/focused-08-after-submit.png', fullPage: true });
console.log('');
console.log('========================================');
console.log('FOCUSED TEST COMPLETE');
console.log('========================================');
console.log('NOTE: This test completes the lyrics review step only.');
console.log(' The instrumental selection UI should now be visible on this page.');
console.log(' For the full flow, use happy-path-real-user.spec.ts');
});
test('Add Reference Lyrics via API', async ({ page, context, request }) => {
@@ -256,28 +256,27 @@ async function testLyricsReview(page: any, context: any, jobId: string) {
console.log(' WARNING: No video element found');
}
// Click "Proceed to Instrumental Review" button (previously "Complete Review")
console.log('10. Clicking "Proceed to Instrumental Review" button...');
// Click "Complete Review" button
console.log('10. Clicking "Complete Review" button...');
const modalButtons = await previewModal.getByRole('button').allTextContents();
console.log(` Modal buttons: ${modalButtons.join(', ')}`);
const proceedBtn = reviewPage.getByRole('button', { name: /proceed to instrumental/i });
const completeReviewBtn = reviewPage.getByRole('button', { name: /complete review/i });
if (await proceedBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
await proceedBtn.click();
console.log(' Clicked Proceed to Instrumental Review button');
if (await completeReviewBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
await completeReviewBtn.click();
console.log(' Clicked Complete Review button');
} else {
await reviewPage.screenshot({ path: 'test-results/existing-09-no-proceed-btn.png', fullPage: true });
throw new Error('Could not find Proceed to Instrumental Review button');
await reviewPage.screenshot({ path: 'test-results/existing-09-no-complete-btn.png', fullPage: true });
throw new Error('Could not find Complete Review button');
}
// Wait for navigation to instrumental selection (happens via hash change on same page)
console.log('11. Waiting for navigation to instrumental selection...');
// Wait for submission
console.log('11. Waiting for submission...');
await reviewPage.waitForTimeout(5000);
await reviewPage.screenshot({ path: 'test-results/existing-10-after-proceed.png', fullPage: true });
console.log(' Lyrics review step completed - now on instrumental selection!');
console.log(' NOTE: With combined UI, instrumental selection is on the same page.');
await reviewPage.screenshot({ path: 'test-results/existing-10-after-submit.png', fullPage: true });
console.log(' Lyrics review completed successfully!');
await reviewPage.close();
}
-93
View File
@@ -1454,59 +1454,6 @@ export const adminApi = {
return handleResponse(response);
},
/**
* Regenerate title and end screens with current artist/title metadata (admin only).
* Use when you've edited artist/title and need screens to reflect the new metadata.
*/
async regenerateScreens(jobId: string): Promise<RegenerateScreensResponse> {
const response = await fetch(
`${API_BASE_URL}/api/admin/jobs/${jobId}/regenerate-screens`,
{
method: 'POST',
headers: getAuthHeaders()
}
);
return handleResponse(response);
},
/**
* Fully restart a job from the beginning (admin only).
* Unlike reset (which just changes state), restart actually triggers workers.
*/
async restartJob(jobId: string, options: RestartJobRequest): Promise<RestartJobResponse> {
const response = await fetch(
`${API_BASE_URL}/api/admin/jobs/${jobId}/restart`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
...getAuthHeaders()
},
body: JSON.stringify(options),
}
);
return handleResponse(response);
},
/**
* Override the audio source for a job (admin only).
* Switch from YouTube URL to audio search mode.
*/
async overrideAudioSource(jobId: string, request: OverrideAudioSourceRequest): Promise<OverrideAudioSourceResponse> {
const response = await fetch(
`${API_BASE_URL}/api/admin/jobs/${jobId}/override-audio-source`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
...getAuthHeaders()
},
body: JSON.stringify(request),
}
);
return handleResponse(response);
},
// =========================================================================
// Rate Limits API
// =========================================================================
@@ -1745,46 +1692,6 @@ export interface ClearWorkersResponse {
cleared_keys: string[];
}
export interface RegenerateScreensResponse {
status: string;
job_id: string;
message: string;
previous_screens_deleted: boolean;
worker_triggered: boolean;
error?: string;
}
export interface RestartJobRequest {
preserve_audio_stems: boolean;
delete_outputs: boolean;
}
export interface RestartJobResponse {
status: string;
job_id: string;
message: string;
previous_status: string;
new_status: string;
cleared_data: string[];
deleted_gcs_paths: string[];
workers_triggered: string[];
error?: string;
}
export interface OverrideAudioSourceRequest {
source_type: "audio_search";
}
export interface OverrideAudioSourceResponse {
status: string;
job_id: string;
message: string;
previous_source: string;
new_source: string;
cleared_data: string[];
new_status: string;
}
// ==========================================================================
// Lyrics Review API endpoints
// ==========================================================================
+2 -17
View File
@@ -37,7 +37,7 @@ from config import (
DiskSizes,
)
from modules import database, storage as storage_module, artifact_registry, secrets
from modules import cloud_tasks, cloud_run, monitoring, networking, runner_manager
from modules import cloud_tasks, cloud_run, monitoring, networking
from modules.iam import backend_sa, github_actions_sa, claude_automation_sa, worker_sas
from compute import encoding_worker_vm, github_runners
@@ -249,18 +249,8 @@ encoding_worker_instance = encoding_worker_vm.create_encoding_worker_vm(
encoding_worker_firewall = encoding_worker_vm.create_encoding_worker_firewall()
# GitHub Runners (CI/CD self-hosted runners)
# Create Cloud NAT for outbound internet access (no external IPs needed)
github_runners_router, github_runners_nat = github_runners.create_cloud_nat()
# Create runner VMs (Spot instances for 60-91% cost savings)
github_runner_vms = github_runners.create_github_runners(
github_runner_sa, all_secrets["github-runner-pat"], github_runners_nat
)
# Create runner manager (auto-start/stop VMs based on CI activity)
runner_manager_resources = runner_manager.create_runner_manager_resources(
all_secrets["github-webhook-secret"],
all_secrets["github-runner-pat"],
github_runner_sa, all_secrets["github-runner-pat"]
)
@@ -344,8 +334,3 @@ pulumi.export("encoding_worker_vm_name", encoding_worker_instance.name)
# GitHub runners
pulumi.export("github_runner_vm_names", [vm.name for vm in github_runner_vms])
pulumi.export("github_runner_service_account", github_runner_sa.email)
pulumi.export("github_runners_nat", github_runners_nat.name)
# GitHub runner manager (auto-start/stop)
pulumi.export("runner_manager_function_url", runner_manager_resources["function"].url)
pulumi.export("runner_manager_scheduler_job", runner_manager_resources["scheduler_job"].name)
+1 -2
View File
@@ -11,12 +11,11 @@ from .encoding_worker_vm import (
create_encoding_worker_ip,
create_encoding_worker_firewall,
)
from .github_runners import create_github_runners, create_cloud_nat
from .github_runners import create_github_runners
__all__ = [
"create_encoding_worker_vm",
"create_encoding_worker_ip",
"create_encoding_worker_firewall",
"create_github_runners",
"create_cloud_nat",
]
+15 -125
View File
@@ -1,82 +1,30 @@
"""
GitHub Actions Self-Hosted Runners on GCP.
GitHub Actions self-hosted runner VMs.
Creates Spot/Preemptible VMs that register as self-hosted runners
at the organization level. Uses Spot instances for 60-91% cost savings.
Resources created:
- Cloud Router and Cloud NAT (for outbound internet without external IPs)
- N Compute Engine VMs (Spot instances)
- Each VM runs the GitHub Actions runner agent
The VMs are managed by the runner_manager Cloud Function which:
- Starts VMs when CI jobs are queued (via GitHub webhook)
- Stops VMs when idle for too long (via Cloud Scheduler)
Manages multiple VM instances that serve as self-hosted runners
for GitHub Actions CI/CD workflows.
"""
import pulumi
from pulumi_gcp import compute, secretmanager, serviceaccount
from pulumi_gcp import compute, serviceaccount, secretmanager
from config import REGION, ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
from config import ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
from .startup_scripts import read_script
def create_cloud_nat() -> tuple[compute.Router, compute.RouterNat]:
"""
Create Cloud Router and Cloud NAT for outbound internet access.
This allows VMs without external IPs to access the internet,
which saves costs and improves security.
Returns:
tuple: (router, nat) resources
"""
# Create Cloud Router
router = compute.Router(
"github-runners-router",
name="github-runners-router",
region=REGION,
network="default",
description="Router for GitHub Actions runner NAT",
)
# Create Cloud NAT
nat = compute.RouterNat(
"github-runners-nat",
name="github-runners-nat",
router=router.name,
region=REGION,
nat_ip_allocate_option="AUTO_ONLY",
source_subnetwork_ip_ranges_to_nat="ALL_SUBNETWORKS_ALL_IP_RANGES",
log_config=compute.RouterNatLogConfigArgs(
enable=True,
filter="ERRORS_ONLY",
),
opts=pulumi.ResourceOptions(depends_on=[router]),
)
return router, nat
def create_github_runners(
service_account: serviceaccount.Account,
runner_pat_secret: secretmanager.Secret,
nat: compute.RouterNat,
) -> list[compute.Instance]:
"""
Create GitHub Actions self-hosted runner VM instances.
Creates NUM_GITHUB_RUNNERS Spot VMs, each configured as a GitHub Actions
Creates NUM_GITHUB_RUNNERS VMs, each configured as a GitHub Actions
self-hosted runner for the nomadkaraoke organization (available to all repos).
Uses Spot instances (preemptible) for 60-91% cost savings.
VMs start in TERMINATED state and are started by the runner_manager
Cloud Function when CI jobs are queued.
Args:
service_account: The GitHub runner service account.
runner_pat_secret: The Secret Manager secret containing the GitHub PAT.
nat: Cloud NAT for outbound internet access (VMs have no external IPs).
Returns:
list[compute.Instance]: List of runner VM instances.
@@ -85,91 +33,33 @@ def create_github_runners(
runner_vms = []
for i in range(1, NUM_GITHUB_RUNNERS + 1):
instance_name = f"github-runner-{i}"
vm = compute.Instance(
instance_name,
name=instance_name,
f"github-runner-{i}",
name=f"github-runner-{i}",
machine_type=MachineTypes.GITHUB_RUNNER,
zone=ZONE,
# Boot disk configuration
boot_disk=compute.InstanceBootDiskArgs(
initialize_params=compute.InstanceBootDiskInitializeParamsArgs(
image="debian-cloud/debian-12",
size=DiskSizes.GITHUB_RUNNER,
type="pd-ssd", # SSD for faster Docker builds
type="pd-ssd",
),
auto_delete=True,
),
# Network configuration - no external IP (uses Cloud NAT)
# This saves ~$3/month per VM and improves security
network_interfaces=[
compute.InstanceNetworkInterfaceArgs(
network="default",
# No access_configs = no external IP
# Outbound internet via Cloud NAT
)
],
# Service account for GCP API access
network_interfaces=[compute.InstanceNetworkInterfaceArgs(
network="default",
access_configs=[compute.InstanceNetworkInterfaceAccessConfigArgs()],
)],
service_account=compute.InstanceServiceAccountArgs(
email=service_account.email,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
scopes=["cloud-platform"],
),
# Spot/Preemptible instance configuration (60-91% cheaper)
# Preemptible VMs may be terminated at any time but cost much less.
# The startup script re-registers the runner on boot.
scheduling=compute.InstanceSchedulingArgs(
preemptible=True,
automatic_restart=False, # Required for preemptible
on_host_maintenance="TERMINATE", # Required for preemptible
),
# Metadata for startup script
metadata={
"github-runner-pat-secret": "github-runner-pat",
"github-org": "nomadkaraoke",
"runner-labels": "self-hosted,linux,x64,gcp,large-disk",
},
metadata_startup_script=startup_script,
# Labels for organization and cost tracking
labels={
"purpose": "github-runner",
"managed-by": "pulumi",
},
tags=["github-runner"],
# Allow stopping for updates
allow_stopping_for_update=True,
# Deletion protection disabled (runners are ephemeral)
deletion_protection=False,
opts=pulumi.ResourceOptions(
depends_on=[runner_pat_secret, nat],
# Runners can be replaced without affecting other infrastructure
delete_before_replace=True,
depends_on=[runner_pat_secret],
),
)
runner_vms.append(vm)
return runner_vms
def create_instance_group_for_restart(
instances: list[compute.Instance],
) -> compute.InstanceGroup:
"""
Create an unmanaged instance group for the runner VMs.
This enables bulk operations like restarting all runners.
Args:
instances: List of runner VM instances.
Returns:
compute.InstanceGroup: The instance group.
"""
return compute.InstanceGroup(
"github-runners-group",
name="github-runners",
description="Self-hosted GitHub Actions runners",
zone=ZONE,
instances=[instance.self_link for instance in instances],
)
+1 -19
View File
@@ -87,29 +87,11 @@ class QueueConfigs:
# Number of self-hosted GitHub Action runners
# Reduced from 20 to 3 - runners auto-scale with start/stop management
NUM_GITHUB_RUNNERS = 3
NUM_GITHUB_RUNNERS = 20
# Google Drive folder ID for validator (public share folder)
GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
class RunnerManagerConfig:
"""Configuration for GitHub runner auto-start/stop management."""
FUNCTION_NAME = "github-runner-manager"
FUNCTION_MEMORY = "256M"
FUNCTION_TIMEOUT = 300 # 5 minutes
IDLE_TIMEOUT_HOURS = 1 # Shorter than Aquarius (3h) since we have more CI activity
IDLE_CHECK_SCHEDULE = "*/15 * * * *" # Every 15 minutes
class SecretNames:
"""Secret Manager secret names."""
GITHUB_RUNNER_PAT = "github-runner-pat"
GITHUB_WEBHOOK_SECRET = "github-webhook-secret"
# GitHub repository for runner registration
GITHUB_REPO_OWNER = "nomadkaraoke"
GITHUB_REPO_NAME = "karaoke-gen"
@@ -1,417 +0,0 @@
"""
GitHub Actions Runner Manager Cloud Function.
Handles two types of triggers:
1. GitHub Webhook (workflow_job.queued) - Starts runner VMs when CI jobs are queued
2. Cloud Scheduler (every 15 min) - Stops idle runner VMs after timeout
Environment variables:
- GCP_PROJECT: GCP project ID
- GCP_ZONE: Zone where runner VMs are located
- WEBHOOK_SECRET_NAME: Secret Manager secret name for webhook verification
- RUNNER_PAT_SECRET_NAME: Secret Manager secret name for GitHub PAT
- IDLE_TIMEOUT_HOURS: Hours of inactivity before stopping runners (default: 1)
"""
import hashlib
import hmac
import json
import os
from datetime import datetime, timezone
import functions_framework
from flask import Request
from google.cloud import compute_v1, secretmanager
# Configuration from environment
PROJECT_ID = os.environ.get("GCP_PROJECT", "nomadkaraoke")
ZONE = os.environ.get("GCP_ZONE", "us-central1-a")
WEBHOOK_SECRET_NAME = os.environ.get("WEBHOOK_SECRET_NAME", "github-webhook-secret")
RUNNER_PAT_SECRET_NAME = os.environ.get("RUNNER_PAT_SECRET_NAME", "github-runner-pat")
IDLE_TIMEOUT_HOURS = int(os.environ.get("IDLE_TIMEOUT_HOURS", "1"))
RUNNER_PREFIX = "github-runner-"
NUM_RUNNERS = 3
# Lazy-loaded clients and secrets
_compute_client = None
_secret_client = None
_webhook_secret = None
_github_pat = None
def get_compute_client() -> compute_v1.InstancesClient:
"""Get or create the Compute Engine client."""
global _compute_client
if _compute_client is None:
_compute_client = compute_v1.InstancesClient()
return _compute_client
def get_secret_client() -> secretmanager.SecretManagerServiceClient:
"""Get or create the Secret Manager client."""
global _secret_client
if _secret_client is None:
_secret_client = secretmanager.SecretManagerServiceClient()
return _secret_client
def get_webhook_secret() -> str:
"""Get the webhook secret from Secret Manager (cached)."""
global _webhook_secret
if _webhook_secret is None:
client = get_secret_client()
name = f"projects/{PROJECT_ID}/secrets/{WEBHOOK_SECRET_NAME}/versions/latest"
response = client.access_secret_version(request={"name": name})
_webhook_secret = response.payload.data.decode("utf-8").strip()
return _webhook_secret
def get_github_pat() -> str:
"""Get the GitHub PAT from Secret Manager (cached)."""
global _github_pat
if _github_pat is None:
client = get_secret_client()
name = f"projects/{PROJECT_ID}/secrets/{RUNNER_PAT_SECRET_NAME}/versions/latest"
response = client.access_secret_version(request={"name": name})
_github_pat = response.payload.data.decode("utf-8").strip()
return _github_pat
def verify_webhook_signature(request: Request) -> bool:
"""
Verify the GitHub webhook signature.
GitHub sends a signature in the X-Hub-Signature-256 header.
We compute the expected signature and compare.
"""
signature_header = request.headers.get("X-Hub-Signature-256")
if not signature_header:
print("Missing X-Hub-Signature-256 header")
return False
# Get the raw body for signature verification
body = request.get_data()
secret = get_webhook_secret()
# Compute expected signature
expected_signature = (
"sha256="
+ hmac.new(
secret.encode("utf-8"),
body,
hashlib.sha256,
).hexdigest()
)
# Constant-time comparison to prevent timing attacks
return hmac.compare_digest(expected_signature, signature_header)
def get_runner_instances() -> list[compute_v1.Instance]:
"""Get all GitHub runner instances."""
client = get_compute_client()
instances = []
for i in range(1, NUM_RUNNERS + 1):
instance_name = f"{RUNNER_PREFIX}{i}"
try:
instance = client.get(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
)
instances.append(instance)
except Exception as e:
print(f"Error getting instance {instance_name}: {e}")
return instances
def start_runners() -> dict:
"""
Start any TERMINATED runner VMs.
Returns a dict with lists of started, already_running, and failed instances.
"""
client = get_compute_client()
result = {"started": [], "already_running": [], "failed": []}
for i in range(1, NUM_RUNNERS + 1):
instance_name = f"{RUNNER_PREFIX}{i}"
try:
instance = client.get(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
)
if instance.status == "TERMINATED":
print(f"Starting {instance_name}...")
operation = client.start(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
)
# Wait for the operation to complete (timeout after 5 min)
try:
operation.result(timeout=300)
print(f"{instance_name} started successfully")
result["started"].append(instance_name)
except Exception as op_error:
print(f"Start operation failed for {instance_name}: {op_error}")
result["failed"].append(instance_name)
elif instance.status == "RUNNING":
result["already_running"].append(instance_name)
else:
print(f"{instance_name} is in state {instance.status}")
result["already_running"].append(instance_name)
except Exception as e:
print(f"Error with {instance_name}: {e}")
result["failed"].append(instance_name)
return result
def update_instance_metadata(instance_name: str, key: str, value: str) -> None:
"""Update a single metadata key on an instance."""
client = get_compute_client()
# Get current instance to get fingerprint
instance = client.get(project=PROJECT_ID, zone=ZONE, instance=instance_name)
current_metadata = instance.metadata
# Build new items list, replacing or adding the key
new_items = []
found = False
for item in current_metadata.items:
if item.key == key:
new_items.append(compute_v1.Items(key=key, value=value))
found = True
else:
new_items.append(item)
if not found:
new_items.append(compute_v1.Items(key=key, value=value))
# Update metadata
metadata = compute_v1.Metadata(
fingerprint=current_metadata.fingerprint,
items=new_items,
)
client.set_metadata(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
metadata_resource=metadata,
)
def get_instance_metadata(instance: compute_v1.Instance, key: str) -> str | None:
"""Get a metadata value from an instance."""
if not instance.metadata or not instance.metadata.items:
return None
for item in instance.metadata.items:
if item.key == key:
return item.value
return None
def check_github_for_pending_jobs() -> bool:
"""
Check GitHub API for any pending/queued jobs for our runners.
Returns True if there are jobs waiting for our self-hosted runners.
"""
import urllib.request
pat = get_github_pat()
org = "nomadkaraoke"
# Check for queued workflow runs in karaoke-gen repo
url = f"https://api.github.com/repos/{org}/karaoke-gen/actions/runs?status=queued"
req = urllib.request.Request(
url,
headers={
"Authorization": f"Bearer {pat}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
},
)
try:
with urllib.request.urlopen(req, timeout=10) as response:
data = json.loads(response.read().decode("utf-8"))
if data.get("total_count", 0) > 0:
print(f"Found {data['total_count']} queued runs in karaoke-gen")
return True
except Exception as e:
print(f"Error checking GitHub API: {e}")
return False
def check_and_stop_idle_runners() -> dict:
"""
Check for idle runners and stop them if idle for IDLE_TIMEOUT_HOURS.
A runner is considered idle if:
1. It's RUNNING
2. No jobs are pending/queued on GitHub
3. last-activity metadata is older than IDLE_TIMEOUT_HOURS
Returns a dict with stopped and kept instances.
"""
result = {"stopped": [], "kept": [], "errors": []}
# First check if there are any pending jobs
if check_github_for_pending_jobs():
print("Jobs are pending - keeping all runners active")
# Update last-activity on all running instances
for instance in get_runner_instances():
if instance.status == "RUNNING":
result["kept"].append(instance.name)
# Refresh activity timestamp since there are pending jobs
try:
update_instance_metadata(
instance.name,
"last-activity",
datetime.now(timezone.utc).isoformat(),
)
except Exception as e:
print(f"Error updating metadata for {instance.name}: {e}")
return result
# No pending jobs - check each instance for idle timeout
client = get_compute_client()
now = datetime.now(timezone.utc)
for instance in get_runner_instances():
if instance.status != "RUNNING":
continue
# Get last-activity timestamp
last_activity_str = get_instance_metadata(instance, "last-activity")
if not last_activity_str:
# No activity recorded - set it now and don't stop
print(f"{instance.name}: No last-activity metadata, setting now")
try:
update_instance_metadata(
instance.name,
"last-activity",
now.isoformat(),
)
except Exception as e:
print(f"Error setting metadata for {instance.name}: {e}")
result["kept"].append(instance.name)
continue
try:
last_activity = datetime.fromisoformat(last_activity_str.replace("Z", "+00:00"))
idle_hours = (now - last_activity).total_seconds() / 3600
if idle_hours >= IDLE_TIMEOUT_HOURS:
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, stopping...")
client.stop(
project=PROJECT_ID,
zone=ZONE,
instance=instance.name,
)
result["stopped"].append(instance.name)
else:
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, keeping...")
result["kept"].append(instance.name)
except Exception as e:
print(f"Error processing {instance.name}: {e}")
result["errors"].append(instance.name)
return result
@functions_framework.http
def handle_request(request: Request):
"""
Main entry point for the Cloud Function.
Handles:
1. Cloud Scheduler triggers (action=check_idle parameter)
2. GitHub webhook events (workflow_job.queued)
"""
# Check if this is a scheduler trigger (idle check)
action = request.args.get("action")
if action == "check_idle":
print("Scheduler trigger: checking for idle runners")
result = check_and_stop_idle_runners()
return json.dumps(result), 200, {"Content-Type": "application/json"}
# Otherwise, this should be a GitHub webhook
# Verify the webhook signature
if not verify_webhook_signature(request):
print("Webhook signature verification failed")
return "Unauthorized", 401
# Get the event type
event = request.headers.get("X-GitHub-Event")
if not event:
return "Missing X-GitHub-Event header", 400
# We only care about workflow_job events
if event != "workflow_job":
return "OK - event ignored", 200
# Parse the payload
try:
payload = request.get_json()
except Exception as e:
print(f"Error parsing JSON: {e}")
return "Invalid JSON", 400
action = payload.get("action")
job = payload.get("workflow_job", {})
labels = job.get("labels", [])
print(f"Received workflow_job.{action} event")
print(f"Job labels: {labels}")
# Check if this job requires our self-hosted runners
if "self-hosted" not in labels:
print("Job doesn't require self-hosted runners, ignoring")
return "OK - not for self-hosted runners", 200
# Handle different actions
if action == "queued":
# Job is queued - ensure runners are started
print("Job queued - starting runners if needed")
result = start_runners()
return json.dumps(result), 200, {"Content-Type": "application/json"}
elif action == "in_progress":
# Job started - update activity timestamp on all running instances
print("Job in progress - updating activity timestamps")
now = datetime.now(timezone.utc).isoformat()
for instance in get_runner_instances():
if instance.status == "RUNNING":
try:
update_instance_metadata(instance.name, "last-activity", now)
except Exception as e:
print(f"Error updating {instance.name}: {e}")
return "OK", 200
elif action == "completed":
# Job completed - update activity timestamp
print("Job completed - updating activity timestamps")
now = datetime.now(timezone.utc).isoformat()
for instance in get_runner_instances():
if instance.status == "RUNNING":
try:
update_instance_metadata(instance.name, "last-activity", now)
except Exception as e:
print(f"Error updating {instance.name}: {e}")
return "OK", 200
return "OK", 200
@@ -1,4 +0,0 @@
functions-framework==3.*
google-cloud-compute==1.*
google-cloud-secret-manager==2.*
flask>=2.0.0
-2
View File
@@ -13,7 +13,6 @@ from . import cloud_run
from . import monitoring
from . import iam
from . import networking
from . import runner_manager
__all__ = [
"database",
@@ -25,5 +24,4 @@ __all__ = [
"monitoring",
"iam",
"networking",
"runner_manager",
]
+1 -5
View File
@@ -73,11 +73,7 @@ def create_database() -> dict:
firestore.IndexFieldArgs(field_path="made_for_you", order="ASCENDING"),
firestore.IndexFieldArgs(field_path="created_at", order="DESCENDING"),
],
opts=pulumi.ResourceOptions(
depends_on=[firestore_db],
# Import existing index created before Pulumi managed it
import_="projects/nomadkaraoke/databases/(default)/collectionGroups/jobs/indexes/CICAgOi36pgK",
),
opts=pulumi.ResourceOptions(depends_on=[firestore_db]),
)
# Sessions: query by user_email + is_active, order by created_at
-297
View File
@@ -1,297 +0,0 @@
"""
GitHub Actions Runner Manager - Auto-start and auto-stop self-hosted runners.
Creates infrastructure to automatically manage runner lifecycle:
1. Cloud Function (Gen2) - Handles GitHub webhooks and idle checks
2. Cloud Scheduler - Triggers idle checks every 15 minutes
3. IAM bindings - Permissions to start/stop Compute Engine instances
Architecture:
- GitHub sends workflow_job webhooks to the Cloud Function
- When a job is queued, the function starts any TERMINATED runner VMs
- Cloud Scheduler triggers idle checks every 15 minutes
- If no jobs are pending and runners have been idle, they're stopped
Manual Setup Required:
1. Create a webhook secret:
python -c "import secrets; print(secrets.token_hex(32))" | \
gcloud secrets versions add github-webhook-secret --data-file=-
2. Configure GitHub org webhook at:
https://github.com/organizations/nomadkaraoke/settings/hooks
- Payload URL: <Cloud Function URL from pulumi output>
- Content type: application/json
- Secret: <value from step 1>
- Events: Workflow jobs
"""
import base64
import json
from pathlib import Path
import pulumi
import pulumi_gcp as gcp
from config import (
PROJECT_ID,
REGION,
ZONE,
RunnerManagerConfig,
SecretNames,
)
def create_runner_manager_service_account() -> gcp.serviceaccount.Account:
"""Create service account for the runner manager Cloud Function."""
return gcp.serviceaccount.Account(
"runner-manager-sa",
account_id="runner-manager",
display_name="GitHub Runner Manager",
description="Service account for the GitHub Actions runner manager Cloud Function",
)
def grant_runner_manager_permissions(
service_account: gcp.serviceaccount.Account,
webhook_secret: gcp.secretmanager.Secret,
pat_secret: gcp.secretmanager.Secret,
) -> dict:
"""Grant permissions to the runner manager service account."""
bindings = {}
# Grant Compute Instance Admin - needed to start/stop VMs
bindings["compute_admin"] = gcp.projects.IAMMember(
"runner-manager-compute-admin",
project=PROJECT_ID,
role="roles/compute.instanceAdmin.v1",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Secret Manager accessor for webhook secret
bindings["webhook_secret_accessor"] = gcp.secretmanager.SecretIamMember(
"runner-manager-webhook-secret-access",
secret_id=webhook_secret.id,
role="roles/secretmanager.secretAccessor",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Secret Manager accessor for GitHub PAT (for checking pending jobs)
bindings["pat_secret_accessor"] = gcp.secretmanager.SecretIamMember(
"runner-manager-pat-secret-access",
secret_id=pat_secret.id,
role="roles/secretmanager.secretAccessor",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant logging writer
bindings["logging_writer"] = gcp.projects.IAMMember(
"runner-manager-logging-writer",
project=PROJECT_ID,
role="roles/logging.logWriter",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
return bindings
def create_function_storage_bucket() -> gcp.storage.Bucket:
"""Create a GCS bucket for Cloud Function source code."""
return gcp.storage.Bucket(
"runner-manager-source-bucket",
name=f"{PROJECT_ID}-runner-manager-source",
location=REGION,
uniform_bucket_level_access=True,
lifecycle_rules=[
gcp.storage.BucketLifecycleRuleArgs(
action=gcp.storage.BucketLifecycleRuleActionArgs(type="Delete"),
condition=gcp.storage.BucketLifecycleRuleConditionArgs(age=7),
)
],
)
def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.BucketObject:
"""Create a GCS object with the Cloud Function source code."""
source_dir = Path(__file__).parent.parent / "functions" / "runner_manager"
return gcp.storage.BucketObject(
"runner-manager-source",
bucket=bucket.name,
name="runner-manager-source.zip",
source=pulumi.FileArchive(str(source_dir)),
)
def create_cloud_function(
service_account: gcp.serviceaccount.Account,
bucket: gcp.storage.Bucket,
source_archive: gcp.storage.BucketObject,
permissions: dict,
) -> gcp.cloudfunctionsv2.Function:
"""Create the Cloud Function (Gen2) for runner management."""
return gcp.cloudfunctionsv2.Function(
"runner-manager-function",
name=RunnerManagerConfig.FUNCTION_NAME,
location=REGION,
description="Manages GitHub Actions runner VM lifecycle (auto-start/stop)",
build_config=gcp.cloudfunctionsv2.FunctionBuildConfigArgs(
runtime="python312",
entry_point="handle_request",
source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceArgs(
storage_source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceStorageSourceArgs(
bucket=bucket.name,
object=source_archive.name,
),
),
),
service_config=gcp.cloudfunctionsv2.FunctionServiceConfigArgs(
available_memory=RunnerManagerConfig.FUNCTION_MEMORY,
timeout_seconds=RunnerManagerConfig.FUNCTION_TIMEOUT,
service_account_email=service_account.email,
environment_variables={
"GCP_PROJECT": PROJECT_ID,
"GCP_ZONE": ZONE,
"WEBHOOK_SECRET_NAME": SecretNames.GITHUB_WEBHOOK_SECRET,
"RUNNER_PAT_SECRET_NAME": SecretNames.GITHUB_RUNNER_PAT,
"IDLE_TIMEOUT_HOURS": str(RunnerManagerConfig.IDLE_TIMEOUT_HOURS),
},
min_instance_count=0, # Scale to zero when not in use
max_instance_count=5, # Allow some concurrency for multiple webhooks
),
opts=pulumi.ResourceOptions(
depends_on=list(permissions.values()),
),
)
def allow_unauthenticated_invocation(
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudfunctionsv2.FunctionIamMember:
"""
Allow unauthenticated invocations of the Cloud Function.
Required for GitHub webhooks since GitHub doesn't support GCP auth.
Security is provided by webhook signature verification.
"""
return gcp.cloudfunctionsv2.FunctionIamMember(
"runner-manager-public-invoker",
project=PROJECT_ID,
location=REGION,
cloud_function=function.name,
role="roles/cloudfunctions.invoker",
member="allUsers",
)
def create_scheduler_service_account() -> gcp.serviceaccount.Account:
"""Create service account for Cloud Scheduler to invoke the function."""
return gcp.serviceaccount.Account(
"runner-manager-scheduler-sa",
account_id="runner-manager-scheduler",
display_name="Runner Manager Scheduler",
description="Service account for Cloud Scheduler to trigger runner idle checks",
)
def grant_scheduler_invoker_permission(
scheduler_sa: gcp.serviceaccount.Account,
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudfunctionsv2.FunctionIamMember:
"""Grant the scheduler service account permission to invoke the function."""
return gcp.cloudfunctionsv2.FunctionIamMember(
"runner-manager-scheduler-invoker",
project=PROJECT_ID,
location=REGION,
cloud_function=function.name,
role="roles/cloudfunctions.invoker",
member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"),
)
def create_idle_check_scheduler(
scheduler_sa: gcp.serviceaccount.Account,
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudscheduler.Job:
"""Create Cloud Scheduler job for idle runner checks."""
return gcp.cloudscheduler.Job(
"runner-manager-idle-check",
name="runner-manager-idle-check",
description="Triggers idle check for GitHub Actions runners every 15 minutes",
schedule=RunnerManagerConfig.IDLE_CHECK_SCHEDULE,
time_zone="UTC",
region=REGION,
http_target=gcp.cloudscheduler.JobHttpTargetArgs(
uri=function.url.apply(lambda url: f"{url}?action=check_idle"),
http_method="POST",
headers={
"Content-Type": "application/json",
},
body=base64.b64encode(json.dumps({"trigger": "scheduler"}).encode()).decode(),
oidc_token=gcp.cloudscheduler.JobHttpTargetOidcTokenArgs(
service_account_email=scheduler_sa.email,
audience=function.url,
),
),
retry_config=gcp.cloudscheduler.JobRetryConfigArgs(
retry_count=2,
max_retry_duration="60s",
),
)
def create_runner_manager_resources(
webhook_secret: gcp.secretmanager.Secret,
pat_secret: gcp.secretmanager.Secret,
) -> dict:
"""
Create all resources for the GitHub Actions runner manager.
Args:
webhook_secret: The webhook secret for signature verification.
pat_secret: The GitHub PAT secret for API calls.
Returns:
dict: Dictionary of all created resources.
"""
# Create service accounts
function_sa = create_runner_manager_service_account()
scheduler_sa = create_scheduler_service_account()
# Grant permissions to function service account
permissions = grant_runner_manager_permissions(
function_sa,
webhook_secret,
pat_secret,
)
# Create source storage bucket and archive
bucket = create_function_storage_bucket()
source_archive = create_function_source_archive(bucket)
# Create the Cloud Function
function = create_cloud_function(
function_sa,
bucket,
source_archive,
permissions,
)
# Allow unauthenticated invocation (GitHub webhooks)
public_access = allow_unauthenticated_invocation(function)
# Grant scheduler permission to invoke function
scheduler_invoker = grant_scheduler_invoker_permission(scheduler_sa, function)
# Create the idle check scheduler job
scheduler_job = create_idle_check_scheduler(scheduler_sa, function)
return {
"function_service_account": function_sa,
"scheduler_service_account": scheduler_sa,
"permissions": permissions,
"source_bucket": bucket,
"source_archive": source_archive,
"function": function,
"public_access": public_access,
"scheduler_invoker": scheduler_invoker,
"scheduler_job": scheduler_job,
}
-1
View File
@@ -47,7 +47,6 @@ def create_secrets() -> dict[str, secretmanager.Secret]:
# GitHub
"github-runner-pat",
"github-webhook-secret", # For runner manager webhook verification
# E2E Testing
"e2e-bypass-key",
@@ -219,8 +219,7 @@ class VideoGenerator:
# Create a temporary copy of the ASS file with a unique filename
import time
# Only allow ASCII alphanumeric characters to prevent FFmpeg path issues with Unicode
safe_prefix = "".join(c if c.isascii() and c.isalnum() else "_" for c in output_prefix)
safe_prefix = "".join(c if c.isalnum() else "_" for c in output_prefix)
timestamp = int(time.time() * 1000)
temp_ass_path = os.path.join(self.cache_dir, f"temp_subtitles_{safe_prefix}_{timestamp}.ass")
import shutil
@@ -280,8 +279,7 @@ class VideoGenerator:
import time
import shutil
# Only allow ASCII alphanumeric characters to prevent FFmpeg path issues with Unicode
safe_prefix = "".join(c if c.isascii() and c.isalnum() else "_" for c in output_prefix)
safe_prefix = "".join(c if c.isalnum() else "_" for c in output_prefix)
timestamp = int(time.time() * 1000)
temp_ass_path = os.path.join(self.cache_dir, f"temp_preview_subtitles_{safe_prefix}_{timestamp}.ass")
+4 -8
View File
@@ -1965,14 +1965,10 @@ class JobMonitor:
folder_name = f"{brand_code} - {artist} - {title}"
else:
folder_name = f"{artist} - {title}"
# Sanitize folder name - only allow ASCII alphanumeric to prevent filesystem issues
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
# Guard against empty or separator-only folder name (e.g., if title/artist were entirely non-ASCII)
if not folder_name or not any(c.isascii() and c.isalnum() for c in folder_name):
folder_name = f"job_{job_id}"
# Sanitize folder name
folder_name = "".join(c for c in folder_name if c.isalnum() or c in " -_").strip()
output_dir = Path(self.config.output_dir) / folder_name
output_dir.mkdir(parents=True, exist_ok=True)
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.115.2"
version = "0.114.15"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
@@ -1,133 +0,0 @@
"""Unit tests for remote_cli Unicode folder name sanitization.
Tests the fix for job 1eab1172 where Cyrillic characters in folder names
could cause filesystem issues.
"""
import pytest
class TestRemoteCliFolderNameSanitization:
"""Test folder name sanitization logic for Unicode characters."""
@pytest.mark.parametrize("artist,title,expected_has,expected_not_has", [
("zivert", "я куплю тебе дом", ["zivert"], ["я", "куплю", "тебе", "дом", " - ", "_"]), # Cyrillic removed, only artist remains
("artist", "中文标题", ["artist"], ["中", "文", "标", "题", " - ", "_"]), # Chinese removed
("artist", "العربية", ["artist"], ["ا", "ل", "ع", "ر", "ب", " - ", "_"]), # Arabic removed
("artist", "título", ["artist", " - ", "tulo"], ["í"]), # Accented (í removed, t+ulo remain)
("Artist", "Normal Title", ["Artist", " - ", "Normal", "Title"], []), # ASCII preserved
("Test", "Song (2024)", ["Test", " - ", "Song", "2024"], ["(", ")"]), # Parens removed
])
def test_folder_name_sanitization_logic(self, artist, title, expected_has, expected_not_has):
"""Test the folder name sanitization logic directly."""
# This is the exact logic from remote_cli.py:1970
folder_name = f"{artist} - {title}"
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
# Verify folder name is ASCII only
assert folder_name.isascii(), f"Folder name should be ASCII-only: {folder_name}"
# Verify expected parts are present
for expected in expected_has:
assert expected in folder_name, \
f"Expected '{expected}' to be in folder name: {folder_name}"
# Verify unwanted parts are removed
for not_expected in expected_not_has:
assert not_expected not in folder_name, \
f"Did not expect '{not_expected}' in folder name: {folder_name}"
def test_folder_name_brand_code_logic(self):
"""Test folder name with brand code prefix."""
brand_code = "BRAND-001"
artist = "Artist"
title = "Title"
folder_name = f"{brand_code} - {artist} - {title}"
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
assert folder_name.startswith("BRAND-001 - ")
assert "Artist" in folder_name
assert "Title" in folder_name
def test_folder_name_strips_whitespace(self):
"""Test that leading/trailing whitespace is stripped."""
artist = " Artist "
title = " Title "
folder_name = f"{artist} - {title}"
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
assert not folder_name.startswith(" ")
assert not folder_name.endswith(" ")
assert "Artist" in folder_name
assert "Title" in folder_name
def test_cyrillic_real_world_case_job_1eab1172(self):
"""Test the exact case from failed job 1eab1172 - Russian title."""
artist = "zivert"
title = "я куплю тебе дом" # Russian: "I will buy you a house"
folder_name = f"{artist} - {title}"
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
# Should be ASCII-only
assert folder_name.isascii()
# Should contain the artist name
assert "zivert" in folder_name
# Cyrillic characters should be removed
assert "я" not in folder_name
assert "куплю" not in folder_name
assert "тебе" not in folder_name
assert "дом" not in folder_name
def test_emoji_and_special_unicode(self):
"""Test that emojis and special Unicode characters are removed."""
artist = "Artist 😀"
title = "Song 🎵"
folder_name = f"{artist} - {title}"
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
assert folder_name.isascii()
assert "😀" not in folder_name
assert "🎵" not in folder_name
assert "Artist" in folder_name
assert "Song" in folder_name
def test_empty_folder_name_fallback(self):
"""Test that if sanitization results in empty or separator-only string, fallback to job_id."""
job_id = "abc123xyz"
# Test 1: Emoji with separator results in "-" after sanitization
artist = "😀😀😀"
title = "🎵🎵🎵"
folder_name = f"{artist} - {title}"
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
# After sanitization, only "-" remains (spaces/hyphens allowed but emoji removed)
assert folder_name == "-"
# Apply the fallback logic (updated to check for alphanumeric content)
if not folder_name or not any(c.isascii() and c.isalnum() for c in folder_name):
folder_name = f"job_{job_id}"
# Verify fallback is used for separator-only result
assert folder_name == "job_abc123xyz"
# Test 2: Truly empty result (all characters removed, no separator)
artist2 = "😀"
title2 = "🎵"
folder_name2 = f"{artist2}{title2}" # No separator, just emoji
folder_name2 = "".join(c for c in folder_name2 if (c.isascii() and c.isalnum()) or c in " -_").strip()
# Should be empty
assert folder_name2 == ""
# Apply fallback
if not folder_name2 or not any(c.isascii() and c.isalnum() for c in folder_name2):
folder_name2 = f"job_{job_id}"
assert folder_name2 == "job_abc123xyz"
-283
View File
@@ -1,283 +0,0 @@
"""Unit tests for VideoGenerator Unicode filename handling.
Tests the fix for job 1eab1172 where Cyrillic characters in temp subtitle
filenames caused FFmpeg to fail with exit code 183.
"""
import os
import tempfile
import pytest
from unittest.mock import MagicMock, patch
from karaoke_gen.lyrics_transcriber.output.video import VideoGenerator
class TestVideoGeneratorUnicodeSanitization:
"""Test that VideoGenerator properly sanitizes Unicode characters in temp filenames."""
@pytest.fixture
def video_generator(self):
"""Create a VideoGenerator instance for testing."""
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = os.path.join(temp_dir, "output")
cache_dir = os.path.join(temp_dir, "cache")
os.makedirs(output_dir)
os.makedirs(cache_dir)
styles = {
"karaoke": {
"background_color": "black",
"font_path": None,
}
}
generator = VideoGenerator(
output_dir=output_dir,
cache_dir=cache_dir,
video_resolution=(1920, 1080),
styles=styles,
logger=MagicMock(),
)
yield generator
def test_generate_video_sanitizes_cyrillic_characters(self, video_generator):
"""Test that Cyrillic characters are replaced with underscores in temp filename.
This was the bug in job 1eab1172 with Russian title "я куплю тебе дом".
"""
output_prefix = "zivert - я куплю тебе дом"
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
# This should not raise an exception
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Verify FFmpeg was called
assert mock_ffmpeg.called
# Get the temp file path that was created
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
assert ass_filter_arg is not None, "Should have ASS filter in FFmpeg command"
# Extract the temp ASS path from the filter
# Format: ass=/path/to/temp_subtitles_...:fontsdir=...
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
# Verify the temp filename only contains ASCII characters
temp_filename = os.path.basename(temp_ass_path)
assert temp_filename.isascii(), f"Temp filename should be ASCII-only: {temp_filename}"
# Verify Cyrillic characters were replaced with underscores
assert "я" not in temp_filename
assert "куплю" not in temp_filename
assert "дом" not in temp_filename
assert "_" in temp_filename # Should have underscores as replacements
finally:
# Cleanup
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
def test_generate_preview_video_sanitizes_unicode(self, video_generator):
"""Test that preview video generation also sanitizes Unicode characters."""
output_prefix = "artist - 中文标题" # Chinese characters
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
# Mock the LocalPreviewEncodingService
with patch('backend.services.local_preview_encoding_service.LocalPreviewEncodingService') as mock_service_class:
mock_service = MagicMock()
mock_result = MagicMock()
mock_result.success = True
mock_service.encode_preview.return_value = mock_result
mock_service_class.return_value = mock_service
# This should not raise an exception
video_generator.generate_preview_video(ass_path, audio_path, output_prefix)
# Verify the service was called
assert mock_service.encode_preview.called
# Get the config that was passed
config = mock_service.encode_preview.call_args[0][0]
temp_ass_path = config.ass_path
# Verify the temp filename only contains ASCII characters
temp_filename = os.path.basename(temp_ass_path)
assert temp_filename.isascii(), f"Temp filename should be ASCII-only: {temp_filename}"
# Verify Chinese characters were replaced
assert "中文标题" not in temp_filename
finally:
# Cleanup
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
@pytest.mark.parametrize("output_prefix,unicode_chars,should_contain_ascii", [
("artist - я куплю тебе дом", ["я", "к", "у", "п", "л", "ю", "т", "е", "б", "д", "о", "м"], "artist"), # Cyrillic
("artist - 中文标题", ["中", "文", "标", "题"], "artist"), # Chinese
("artist - العربية", ["ا", "ل", "ع", "ر", "ب", "ي", "ة"], "artist"), # Arabic
("artist - 😀🎵", ["😀", "🎵"], "artist"), # Emoji
("artist - título", ["í"], "tulo"), # Spanish with accent - 't' and 'ulo' should remain
("normal title", [], "normal_title"), # ASCII-only (control)
])
def test_various_unicode_characters_sanitized(self, video_generator, output_prefix, unicode_chars, should_contain_ascii):
"""Test that various Unicode character sets are properly sanitized."""
import re
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Get the temp file path
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
temp_filename = os.path.basename(temp_ass_path)
# Verify filename only contains ASCII
assert temp_filename.isascii(), f"Should be ASCII-only: {temp_filename}"
# Verify all Unicode characters were removed
for unicode_char in unicode_chars:
assert unicode_char not in temp_filename, \
f"Unicode character '{unicode_char}' should not be in filename: {temp_filename}"
# Verify ASCII parts were preserved
if should_contain_ascii:
assert should_contain_ascii in temp_filename, \
f"Expected '{should_contain_ascii}' to be in filename: {temp_filename}"
# Verify it matches the general pattern (starts with temp_subtitles_, ends with .ass)
assert temp_filename.startswith("temp_subtitles_"), \
f"Filename should start with 'temp_subtitles_': {temp_filename}"
assert temp_filename.endswith(".ass"), \
f"Filename should end with '.ass': {temp_filename}"
finally:
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
def test_sanitization_preserves_ascii_alphanumeric(self, video_generator):
"""Test that ASCII alphanumeric characters are preserved."""
output_prefix = "ABC123xyz"
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Get the temp file path
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
temp_filename = os.path.basename(temp_ass_path)
# Verify ASCII alphanumeric characters are preserved
assert "ABC123xyz" in temp_filename
finally:
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
def test_special_characters_replaced_with_underscores(self, video_generator):
"""Test that special characters (spaces, hyphens, etc.) are replaced."""
output_prefix = "artist - title (version) [2024]"
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Get the temp file path
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
temp_filename = os.path.basename(temp_ass_path)
# Verify special characters were replaced
assert " " not in temp_filename
assert "-" not in temp_filename
assert "(" not in temp_filename
assert ")" not in temp_filename
assert "[" not in temp_filename
assert "]" not in temp_filename
# But alphanumeric should remain
assert "artist" in temp_filename
assert "title" in temp_filename
assert "version" in temp_filename
assert "2024" in temp_filename
finally:
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)