mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-21 05:18:08 +09:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
275fc909f6 |
No files matched your search
+17
-710
@@ -20,7 +20,6 @@ from backend.services.user_service import get_user_service, UserService, USERS_C
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.flacfetch_client import get_flacfetch_client, FlacfetchServiceError
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.audio_search_service import get_audio_search_service, NoResultsError, AudioSearchError
|
||||
from backend.models.job import JobStatus
|
||||
from backend.utils.test_data import is_test_email
|
||||
from karaoke_gen.utils import sanitize_filename
|
||||
@@ -1013,8 +1012,8 @@ class TriggerWorkerResponse(BaseModel):
|
||||
ALLOWED_RESET_STATES = {
|
||||
"pending",
|
||||
"awaiting_audio_selection",
|
||||
"awaiting_review", # Combined review (lyrics + instrumental selection)
|
||||
# Note: awaiting_instrumental_selection removed - deprecated by combined review flow (Jan 2026)
|
||||
"awaiting_review",
|
||||
"awaiting_instrumental_selection",
|
||||
"instrumental_selected", # Re-run video generation with same settings
|
||||
}
|
||||
|
||||
@@ -1035,9 +1034,6 @@ STATE_DATA_CLEAR_KEYS = {
|
||||
"render_progress",
|
||||
"screens_progress",
|
||||
"encoding_progress",
|
||||
# Parallel processing flags - must clear to allow workers to run again
|
||||
"audio_complete",
|
||||
"lyrics_complete",
|
||||
],
|
||||
"awaiting_audio_selection": [
|
||||
"audio_selection",
|
||||
@@ -1050,7 +1046,6 @@ STATE_DATA_CLEAR_KEYS = {
|
||||
"encoding_progress",
|
||||
],
|
||||
"awaiting_review": [
|
||||
# Combined review - user reviews lyrics AND selects instrumental in one step
|
||||
"review_complete",
|
||||
"corrected_lyrics",
|
||||
"instrumental_selection",
|
||||
@@ -1059,7 +1054,13 @@ STATE_DATA_CLEAR_KEYS = {
|
||||
"screens_progress",
|
||||
"encoding_progress",
|
||||
],
|
||||
# Note: awaiting_instrumental_selection removed - deprecated by combined review flow (Jan 2026)
|
||||
"awaiting_instrumental_selection": [
|
||||
"instrumental_selection",
|
||||
"video_progress",
|
||||
"render_progress",
|
||||
"screens_progress",
|
||||
"encoding_progress",
|
||||
],
|
||||
"instrumental_selected": [
|
||||
# Clear video/encoding/distribution state to allow re-processing
|
||||
# Preserves: instrumental_selection, lyrics_metadata, review data
|
||||
@@ -1147,8 +1148,14 @@ async def reset_job(
|
||||
}
|
||||
|
||||
# Perform the update with state_data clearing
|
||||
# Note: update_job() raises exceptions on failure, returns None on success
|
||||
job_manager.update_job(job_id, updates)
|
||||
# We need to set the cleared keys to DELETE_FIELD
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to reset job. Please try again."
|
||||
)
|
||||
|
||||
# Clear the state data keys and error state using direct Firestore update
|
||||
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
|
||||
@@ -1856,703 +1863,3 @@ async def impersonate_user(
|
||||
user_email=target_email,
|
||||
message=f"Now impersonating {target_email}",
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Regenerate Screens Endpoint
|
||||
# =============================================================================
|
||||
|
||||
class RegenerateScreensResponse(BaseModel):
|
||||
"""Response from regenerate screens endpoint."""
|
||||
status: str
|
||||
job_id: str
|
||||
message: str
|
||||
previous_screens_deleted: bool
|
||||
worker_triggered: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
# States where regenerate screens is allowed
|
||||
REGENERATE_SCREENS_ALLOWED_STATES = {
|
||||
"complete",
|
||||
"failed",
|
||||
"awaiting_review",
|
||||
"awaiting_instrumental_selection",
|
||||
"instrumental_selected",
|
||||
"prep_complete",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/regenerate-screens", response_model=RegenerateScreensResponse)
|
||||
async def regenerate_screens(
|
||||
job_id: str,
|
||||
auth_data: AuthResult = Depends(require_admin),
|
||||
):
|
||||
"""
|
||||
Regenerate title and end screens with current artist/title metadata (admin only).
|
||||
|
||||
Use this when you've edited the artist or title fields and need the
|
||||
title/end screens to reflect the new metadata.
|
||||
|
||||
This endpoint:
|
||||
1. Validates the job is in an appropriate state
|
||||
2. Deletes existing screen files from GCS
|
||||
3. Triggers the screens worker to regenerate with current metadata
|
||||
4. Returns immediately (does not wait for completion)
|
||||
|
||||
Monitor progress via the job detail page or logs.
|
||||
|
||||
Allowed states: complete, failed, awaiting_review, awaiting_instrumental_selection,
|
||||
instrumental_selected, prep_complete
|
||||
"""
|
||||
admin_email = auth_data.user_email or "unknown"
|
||||
job_manager = JobManager()
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
|
||||
# Validate job state
|
||||
if job.status not in REGENERATE_SCREENS_ALLOWED_STATES:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Cannot regenerate screens for job in '{job.status}' state. "
|
||||
f"Allowed states: {', '.join(sorted(REGENERATE_SCREENS_ALLOWED_STATES))}"
|
||||
)
|
||||
|
||||
# Validate job has artist and title
|
||||
if not job.artist or not job.title:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Job must have artist and title set to regenerate screens"
|
||||
)
|
||||
|
||||
# Check that audio and lyrics processing completed at some point
|
||||
state_data = job.state_data or {}
|
||||
audio_progress = state_data.get('audio_progress', {})
|
||||
lyrics_progress = state_data.get('lyrics_progress', {})
|
||||
|
||||
if audio_progress.get('stage') != 'audio_complete':
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Audio processing must be complete before regenerating screens"
|
||||
)
|
||||
|
||||
if lyrics_progress.get('stage') != 'lyrics_complete':
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Lyrics processing must be complete before regenerating screens"
|
||||
)
|
||||
|
||||
# Delete existing screen files from GCS
|
||||
storage = StorageService()
|
||||
screens_deleted = False
|
||||
try:
|
||||
# Delete screen video and image files
|
||||
screen_paths = [
|
||||
f"jobs/{job_id}/screens/title.mov",
|
||||
f"jobs/{job_id}/screens/title.jpg",
|
||||
f"jobs/{job_id}/screens/title.png",
|
||||
f"jobs/{job_id}/screens/end.mov",
|
||||
f"jobs/{job_id}/screens/end.jpg",
|
||||
f"jobs/{job_id}/screens/end.png",
|
||||
]
|
||||
for path in screen_paths:
|
||||
try:
|
||||
storage.delete_file(path)
|
||||
except Exception:
|
||||
pass # File may not exist
|
||||
screens_deleted = True
|
||||
logger.info(f"Admin {admin_email} deleted existing screens for job {job_id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error deleting screens for job {job_id}: {e}")
|
||||
|
||||
# Clear screens_progress to allow worker to run
|
||||
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
|
||||
db = job_manager.firestore.db
|
||||
job_ref = db.collection("jobs").document(job_id)
|
||||
|
||||
# Update state to allow screens worker to run
|
||||
job_ref.update({
|
||||
"state_data.screens_progress": DELETE_FIELD,
|
||||
"state_data.audio_complete": True,
|
||||
"state_data.lyrics_complete": True,
|
||||
"timeline": ArrayUnion([{
|
||||
"status": job.status,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} triggered screen regeneration"
|
||||
}])
|
||||
})
|
||||
|
||||
# Trigger screens worker
|
||||
worker_triggered = False
|
||||
error_msg = None
|
||||
try:
|
||||
from backend.services.worker_service import get_worker_service
|
||||
worker_service = get_worker_service()
|
||||
worker_triggered = await worker_service.trigger_screens_worker(job_id)
|
||||
if worker_triggered:
|
||||
logger.info(f"Admin {admin_email} triggered screens regeneration for job {job_id}")
|
||||
else:
|
||||
error_msg = "Worker service returned False - check service logs"
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Error triggering screens worker for job {job_id}: {e}", exc_info=True)
|
||||
|
||||
return RegenerateScreensResponse(
|
||||
status="success" if worker_triggered else "error",
|
||||
job_id=job_id,
|
||||
message="Screens regeneration started" if worker_triggered else "Failed to trigger screens worker",
|
||||
previous_screens_deleted=screens_deleted,
|
||||
worker_triggered=worker_triggered,
|
||||
error=error_msg,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Restart Job Endpoint
|
||||
# =============================================================================
|
||||
|
||||
class RestartJobRequest(BaseModel):
|
||||
"""Request model for restarting a job."""
|
||||
preserve_audio_stems: bool = False # If true, keep existing stems (faster)
|
||||
delete_outputs: bool = True # Delete existing output files from GCS
|
||||
|
||||
|
||||
class RestartJobResponse(BaseModel):
|
||||
"""Response from restart job endpoint."""
|
||||
status: str
|
||||
job_id: str
|
||||
message: str
|
||||
previous_status: str
|
||||
new_status: str
|
||||
cleared_data: List[str]
|
||||
deleted_gcs_paths: List[str]
|
||||
workers_triggered: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
# States where restart is allowed
|
||||
RESTART_ALLOWED_STATES = {
|
||||
"pending",
|
||||
"complete",
|
||||
"failed",
|
||||
"awaiting_review",
|
||||
"awaiting_audio_selection",
|
||||
"awaiting_instrumental_selection",
|
||||
"instrumental_selected",
|
||||
"prep_complete",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/restart", response_model=RestartJobResponse)
|
||||
async def restart_job(
|
||||
job_id: str,
|
||||
request: RestartJobRequest,
|
||||
auth_data: AuthResult = Depends(require_admin),
|
||||
):
|
||||
"""
|
||||
Fully restart a job from the beginning (admin only).
|
||||
|
||||
Unlike reset (which just changes state), restart actually triggers
|
||||
the appropriate workers to begin processing again.
|
||||
|
||||
Options:
|
||||
- preserve_audio_stems: If True, keeps existing audio separation (faster restart).
|
||||
Useful when you just need to regenerate screens with updated metadata.
|
||||
- delete_outputs: If True (default), deletes existing output files from GCS.
|
||||
|
||||
When preserve_audio_stems=True:
|
||||
- Keeps stems, lyrics, and transcription data
|
||||
- Clears screens, video, encoding state
|
||||
- Triggers screens worker immediately
|
||||
|
||||
When preserve_audio_stems=False:
|
||||
- Clears all processing state
|
||||
- For YouTube URL jobs: triggers download then audio/lyrics workers
|
||||
- For audio search jobs: transitions to awaiting_audio_selection
|
||||
- For file upload jobs: triggers audio/lyrics workers
|
||||
|
||||
Allowed states: pending, complete, failed, awaiting_review, awaiting_audio_selection,
|
||||
awaiting_instrumental_selection, instrumental_selected, prep_complete
|
||||
"""
|
||||
admin_email = auth_data.user_email or "unknown"
|
||||
job_manager = JobManager()
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
|
||||
# Validate job state
|
||||
if job.status not in RESTART_ALLOWED_STATES:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Cannot restart job in '{job.status}' state. "
|
||||
f"Allowed states: {', '.join(sorted(RESTART_ALLOWED_STATES))}"
|
||||
)
|
||||
|
||||
previous_status = job.status
|
||||
state_data = job.state_data or {}
|
||||
cleared_keys = []
|
||||
deleted_paths = []
|
||||
workers_triggered = []
|
||||
error_msg = None
|
||||
|
||||
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
|
||||
db = job_manager.firestore.db
|
||||
job_ref = db.collection("jobs").document(job_id)
|
||||
|
||||
# Delete output files if requested
|
||||
if request.delete_outputs:
|
||||
storage = StorageService()
|
||||
# Delete screens
|
||||
for screen_file in ["title.mov", "title.jpg", "title.png", "end.mov", "end.jpg", "end.png"]:
|
||||
try:
|
||||
path = f"jobs/{job_id}/screens/{screen_file}"
|
||||
storage.delete_file(path)
|
||||
deleted_paths.append(path)
|
||||
except Exception:
|
||||
pass
|
||||
# Delete final outputs
|
||||
for output_file in ["final_4k.mp4", "final_720p.mp4", "with_vocals.mkv", "preview.mp4"]:
|
||||
try:
|
||||
path = f"jobs/{job_id}/output/{output_file}"
|
||||
storage.delete_file(path)
|
||||
deleted_paths.append(path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if request.preserve_audio_stems:
|
||||
# Quick restart: keep stems/lyrics, just regenerate screens and video
|
||||
# Validate that audio and lyrics completed
|
||||
audio_progress = state_data.get('audio_progress', {})
|
||||
lyrics_progress = state_data.get('lyrics_progress', {})
|
||||
|
||||
if audio_progress.get('stage') != 'audio_complete':
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot preserve audio stems - audio processing was not complete"
|
||||
)
|
||||
if lyrics_progress.get('stage') != 'lyrics_complete':
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot preserve audio stems - lyrics processing was not complete"
|
||||
)
|
||||
|
||||
# Clear only screens/video/encoding state
|
||||
keys_to_clear = [
|
||||
"screens_progress",
|
||||
"video_progress",
|
||||
"render_progress",
|
||||
"encoding_progress",
|
||||
"review_complete",
|
||||
"instrumental_selection",
|
||||
]
|
||||
|
||||
update_payload = {}
|
||||
for key in keys_to_clear:
|
||||
if key in state_data:
|
||||
update_payload[f"state_data.{key}"] = DELETE_FIELD
|
||||
cleared_keys.append(key)
|
||||
|
||||
# Set coordination flags to trigger screens worker
|
||||
update_payload["state_data.audio_complete"] = True
|
||||
update_payload["state_data.lyrics_complete"] = True
|
||||
update_payload["status"] = "transcribing" # Allows screens worker to run
|
||||
update_payload["message"] = "Restarting with preserved audio stems"
|
||||
update_payload["error_message"] = DELETE_FIELD
|
||||
update_payload["error_details"] = DELETE_FIELD
|
||||
update_payload["timeline"] = ArrayUnion([{
|
||||
"status": "transcribing",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} restarted job (preserving audio stems)"
|
||||
}])
|
||||
|
||||
job_ref.update(update_payload)
|
||||
new_status = "transcribing"
|
||||
|
||||
# Trigger screens worker
|
||||
try:
|
||||
from backend.services.worker_service import get_worker_service
|
||||
worker_service = get_worker_service()
|
||||
if await worker_service.trigger_screens_worker(job_id):
|
||||
workers_triggered.append("screens")
|
||||
logger.info(f"Admin {admin_email} triggered screens worker for restart of job {job_id}")
|
||||
else:
|
||||
error_msg = "Failed to trigger screens worker"
|
||||
except Exception as e:
|
||||
error_msg = f"Error triggering screens worker: {e}"
|
||||
logger.error(f"Error triggering screens worker for job {job_id}: {e}", exc_info=True)
|
||||
|
||||
else:
|
||||
# Full restart: clear everything and start from beginning
|
||||
keys_to_clear = [
|
||||
"audio_complete",
|
||||
"lyrics_complete",
|
||||
"audio_progress",
|
||||
"lyrics_progress",
|
||||
"screens_progress",
|
||||
"video_progress",
|
||||
"render_progress",
|
||||
"encoding_progress",
|
||||
"review_complete",
|
||||
"corrected_lyrics",
|
||||
"instrumental_selection",
|
||||
"instrumental_options",
|
||||
"lyrics_metadata",
|
||||
]
|
||||
|
||||
update_payload = {}
|
||||
for key in keys_to_clear:
|
||||
if key in state_data:
|
||||
update_payload[f"state_data.{key}"] = DELETE_FIELD
|
||||
cleared_keys.append(key)
|
||||
|
||||
update_payload["error_message"] = DELETE_FIELD
|
||||
update_payload["error_details"] = DELETE_FIELD
|
||||
|
||||
# Determine what workers to trigger based on job input type
|
||||
has_youtube_url = bool(job.url)
|
||||
has_uploaded_file = bool(job.input_media_gcs_path)
|
||||
has_audio_search = bool(state_data.get('audio_search_results') or state_data.get('audio_selection'))
|
||||
|
||||
if has_youtube_url and not has_audio_search:
|
||||
# YouTube URL job - need to re-download and process
|
||||
# Clear the downloaded file to force re-download
|
||||
update_payload["input_media_gcs_path"] = DELETE_FIELD
|
||||
cleared_keys.append("input_media_gcs_path")
|
||||
update_payload["status"] = "downloading"
|
||||
update_payload["message"] = "Re-downloading audio from YouTube"
|
||||
update_payload["timeline"] = ArrayUnion([{
|
||||
"status": "downloading",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} restarted job - downloading from YouTube"
|
||||
}])
|
||||
job_ref.update(update_payload)
|
||||
new_status = "downloading"
|
||||
|
||||
# Download YouTube audio using YouTubeDownloadService, then trigger processing
|
||||
try:
|
||||
from backend.services.youtube_download_service import (
|
||||
get_youtube_download_service,
|
||||
YouTubeDownloadError,
|
||||
)
|
||||
from backend.services.worker_service import get_worker_service
|
||||
|
||||
youtube_service = get_youtube_download_service()
|
||||
audio_gcs_path = await youtube_service.download(
|
||||
url=job.url,
|
||||
job_id=job_id,
|
||||
artist=job.artist,
|
||||
title=job.title,
|
||||
)
|
||||
|
||||
# Update job with downloaded audio path
|
||||
job_manager.update_job(job_id, {'input_media_gcs_path': audio_gcs_path})
|
||||
workers_triggered.append("download")
|
||||
logger.info(f"YouTube audio re-downloaded to GCS: {audio_gcs_path}")
|
||||
|
||||
# Now trigger audio and lyrics workers
|
||||
worker_service = get_worker_service()
|
||||
if await worker_service.trigger_audio_worker(job_id):
|
||||
workers_triggered.append("audio")
|
||||
if await worker_service.trigger_lyrics_worker(job_id):
|
||||
workers_triggered.append("lyrics")
|
||||
|
||||
if "audio" not in workers_triggered and "lyrics" not in workers_triggered:
|
||||
error_msg = "Audio downloaded but failed to trigger processing workers"
|
||||
except YouTubeDownloadError as e:
|
||||
error_msg = f"YouTube download failed: {e}"
|
||||
logger.error(f"YouTube download failed for job {job_id}: {e}")
|
||||
# Update job to failed state
|
||||
job_manager.mark_job_failed(job_id, f"YouTube download failed: {e}")
|
||||
new_status = "failed"
|
||||
except Exception as e:
|
||||
error_msg = f"Error during YouTube restart: {e}"
|
||||
logger.error(f"Error restarting YouTube job {job_id}: {e}", exc_info=True)
|
||||
|
||||
elif has_audio_search:
|
||||
# Audio search job - go back to audio selection
|
||||
# Clear audio search results to allow re-search
|
||||
for key in ["audio_search_results", "audio_search_count", "remote_search_id", "audio_selection"]:
|
||||
if key in state_data:
|
||||
update_payload[f"state_data.{key}"] = DELETE_FIELD
|
||||
cleared_keys.append(key)
|
||||
|
||||
update_payload["input_media_gcs_path"] = DELETE_FIELD
|
||||
update_payload["status"] = "awaiting_audio_selection"
|
||||
update_payload["message"] = "Ready for audio source selection"
|
||||
update_payload["timeline"] = ArrayUnion([{
|
||||
"status": "awaiting_audio_selection",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} restarted job - select audio source"
|
||||
}])
|
||||
job_ref.update(update_payload)
|
||||
new_status = "awaiting_audio_selection"
|
||||
# No workers to trigger - waiting for admin to select audio
|
||||
|
||||
elif has_uploaded_file:
|
||||
# File upload job - just trigger audio/lyrics workers
|
||||
update_payload["status"] = "downloading"
|
||||
update_payload["message"] = "Restarting audio and lyrics processing"
|
||||
update_payload["timeline"] = ArrayUnion([{
|
||||
"status": "downloading",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} restarted job from beginning"
|
||||
}])
|
||||
job_ref.update(update_payload)
|
||||
new_status = "downloading"
|
||||
|
||||
try:
|
||||
from backend.services.worker_service import get_worker_service
|
||||
worker_service = get_worker_service()
|
||||
if await worker_service.trigger_audio_worker(job_id):
|
||||
workers_triggered.append("audio")
|
||||
if await worker_service.trigger_lyrics_worker(job_id):
|
||||
workers_triggered.append("lyrics")
|
||||
if not workers_triggered:
|
||||
error_msg = "Failed to trigger audio/lyrics workers"
|
||||
except Exception as e:
|
||||
error_msg = f"Error triggering workers: {e}"
|
||||
logger.error(f"Error triggering workers for job {job_id}: {e}", exc_info=True)
|
||||
else:
|
||||
# Unknown job type - just set to pending
|
||||
update_payload["status"] = "pending"
|
||||
update_payload["message"] = "Job reset - manual intervention may be needed"
|
||||
update_payload["timeline"] = ArrayUnion([{
|
||||
"status": "pending",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} restarted job (no automatic worker trigger)"
|
||||
}])
|
||||
job_ref.update(update_payload)
|
||||
new_status = "pending"
|
||||
error_msg = "Job type unclear - may need manual worker trigger"
|
||||
|
||||
logger.info(
|
||||
f"Admin {admin_email} restarted job {job_id}: "
|
||||
f"{previous_status} -> {new_status}, cleared: {cleared_keys}, workers: {workers_triggered}"
|
||||
)
|
||||
|
||||
return RestartJobResponse(
|
||||
status="success" if workers_triggered or new_status == "awaiting_audio_selection" else "partial",
|
||||
job_id=job_id,
|
||||
message=f"Job restarted from {previous_status} to {new_status}",
|
||||
previous_status=previous_status,
|
||||
new_status=new_status,
|
||||
cleared_data=cleared_keys,
|
||||
deleted_gcs_paths=deleted_paths,
|
||||
workers_triggered=workers_triggered,
|
||||
error=error_msg,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Override Audio Source Endpoint
|
||||
# =============================================================================
|
||||
|
||||
class OverrideAudioSourceRequest(BaseModel):
|
||||
"""Request model for overriding audio source."""
|
||||
source_type: str # "audio_search" - switch to audio search mode
|
||||
|
||||
|
||||
class OverrideAudioSourceResponse(BaseModel):
|
||||
"""Response from override audio source endpoint."""
|
||||
status: str
|
||||
job_id: str
|
||||
message: str
|
||||
previous_source: str # "youtube", "audio_search", "file_upload"
|
||||
new_source: str
|
||||
cleared_data: List[str]
|
||||
new_status: str
|
||||
search_results_count: Optional[int] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
# States where audio source override is allowed
|
||||
OVERRIDE_AUDIO_ALLOWED_STATES = {
|
||||
"pending",
|
||||
"complete",
|
||||
"failed",
|
||||
"awaiting_review",
|
||||
"awaiting_audio_selection",
|
||||
"instrumental_selected",
|
||||
"prep_complete",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/override-audio-source", response_model=OverrideAudioSourceResponse)
|
||||
async def override_audio_source(
|
||||
job_id: str,
|
||||
request: OverrideAudioSourceRequest,
|
||||
auth_data: AuthResult = Depends(require_admin),
|
||||
):
|
||||
"""
|
||||
Override the audio source for a job (admin only).
|
||||
|
||||
Use this when a Made-For-You order was submitted with a YouTube URL
|
||||
but you want to use audio search to find higher quality audio instead.
|
||||
|
||||
Currently supports switching to audio_search mode, which:
|
||||
1. Clears existing audio-related state
|
||||
2. Performs an audio search using the job's artist/title
|
||||
3. Stores search results in state_data
|
||||
4. Transitions job to awaiting_audio_selection with results ready
|
||||
5. Admin can then select an audio source from the search results
|
||||
|
||||
Allowed states: pending, complete, failed, awaiting_review, awaiting_audio_selection,
|
||||
instrumental_selected, prep_complete
|
||||
"""
|
||||
admin_email = auth_data.user_email or "unknown"
|
||||
job_manager = JobManager()
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
|
||||
# Validate job state
|
||||
if job.status not in OVERRIDE_AUDIO_ALLOWED_STATES:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Cannot override audio source for job in '{job.status}' state. "
|
||||
f"Allowed states: {', '.join(sorted(OVERRIDE_AUDIO_ALLOWED_STATES))}"
|
||||
)
|
||||
|
||||
# Validate source type
|
||||
if request.source_type != "audio_search":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Only 'audio_search' source type is currently supported"
|
||||
)
|
||||
|
||||
# Determine current source type
|
||||
state_data = job.state_data or {}
|
||||
if state_data.get('audio_selection') or state_data.get('audio_search_results'):
|
||||
previous_source = "audio_search"
|
||||
elif job.url:
|
||||
previous_source = "youtube"
|
||||
elif job.input_media_gcs_path:
|
||||
previous_source = "file_upload"
|
||||
else:
|
||||
previous_source = "unknown"
|
||||
|
||||
previous_status = job.status
|
||||
cleared_keys = []
|
||||
|
||||
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
|
||||
db = job_manager.firestore.db
|
||||
job_ref = db.collection("jobs").document(job_id)
|
||||
|
||||
# Clear all audio-related state to start fresh with audio search
|
||||
keys_to_clear = [
|
||||
"audio_complete",
|
||||
"lyrics_complete",
|
||||
"audio_progress",
|
||||
"lyrics_progress",
|
||||
"screens_progress",
|
||||
"video_progress",
|
||||
"render_progress",
|
||||
"encoding_progress",
|
||||
"review_complete",
|
||||
"corrected_lyrics",
|
||||
"instrumental_selection",
|
||||
"instrumental_options",
|
||||
"lyrics_metadata",
|
||||
"audio_search_results",
|
||||
"audio_search_count",
|
||||
"remote_search_id",
|
||||
"audio_selection",
|
||||
]
|
||||
|
||||
update_payload = {}
|
||||
for key in keys_to_clear:
|
||||
if key in state_data:
|
||||
update_payload[f"state_data.{key}"] = DELETE_FIELD
|
||||
cleared_keys.append(key)
|
||||
|
||||
# Clear job-level audio fields
|
||||
update_payload["url"] = DELETE_FIELD # Clear YouTube URL
|
||||
update_payload["input_media_gcs_path"] = DELETE_FIELD # Clear downloaded file
|
||||
update_payload["error_message"] = DELETE_FIELD
|
||||
update_payload["error_details"] = DELETE_FIELD
|
||||
|
||||
# Set audio search fields based on current artist/title
|
||||
search_artist = job.artist
|
||||
search_title = job.title
|
||||
update_payload["audio_search_artist"] = search_artist
|
||||
update_payload["audio_search_title"] = search_title
|
||||
|
||||
# First transition to searching_audio
|
||||
update_payload["status"] = "searching_audio"
|
||||
update_payload["message"] = f"Searching for: {search_artist} - {search_title}"
|
||||
update_payload["progress"] = 5
|
||||
update_payload["timeline"] = ArrayUnion([{
|
||||
"status": "searching_audio",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"message": f"Admin {admin_email} switched audio source from {previous_source} to audio_search"
|
||||
}])
|
||||
|
||||
job_ref.update(update_payload)
|
||||
|
||||
# Perform the actual audio search
|
||||
search_results_count = None
|
||||
error_msg = None
|
||||
try:
|
||||
audio_search_service = get_audio_search_service()
|
||||
search_results = audio_search_service.search(search_artist, search_title)
|
||||
search_results_count = len(search_results)
|
||||
|
||||
# Store results in job state_data
|
||||
results_dicts = [r.to_dict() for r in search_results]
|
||||
state_data_update = {
|
||||
'audio_search_results': results_dicts,
|
||||
'audio_search_count': search_results_count,
|
||||
}
|
||||
if audio_search_service.last_remote_search_id:
|
||||
state_data_update['remote_search_id'] = audio_search_service.last_remote_search_id
|
||||
|
||||
# Transition to awaiting_audio_selection with results
|
||||
new_status = "awaiting_audio_selection"
|
||||
job_ref.update({
|
||||
"state_data": state_data_update,
|
||||
"status": new_status,
|
||||
"message": f"Found {search_results_count} audio sources - select one",
|
||||
"progress": 10,
|
||||
})
|
||||
|
||||
logger.info(
|
||||
f"Admin {admin_email} overrode audio source for job {job_id}: "
|
||||
f"{previous_source} -> audio_search, found {search_results_count} results"
|
||||
)
|
||||
|
||||
except NoResultsError as e:
|
||||
new_status = "failed"
|
||||
error_msg = f"No audio sources found for: {search_artist} - {search_title}"
|
||||
job_ref.update({
|
||||
"status": new_status,
|
||||
"message": error_msg,
|
||||
"error_message": error_msg,
|
||||
})
|
||||
logger.warning(f"Audio search failed for job {job_id}: {error_msg}")
|
||||
|
||||
except AudioSearchError as e:
|
||||
new_status = "failed"
|
||||
error_msg = f"Audio search failed: {e}"
|
||||
job_ref.update({
|
||||
"status": new_status,
|
||||
"message": error_msg,
|
||||
"error_message": error_msg,
|
||||
})
|
||||
logger.error(f"Audio search error for job {job_id}: {e}")
|
||||
|
||||
return OverrideAudioSourceResponse(
|
||||
status="success" if not error_msg else "error",
|
||||
job_id=job_id,
|
||||
message=f"Found {search_results_count} audio sources - select one in the admin panel." if not error_msg
|
||||
else error_msg,
|
||||
previous_source=previous_source,
|
||||
new_source="audio_search",
|
||||
cleared_data=cleared_keys,
|
||||
new_status=new_status,
|
||||
search_results_count=search_results_count,
|
||||
error=error_msg,
|
||||
)
|
||||
@@ -236,6 +236,14 @@ def extract_request_metadata(request: Request, created_from: str = "audio_search
|
||||
}
|
||||
|
||||
|
||||
async def _trigger_workers_parallel(job_id: str) -> None:
|
||||
"""Trigger both audio and lyrics workers in parallel."""
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
|
||||
def _extract_gcs_path(filepath: str) -> str:
|
||||
"""
|
||||
Extract the path portion from a GCS path or gs:// URL.
|
||||
@@ -426,31 +434,21 @@ async def _download_and_start_processing(
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp files: {e}")
|
||||
|
||||
# Update job with GCS path
|
||||
# Update job with GCS path and transition to DOWNLOADING
|
||||
job_manager.update_job(job_id, {
|
||||
'input_media_gcs_path': audio_gcs_path,
|
||||
'filename': filename,
|
||||
})
|
||||
|
||||
# Transition from DOWNLOADING_AUDIO to DOWNLOADING and trigger workers
|
||||
# Note: Can't use start_job_processing() here because that expects PENDING status,
|
||||
# but we're in DOWNLOADING_AUDIO state from the audio search flow.
|
||||
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=15,
|
||||
message="Audio downloaded, starting processing"
|
||||
)
|
||||
|
||||
# Trigger workers in background
|
||||
async def trigger_workers():
|
||||
worker_service = get_worker_service()
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
background_tasks.add_task(trigger_workers)
|
||||
|
||||
# Trigger workers
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
return {
|
||||
'selected_index': selection_index,
|
||||
|
||||
@@ -256,6 +256,19 @@ ALLOWED_IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp'}
|
||||
ALLOWED_FONT_EXTENSIONS = {'.ttf', '.otf', '.woff', '.woff2'}
|
||||
|
||||
|
||||
async def _trigger_workers_parallel(job_id: str) -> None:
|
||||
"""
|
||||
Trigger both audio and lyrics workers in parallel.
|
||||
|
||||
FastAPI's BackgroundTasks runs async tasks sequentially, so we use
|
||||
asyncio.gather to ensure both workers start at the same time.
|
||||
"""
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
|
||||
async def _trigger_audio_worker_only(job_id: str) -> None:
|
||||
"""
|
||||
Trigger only the audio worker.
|
||||
@@ -742,17 +755,18 @@ async def upload_and_create_job(
|
||||
if style_assets:
|
||||
logger.info(f"Style assets: {list(style_assets.keys())}")
|
||||
|
||||
# Use centralized start_job_processing which handles:
|
||||
# 1. Validates job exists and has input_media_gcs_path
|
||||
# 2. Transitions to DOWNLOADING (raises InvalidStateTransitionError if invalid)
|
||||
# 3. Triggers audio + lyrics workers in parallel
|
||||
background_tasks.add_task(
|
||||
job_manager.start_job_processing,
|
||||
job_id,
|
||||
5, # progress
|
||||
"Files uploaded, preparing to process"
|
||||
# Transition job to DOWNLOADING state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Files uploaded, preparing to process"
|
||||
)
|
||||
|
||||
# Trigger workers in parallel using asyncio.gather
|
||||
# (FastAPI's BackgroundTasks runs async tasks sequentially)
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
# Build distribution services info for response
|
||||
distribution_services: Dict[str, Any] = {}
|
||||
|
||||
@@ -1291,17 +1305,17 @@ async def mark_uploads_complete(
|
||||
|
||||
logger.info(f"Validated uploads for job {job_id}: {body.uploaded_files}")
|
||||
|
||||
# Use centralized start_job_processing which handles:
|
||||
# 1. Validates job exists and has input_media_gcs_path
|
||||
# 2. Transitions to DOWNLOADING (raises InvalidStateTransitionError if invalid)
|
||||
# 3. Triggers audio + lyrics workers in parallel
|
||||
background_tasks.add_task(
|
||||
job_manager.start_job_processing,
|
||||
job_id,
|
||||
5, # progress
|
||||
"Files uploaded, preparing to process"
|
||||
# Transition job to DOWNLOADING state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Files uploaded, preparing to process"
|
||||
)
|
||||
|
||||
# Trigger workers in parallel
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
# Get distribution services info for response
|
||||
credential_manager = get_credential_manager()
|
||||
distribution_services: Dict[str, Any] = {}
|
||||
@@ -1574,6 +1588,14 @@ async def create_job_from_url(
|
||||
if title:
|
||||
logger.info(f" Title: {title}")
|
||||
|
||||
# Transition job to DOWNLOADING state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Starting audio download from URL"
|
||||
)
|
||||
|
||||
# For YouTube URLs, download audio NOW using YouTubeDownloadService
|
||||
# This uses remote flacfetch (if configured) to avoid bot detection on Cloud Run
|
||||
if _is_youtube_url(body.url):
|
||||
@@ -1595,16 +1617,14 @@ async def create_job_from_url(
|
||||
|
||||
logger.info(f"YouTube audio downloaded to GCS: {audio_gcs_path}")
|
||||
|
||||
# Use centralized start_job_processing which handles:
|
||||
# 1. Validates job exists and has input_media_gcs_path
|
||||
# 2. Transitions PENDING → DOWNLOADING (raises if invalid)
|
||||
# 3. Triggers audio + lyrics workers in parallel
|
||||
background_tasks.add_task(
|
||||
job_manager.start_job_processing,
|
||||
job_id,
|
||||
10, # progress
|
||||
"Audio downloaded, starting processing"
|
||||
# Now trigger both workers in parallel (audio already downloaded)
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=10,
|
||||
message="Audio downloaded, starting processing"
|
||||
)
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
except YouTubeDownloadError as e:
|
||||
logger.error(f"YouTube download failed: {e}")
|
||||
@@ -1614,14 +1634,8 @@ async def create_job_from_url(
|
||||
detail=f"YouTube download failed: {e}"
|
||||
)
|
||||
else:
|
||||
# For non-YouTube URLs, transition to DOWNLOADING and trigger audio worker
|
||||
# For non-YouTube URLs, trigger ONLY audio worker first
|
||||
# The audio worker will download the URL and trigger lyrics worker after
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Starting audio download from URL"
|
||||
)
|
||||
background_tasks.add_task(_trigger_audio_worker_only, job_id)
|
||||
|
||||
return CreateJobFromUrlResponse(
|
||||
|
||||
@@ -16,9 +16,6 @@ from backend.services.encoding_service import get_encoding_service
|
||||
from backend.services.spacy_preloader import get_preloaded_model, is_model_preloaded
|
||||
from backend.services.nltk_preloader import get_preloaded_cmudict, is_cmudict_preloaded
|
||||
from backend.services.langfuse_preloader import get_preloaded_langfuse_handler, is_langfuse_preloaded, is_langfuse_configured
|
||||
from backend.services.job_health_service import check_job_consistency_detailed
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.models.job import JobStatus
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -433,69 +430,6 @@ async def preload_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health/job-consistency")
|
||||
async def job_consistency_check(
|
||||
status: Optional[str] = None,
|
||||
limit: int = 50
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Check job consistency across the system.
|
||||
|
||||
This endpoint identifies jobs with state inconsistencies that may indicate
|
||||
bugs in the state machine or trigger logic. Use this to find stuck jobs
|
||||
or jobs in invalid states.
|
||||
|
||||
Args:
|
||||
status: Optional filter by job status (e.g., "pending", "downloading")
|
||||
limit: Maximum number of jobs to check (default 50)
|
||||
|
||||
Returns:
|
||||
Dictionary with:
|
||||
- total_checked: Number of jobs checked
|
||||
- inconsistent_count: Number of jobs with issues
|
||||
- inconsistent_jobs: List of job details with their issues
|
||||
- summary: Summary of issue types found
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
|
||||
# Query jobs to check
|
||||
query_params = {}
|
||||
if status:
|
||||
try:
|
||||
job_status = JobStatus(status)
|
||||
query_params['status'] = job_status
|
||||
except ValueError:
|
||||
return {
|
||||
"error": f"Invalid status: {status}",
|
||||
"valid_statuses": [s.value for s in JobStatus]
|
||||
}
|
||||
|
||||
# Get jobs from Firestore
|
||||
jobs = job_manager.list_jobs(limit=limit, **query_params)
|
||||
|
||||
# Check each job for consistency
|
||||
inconsistent_jobs = []
|
||||
issue_summary = {}
|
||||
|
||||
for job in jobs:
|
||||
result = check_job_consistency_detailed(job)
|
||||
if not result['is_healthy']:
|
||||
inconsistent_jobs.append(result)
|
||||
# Count issue types
|
||||
for issue in result['issues']:
|
||||
issue_type = issue.split(':')[0] if ':' in issue else issue
|
||||
issue_summary[issue_type] = issue_summary.get(issue_type, 0) + 1
|
||||
|
||||
return {
|
||||
"total_checked": len(jobs),
|
||||
"inconsistent_count": len(inconsistent_jobs),
|
||||
"inconsistent_jobs": inconsistent_jobs,
|
||||
"summary": issue_summary,
|
||||
"status_filter": status,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/readiness")
|
||||
async def readiness_check() -> Dict[str, str]:
|
||||
"""Readiness check endpoint for Cloud Run."""
|
||||
|
||||
@@ -65,7 +65,6 @@ from backend.services.youtube_download_service import (
|
||||
get_youtube_download_service,
|
||||
YouTubeDownloadError,
|
||||
)
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -477,12 +476,14 @@ async def _handle_made_for_you_order(
|
||||
"""
|
||||
from backend.models.job import JobCreate, JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.worker_service import get_worker_service
|
||||
from backend.services.audio_search_service import (
|
||||
get_audio_search_service,
|
||||
NoResultsError,
|
||||
AudioSearchError,
|
||||
)
|
||||
from backend.services.storage_service import StorageService
|
||||
import asyncio
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
@@ -500,6 +501,7 @@ async def _handle_made_for_you_order(
|
||||
|
||||
try:
|
||||
job_manager = JobManager()
|
||||
worker_service = get_worker_service()
|
||||
storage_service = StorageService()
|
||||
|
||||
# Apply default theme (Nomad) - same as audio_search endpoint
|
||||
@@ -600,22 +602,13 @@ async def _handle_made_for_you_order(
|
||||
'filename': os.path.basename(audio_gcs_path),
|
||||
})
|
||||
|
||||
logger.info(f"Job {job_id}: YouTube audio downloaded to {audio_gcs_path}")
|
||||
logger.info(f"Job {job_id}: YouTube audio downloaded to {audio_gcs_path}, triggering workers")
|
||||
|
||||
# Use centralized helper that handles transition + worker triggers
|
||||
# This ensures consistent state machine flow and raises on invalid transitions
|
||||
try:
|
||||
await job_manager.start_job_processing(job_id)
|
||||
except InvalidStateTransitionError as e:
|
||||
logger.error(f"Job {job_id}: Failed to start processing - invalid state transition: {e}")
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.FAILED,
|
||||
progress=0,
|
||||
message=f"Failed to start processing: {str(e)}",
|
||||
raise_on_invalid=False # Don't raise if already failed
|
||||
)
|
||||
raise
|
||||
# Now trigger both workers in parallel (audio already downloaded)
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
except YouTubeDownloadError as e:
|
||||
logger.error(f"Job {job_id}: YouTube download failed: {e}")
|
||||
|
||||
@@ -64,39 +64,3 @@ class IPBlockedError(Exception):
|
||||
self.message = message
|
||||
self.ip_address = ip_address
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class InvalidStateTransitionError(Exception):
|
||||
"""
|
||||
Raised when an invalid job state transition is attempted.
|
||||
|
||||
This exception is raised by JobManager.transition_to_state() when the
|
||||
requested transition is not allowed by the state machine defined in
|
||||
STATE_TRANSITIONS.
|
||||
|
||||
Example: Trying to transition from PENDING to GENERATING_SCREENS is invalid
|
||||
because PENDING can only transition to DOWNLOADING, SEARCHING_AUDIO, FAILED,
|
||||
or CANCELLED.
|
||||
|
||||
Attributes:
|
||||
message: Human-readable error message
|
||||
job_id: The job ID that failed to transition
|
||||
from_status: Current status of the job
|
||||
to_status: Attempted target status
|
||||
valid_transitions: List of valid transitions from current status
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
job_id: str = "",
|
||||
from_status: str = "",
|
||||
to_status: str = "",
|
||||
valid_transitions: list = None
|
||||
):
|
||||
self.message = message
|
||||
self.job_id = job_id
|
||||
self.from_status = from_status
|
||||
self.to_status = to_status
|
||||
self.valid_transitions = valid_transitions or []
|
||||
super().__init__(message)
|
||||
@@ -1,265 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate a Mermaid state diagram from the job state machine.
|
||||
|
||||
This script reads the STATE_TRANSITIONS dictionary from models/job.py
|
||||
and generates a Mermaid diagram that can be embedded in documentation.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_state_diagram.py > docs/state-machine.mmd
|
||||
# Or to update ARCHITECTURE.md directly:
|
||||
python scripts/generate_state_diagram.py --update-docs
|
||||
|
||||
Created as part of state machine robustness improvements (2026-02-02).
|
||||
"""
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
# Add backend to path for imports
|
||||
backend_dir = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(backend_dir))
|
||||
|
||||
# Import the state machine data
|
||||
# Note: Must be run from an environment where karaoke_gen is installed
|
||||
try:
|
||||
from models.job import STATE_TRANSITIONS, JobStatus
|
||||
except ImportError:
|
||||
# Fallback: define the state machine inline for documentation purposes
|
||||
from enum import Enum
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
SEARCHING_AUDIO = "searching_audio"
|
||||
AWAITING_AUDIO_SELECTION = "awaiting_audio_selection"
|
||||
DOWNLOADING_AUDIO = "downloading_audio"
|
||||
DOWNLOADING = "downloading"
|
||||
SEPARATING_STAGE1 = "separating_stage1"
|
||||
SEPARATING_STAGE2 = "separating_stage2"
|
||||
AUDIO_COMPLETE = "audio_complete"
|
||||
TRANSCRIBING = "transcribing"
|
||||
CORRECTING = "correcting"
|
||||
LYRICS_COMPLETE = "lyrics_complete"
|
||||
GENERATING_SCREENS = "generating_screens"
|
||||
APPLYING_PADDING = "applying_padding"
|
||||
AWAITING_REVIEW = "awaiting_review"
|
||||
IN_REVIEW = "in_review"
|
||||
REVIEW_COMPLETE = "review_complete"
|
||||
RENDERING_VIDEO = "rendering_video"
|
||||
AWAITING_INSTRUMENTAL_SELECTION = "awaiting_instrumental_selection"
|
||||
INSTRUMENTAL_SELECTED = "instrumental_selected"
|
||||
GENERATING_VIDEO = "generating_video"
|
||||
ENCODING = "encoding"
|
||||
PACKAGING = "packaging"
|
||||
UPLOADING = "uploading"
|
||||
NOTIFYING = "notifying"
|
||||
COMPLETE = "complete"
|
||||
PREP_COMPLETE = "prep_complete"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
QUEUED = "queued"
|
||||
PROCESSING = "processing"
|
||||
ERROR = "error"
|
||||
READY_FOR_FINALIZATION = "ready_for_finalization"
|
||||
FINALIZING = "finalizing"
|
||||
|
||||
STATE_TRANSITIONS = {
|
||||
JobStatus.PENDING: [JobStatus.DOWNLOADING, JobStatus.SEARCHING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.SEARCHING_AUDIO: [JobStatus.AWAITING_AUDIO_SELECTION, JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED],
|
||||
JobStatus.AWAITING_AUDIO_SELECTION: [JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.DOWNLOADING_AUDIO: [JobStatus.DOWNLOADING, JobStatus.FAILED],
|
||||
JobStatus.DOWNLOADING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING, JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
JobStatus.SEPARATING_STAGE1: [JobStatus.SEPARATING_STAGE2, JobStatus.FAILED],
|
||||
JobStatus.SEPARATING_STAGE2: [JobStatus.AUDIO_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.AUDIO_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
JobStatus.TRANSCRIBING: [JobStatus.CORRECTING, JobStatus.FAILED],
|
||||
JobStatus.CORRECTING: [JobStatus.LYRICS_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.LYRICS_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
JobStatus.GENERATING_SCREENS: [JobStatus.APPLYING_PADDING, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.APPLYING_PADDING: [JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.AWAITING_REVIEW: [JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.IN_REVIEW: [JobStatus.REVIEW_COMPLETE, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.REVIEW_COMPLETE: [JobStatus.RENDERING_VIDEO, JobStatus.FAILED],
|
||||
JobStatus.RENDERING_VIDEO: [JobStatus.INSTRUMENTAL_SELECTED, JobStatus.FAILED],
|
||||
JobStatus.INSTRUMENTAL_SELECTED: [JobStatus.GENERATING_VIDEO, JobStatus.FAILED],
|
||||
JobStatus.GENERATING_VIDEO: [JobStatus.ENCODING, JobStatus.FAILED],
|
||||
JobStatus.ENCODING: [JobStatus.PACKAGING, JobStatus.COMPLETE, JobStatus.PREP_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.PACKAGING: [JobStatus.UPLOADING, JobStatus.COMPLETE, JobStatus.PREP_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
}
|
||||
|
||||
|
||||
def generate_mermaid() -> str:
|
||||
"""
|
||||
Generate a Mermaid state diagram from STATE_TRANSITIONS.
|
||||
|
||||
Returns:
|
||||
Mermaid diagram as a string
|
||||
"""
|
||||
lines = ["stateDiagram-v2"]
|
||||
lines.append(" direction TB")
|
||||
lines.append("")
|
||||
|
||||
# Group states by category for better visualization
|
||||
categories = {
|
||||
"Initial": [JobStatus.PENDING],
|
||||
"Audio Search": [
|
||||
JobStatus.SEARCHING_AUDIO,
|
||||
JobStatus.AWAITING_AUDIO_SELECTION,
|
||||
JobStatus.DOWNLOADING_AUDIO,
|
||||
],
|
||||
"Processing": [
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.SEPARATING_STAGE1,
|
||||
JobStatus.SEPARATING_STAGE2,
|
||||
JobStatus.AUDIO_COMPLETE,
|
||||
JobStatus.TRANSCRIBING,
|
||||
JobStatus.CORRECTING,
|
||||
JobStatus.LYRICS_COMPLETE,
|
||||
],
|
||||
"Screens": [
|
||||
JobStatus.GENERATING_SCREENS,
|
||||
JobStatus.APPLYING_PADDING,
|
||||
],
|
||||
"Review": [
|
||||
JobStatus.AWAITING_REVIEW,
|
||||
JobStatus.IN_REVIEW,
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
],
|
||||
"Video": [
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
JobStatus.INSTRUMENTAL_SELECTED,
|
||||
JobStatus.GENERATING_VIDEO,
|
||||
JobStatus.ENCODING,
|
||||
JobStatus.PACKAGING,
|
||||
],
|
||||
"Distribution": [
|
||||
JobStatus.UPLOADING,
|
||||
JobStatus.NOTIFYING,
|
||||
],
|
||||
"Terminal": [
|
||||
JobStatus.COMPLETE,
|
||||
JobStatus.PREP_COMPLETE,
|
||||
JobStatus.FAILED,
|
||||
JobStatus.CANCELLED,
|
||||
],
|
||||
}
|
||||
|
||||
# Add state definitions with notes for key states
|
||||
lines.append(" %% State definitions")
|
||||
key_states = {
|
||||
JobStatus.PENDING: "Job created",
|
||||
JobStatus.AWAITING_AUDIO_SELECTION: "⚠️ USER ACTION",
|
||||
JobStatus.AWAITING_REVIEW: "⚠️ USER ACTION",
|
||||
JobStatus.COMPLETE: "✅ Success",
|
||||
JobStatus.FAILED: "❌ Error",
|
||||
}
|
||||
|
||||
for status, note in key_states.items():
|
||||
lines.append(f" {status.value}: {status.value}")
|
||||
lines.append(f" note right of {status.value}: {note}")
|
||||
|
||||
lines.append("")
|
||||
lines.append(" %% Transitions")
|
||||
|
||||
# Generate transitions
|
||||
for from_status, to_statuses in STATE_TRANSITIONS.items():
|
||||
for to_status in to_statuses:
|
||||
# Skip deprecated statuses
|
||||
if to_status in [JobStatus.QUEUED, JobStatus.PROCESSING, JobStatus.ERROR,
|
||||
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING]:
|
||||
continue
|
||||
if from_status in [JobStatus.QUEUED, JobStatus.PROCESSING, JobStatus.ERROR,
|
||||
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING]:
|
||||
continue
|
||||
|
||||
lines.append(f" {from_status.value} --> {to_status.value}")
|
||||
|
||||
# Add start and end markers
|
||||
lines.append("")
|
||||
lines.append(" %% Start and end")
|
||||
lines.append(" [*] --> pending")
|
||||
lines.append(" complete --> [*]")
|
||||
lines.append(" prep_complete --> [*]")
|
||||
lines.append(" failed --> [*]")
|
||||
lines.append(" cancelled --> [*]")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_markdown_section() -> str:
|
||||
"""
|
||||
Generate a markdown section with the state diagram.
|
||||
|
||||
Returns:
|
||||
Markdown string with the diagram
|
||||
"""
|
||||
diagram = generate_mermaid()
|
||||
|
||||
return f"""## Job State Machine
|
||||
|
||||
The following diagram shows all valid state transitions for a karaoke generation job.
|
||||
|
||||
Key states:
|
||||
- **PENDING**: Initial state when job is created
|
||||
- **AWAITING_AUDIO_SELECTION**: Human selects audio source (for artist+title search)
|
||||
- **AWAITING_REVIEW**: Human reviews lyrics and selects instrumental
|
||||
- **COMPLETE**: All processing finished successfully
|
||||
|
||||
```mermaid
|
||||
{diagram}
|
||||
```
|
||||
|
||||
### Parallel Processing
|
||||
|
||||
Jobs use parallel processing for audio separation and lyrics transcription:
|
||||
|
||||
1. **DOWNLOADING** triggers both:
|
||||
- Audio worker → SEPARATING_STAGE1 → SEPARATING_STAGE2 → AUDIO_COMPLETE
|
||||
- Lyrics worker → TRANSCRIBING → CORRECTING → LYRICS_COMPLETE
|
||||
|
||||
2. When BOTH complete, the screens worker is triggered automatically.
|
||||
|
||||
### Human Review Points
|
||||
|
||||
Jobs wait for human input at these points:
|
||||
- **AWAITING_AUDIO_SELECTION**: User selects audio source from search results
|
||||
- **AWAITING_REVIEW**: User reviews/corrects lyrics AND selects instrumental
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Any state can transition to **FAILED** on unrecoverable errors
|
||||
- **CANCELLED** is set when user cancels the job
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate state machine diagram")
|
||||
parser.add_argument(
|
||||
"--update-docs",
|
||||
action="store_true",
|
||||
help="Update ARCHITECTURE.md with the diagram"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--markdown",
|
||||
action="store_true",
|
||||
help="Output markdown section instead of raw Mermaid"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.update_docs:
|
||||
# TODO: Implement auto-update of ARCHITECTURE.md
|
||||
print("Auto-update of ARCHITECTURE.md not yet implemented")
|
||||
print("Please copy the output below to the appropriate section:")
|
||||
print()
|
||||
print(generate_markdown_section())
|
||||
elif args.markdown:
|
||||
print(generate_markdown_section())
|
||||
else:
|
||||
print(generate_mermaid())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,221 +0,0 @@
|
||||
"""
|
||||
Job health and consistency checking service.
|
||||
|
||||
Provides functions to detect state inconsistencies in jobs that might indicate
|
||||
bugs in the state machine or trigger logic.
|
||||
|
||||
Created as part of state machine robustness improvements (2026-02-02).
|
||||
"""
|
||||
import logging
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Jobs in these statuses should NOT have audio_complete flag set
|
||||
# (unless they're actively processing or beyond)
|
||||
STATUSES_BEFORE_AUDIO_START = {
|
||||
JobStatus.PENDING,
|
||||
JobStatus.SEARCHING_AUDIO,
|
||||
JobStatus.AWAITING_AUDIO_SELECTION,
|
||||
}
|
||||
|
||||
# Jobs in these statuses should NOT have lyrics_complete flag set
|
||||
STATUSES_BEFORE_LYRICS_START = {
|
||||
JobStatus.PENDING,
|
||||
JobStatus.SEARCHING_AUDIO,
|
||||
JobStatus.AWAITING_AUDIO_SELECTION,
|
||||
}
|
||||
|
||||
# Statuses where screens_progress.stage='complete' is valid
|
||||
STATUSES_AFTER_SCREENS_COMPLETE = {
|
||||
JobStatus.AWAITING_REVIEW,
|
||||
JobStatus.IN_REVIEW,
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
JobStatus.AWAITING_INSTRUMENTAL_SELECTION, # Legacy
|
||||
JobStatus.INSTRUMENTAL_SELECTED,
|
||||
JobStatus.GENERATING_VIDEO,
|
||||
JobStatus.ENCODING,
|
||||
JobStatus.PACKAGING,
|
||||
JobStatus.UPLOADING,
|
||||
JobStatus.NOTIFYING,
|
||||
JobStatus.COMPLETE,
|
||||
JobStatus.PREP_COMPLETE,
|
||||
}
|
||||
|
||||
|
||||
def check_job_consistency(job: Job) -> List[str]:
|
||||
"""
|
||||
Check for state inconsistencies in a job.
|
||||
|
||||
This function detects situations where state_data flags indicate
|
||||
progress that doesn't match the job status - which usually indicates
|
||||
a bug in the state machine or trigger logic.
|
||||
|
||||
Args:
|
||||
job: Job object to check
|
||||
|
||||
Returns:
|
||||
List of issue descriptions found. Empty list means no issues.
|
||||
"""
|
||||
issues = []
|
||||
state_data = job.state_data or {}
|
||||
|
||||
# Normalize status to JobStatus enum if it's a string
|
||||
status = job.status
|
||||
if isinstance(status, str):
|
||||
try:
|
||||
status = JobStatus(status)
|
||||
except ValueError:
|
||||
issues.append(f"invalid_status: '{job.status}' is not a valid JobStatus")
|
||||
return issues
|
||||
|
||||
# Check: audio_complete=True but status still in pre-audio states
|
||||
if state_data.get('audio_complete') and status in STATUSES_BEFORE_AUDIO_START:
|
||||
issues.append(
|
||||
f"audio_complete_status_mismatch: audio_complete=True but status={status.value}"
|
||||
)
|
||||
|
||||
# Check: lyrics_complete=True but status still in pre-lyrics states
|
||||
if state_data.get('lyrics_complete') and status in STATUSES_BEFORE_LYRICS_START:
|
||||
issues.append(
|
||||
f"lyrics_complete_status_mismatch: lyrics_complete=True but status={status.value}"
|
||||
)
|
||||
|
||||
# Check: screens_progress.stage='complete' but status not in post-screens states
|
||||
screens_progress = state_data.get('screens_progress', {})
|
||||
if isinstance(screens_progress, dict) and screens_progress.get('stage') == 'complete':
|
||||
if status not in STATUSES_AFTER_SCREENS_COMPLETE:
|
||||
issues.append(
|
||||
f"screens_complete_status_mismatch: screens_progress.stage=complete but status={status.value}"
|
||||
)
|
||||
|
||||
# Check: job has input_media_gcs_path but is stuck at pending
|
||||
# This was the exact bug that got jobs 06cfea29 and 984da08b stuck
|
||||
if job.input_media_gcs_path and status == JobStatus.PENDING:
|
||||
# This could be legitimate if job was just created and workers haven't started
|
||||
# But if it's been more than a few minutes, it's likely stuck
|
||||
# We flag it as a warning, not an error
|
||||
issues.append(
|
||||
f"pending_with_input_media: has input_media_gcs_path but status=pending (may be stuck)"
|
||||
)
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def check_job_consistency_detailed(job: Job) -> Dict[str, Any]:
|
||||
"""
|
||||
Check job consistency with detailed information for debugging.
|
||||
|
||||
Returns a dictionary with:
|
||||
- job_id: The job ID
|
||||
- status: Current job status
|
||||
- issues: List of issue descriptions
|
||||
- state_data_summary: Summary of relevant state_data fields
|
||||
- is_healthy: Boolean indicating if no issues were found
|
||||
|
||||
Args:
|
||||
job: Job object to check
|
||||
|
||||
Returns:
|
||||
Dictionary with detailed consistency information
|
||||
"""
|
||||
issues = check_job_consistency(job)
|
||||
state_data = job.state_data or {}
|
||||
|
||||
return {
|
||||
"job_id": job.job_id,
|
||||
"status": job.status.value if hasattr(job.status, 'value') else str(job.status),
|
||||
"issues": issues,
|
||||
"state_data_summary": {
|
||||
"audio_complete": state_data.get('audio_complete', False),
|
||||
"lyrics_complete": state_data.get('lyrics_complete', False),
|
||||
"screens_progress_stage": state_data.get('screens_progress', {}).get('stage'),
|
||||
"has_input_media_gcs_path": bool(job.input_media_gcs_path),
|
||||
},
|
||||
"is_healthy": len(issues) == 0,
|
||||
}
|
||||
|
||||
|
||||
def get_worker_valid_statuses() -> Dict[str, List[JobStatus]]:
|
||||
"""
|
||||
Get mapping of worker names to their valid job statuses.
|
||||
|
||||
This is used by workers to validate they should run for a given job.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping worker name to list of valid JobStatus values
|
||||
"""
|
||||
return {
|
||||
"audio_worker": [
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.DOWNLOADING_AUDIO,
|
||||
JobStatus.SEPARATING_STAGE1,
|
||||
JobStatus.SEPARATING_STAGE2,
|
||||
],
|
||||
"lyrics_worker": [
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.TRANSCRIBING,
|
||||
JobStatus.CORRECTING,
|
||||
],
|
||||
"screens_worker": [
|
||||
# Screens worker can be triggered when both audio+lyrics complete
|
||||
# which can happen in any of these states
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.AUDIO_COMPLETE,
|
||||
JobStatus.LYRICS_COMPLETE,
|
||||
# Also valid if re-running after a failure
|
||||
JobStatus.GENERATING_SCREENS,
|
||||
],
|
||||
"video_worker": [
|
||||
JobStatus.INSTRUMENTAL_SELECTED,
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
JobStatus.GENERATING_VIDEO,
|
||||
],
|
||||
"render_video_worker": [
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def validate_worker_can_run(worker_name: str, job: Job) -> Optional[str]:
|
||||
"""
|
||||
Validate that a worker should run for the given job.
|
||||
|
||||
Args:
|
||||
worker_name: Name of the worker (e.g., "screens_worker")
|
||||
job: Job object to validate
|
||||
|
||||
Returns:
|
||||
None if valid, or an error message string if invalid
|
||||
"""
|
||||
valid_statuses_map = get_worker_valid_statuses()
|
||||
|
||||
if worker_name not in valid_statuses_map:
|
||||
# Unknown worker, can't validate
|
||||
return None
|
||||
|
||||
valid_statuses = valid_statuses_map[worker_name]
|
||||
|
||||
# Normalize status
|
||||
status = job.status
|
||||
if isinstance(status, str):
|
||||
try:
|
||||
status = JobStatus(status)
|
||||
except ValueError:
|
||||
return f"Invalid job status: {job.status}"
|
||||
|
||||
if status not in valid_statuses:
|
||||
valid_names = [s.value for s in valid_statuses]
|
||||
return (
|
||||
f"{worker_name} called but job status is {status.value}, "
|
||||
f"expected one of {valid_names}. This may indicate a bug in the trigger logic."
|
||||
)
|
||||
|
||||
return None
|
||||
+17
-113
@@ -14,7 +14,7 @@ from datetime import datetime
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
from backend.config import settings
|
||||
from backend.exceptions import RateLimitExceededError, InvalidStateTransitionError
|
||||
from backend.exceptions import RateLimitExceededError
|
||||
from backend.models.job import Job, JobStatus, JobCreate, STATE_TRANSITIONS
|
||||
from backend.models.worker_log import WorkerLogEntry
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
@@ -316,87 +316,53 @@ class JobManager:
|
||||
self.firestore.delete_job(job_id)
|
||||
logger.info(f"Deleted job {job_id}")
|
||||
|
||||
def validate_state_transition(
|
||||
self,
|
||||
job_id: str,
|
||||
new_status: JobStatus,
|
||||
raise_on_invalid: bool = True
|
||||
) -> bool:
|
||||
def validate_state_transition(self, job_id: str, new_status: JobStatus) -> bool:
|
||||
"""
|
||||
Validate that a state transition is legal.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
new_status: Target state
|
||||
raise_on_invalid: If True, raise InvalidStateTransitionError on invalid
|
||||
transitions. If False, return False (legacy behavior).
|
||||
|
||||
|
||||
Returns:
|
||||
True if transition is valid
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If transition is invalid and raise_on_invalid=True
|
||||
ValueError: If job not found and raise_on_invalid=True
|
||||
True if transition is valid, False otherwise
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
error_msg = f"Job {job_id} not found"
|
||||
logger.error(error_msg)
|
||||
if raise_on_invalid:
|
||||
raise ValueError(error_msg)
|
||||
logger.error(f"Job {job_id} not found")
|
||||
return False
|
||||
|
||||
|
||||
current_status = job.status
|
||||
valid_transitions = STATE_TRANSITIONS.get(current_status, [])
|
||||
|
||||
|
||||
if new_status not in valid_transitions:
|
||||
error_msg = (
|
||||
logger.error(
|
||||
f"Invalid state transition for job {job_id}: "
|
||||
f"{current_status} -> {new_status}. "
|
||||
f"Valid transitions: {[s.value for s in valid_transitions]}"
|
||||
f"Valid transitions: {valid_transitions}"
|
||||
)
|
||||
logger.error(error_msg)
|
||||
|
||||
if raise_on_invalid:
|
||||
raise InvalidStateTransitionError(
|
||||
message=error_msg,
|
||||
job_id=job_id,
|
||||
from_status=current_status,
|
||||
to_status=new_status.value if hasattr(new_status, 'value') else str(new_status),
|
||||
valid_transitions=[s.value for s in valid_transitions]
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def transition_to_state(
|
||||
self,
|
||||
job_id: str,
|
||||
new_status: JobStatus,
|
||||
progress: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
state_data_updates: Optional[Dict[str, Any]] = None,
|
||||
raise_on_invalid: bool = True
|
||||
state_data_updates: Optional[Dict[str, Any]] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Transition job to new state with validation.
|
||||
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
new_status: Target state
|
||||
progress: Progress percentage (0-100)
|
||||
message: Timeline message
|
||||
state_data_updates: Updates to state_data field
|
||||
raise_on_invalid: If True (default), raise InvalidStateTransitionError
|
||||
on invalid transitions. If False, return False silently.
|
||||
|
||||
|
||||
Returns:
|
||||
True if transition succeeded, False if failed (only when raise_on_invalid=False)
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If transition is invalid and raise_on_invalid=True
|
||||
True if transition succeeded, False otherwise
|
||||
"""
|
||||
if not self.validate_state_transition(job_id, new_status, raise_on_invalid=raise_on_invalid):
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
return False
|
||||
|
||||
updates = {
|
||||
@@ -802,69 +768,7 @@ class JobManager:
|
||||
lyrics_complete = job.state_data.get('lyrics_complete', False)
|
||||
|
||||
return audio_complete and lyrics_complete
|
||||
|
||||
async def start_job_processing(
|
||||
self,
|
||||
job_id: str,
|
||||
progress: int = 15,
|
||||
message: str = "Starting audio and lyrics processing"
|
||||
) -> None:
|
||||
"""
|
||||
Transition job to DOWNLOADING and trigger audio + lyrics workers.
|
||||
|
||||
This is the single entry point for starting job processing after audio
|
||||
is available. All handlers (file_upload, webhook, audio_search) should
|
||||
use this method to ensure consistent state transitions.
|
||||
|
||||
The method:
|
||||
1. Validates job exists and has input_media_gcs_path
|
||||
2. Transitions to DOWNLOADING status (raises if invalid)
|
||||
3. Triggers audio and lyrics workers in parallel
|
||||
|
||||
Args:
|
||||
job_id: Job ID to start processing
|
||||
progress: Progress percentage to set (default 15)
|
||||
message: Timeline message (default "Starting audio and lyrics processing")
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If job can't transition to DOWNLOADING
|
||||
ValueError: If job not found or missing input_media_gcs_path
|
||||
|
||||
Example:
|
||||
# After downloading audio to GCS:
|
||||
job_manager.update_job(job_id, {'input_media_gcs_path': audio_path})
|
||||
await job_manager.start_job_processing(job_id)
|
||||
"""
|
||||
import asyncio
|
||||
from backend.services.worker_service import get_worker_service
|
||||
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
|
||||
if not job.input_media_gcs_path:
|
||||
raise ValueError(
|
||||
f"Job {job_id} missing input_media_gcs_path. "
|
||||
"Audio must be downloaded to GCS before starting processing."
|
||||
)
|
||||
|
||||
# Transition to DOWNLOADING - this will raise if invalid
|
||||
self.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=progress,
|
||||
message=message
|
||||
)
|
||||
|
||||
# Trigger both workers in parallel
|
||||
worker_service = get_worker_service()
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id}: Started processing (audio + lyrics workers triggered)")
|
||||
|
||||
|
||||
def mark_audio_complete(self, job_id: str) -> None:
|
||||
"""
|
||||
Mark audio processing as complete and check if can proceed.
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
"""
|
||||
Integration tests for admin job reset endpoint.
|
||||
|
||||
These tests use the real Firestore emulator and real JobManager,
|
||||
catching bugs that unit tests with mocks might miss (e.g., incorrect
|
||||
assumptions about return values).
|
||||
|
||||
Run with: scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.tests.emulator.conftest import emulators_running
|
||||
|
||||
# Skip all tests if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Only import if emulators are running
|
||||
if emulators_running():
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
|
||||
|
||||
class TestUpdateJobReturnValue:
|
||||
"""
|
||||
Critical tests documenting that update_job() returns None, not boolean.
|
||||
|
||||
This test class exists because a production bug (PR #371) was caused by
|
||||
assuming update_job() returns a boolean success indicator. The real API
|
||||
returns None on success and raises on failure. Mocks that return True
|
||||
hide this behavior.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def job_manager(self):
|
||||
"""Create a real JobManager using emulator."""
|
||||
return JobManager()
|
||||
|
||||
@pytest.fixture
|
||||
def firestore_service(self):
|
||||
"""Create a real FirestoreService using emulator."""
|
||||
return FirestoreService()
|
||||
|
||||
@pytest.fixture
|
||||
def test_job(self, firestore_service):
|
||||
"""Create a test job for verification."""
|
||||
job_id = f"update-return-test-{datetime.now(UTC).timestamp()}"
|
||||
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
)
|
||||
|
||||
firestore_service.create_job(job)
|
||||
|
||||
yield job
|
||||
|
||||
# Cleanup
|
||||
try:
|
||||
firestore_service.delete_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def test_update_job_returns_none_not_boolean(self, job_manager, test_job):
|
||||
"""
|
||||
Verify update_job() returns None, not a boolean.
|
||||
|
||||
This test exists because a production bug was introduced when code
|
||||
assumed update_job() returns a boolean. The method actually returns
|
||||
None on success and raises an exception on failure.
|
||||
|
||||
Bug pattern that this test prevents:
|
||||
```python
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
if not success: # BUG: None is falsy, so this is always True!
|
||||
raise HTTPException(status_code=500, ...)
|
||||
```
|
||||
|
||||
Correct pattern:
|
||||
```python
|
||||
# update_job() raises on failure, returns None on success
|
||||
job_manager.update_job(job_id, updates)
|
||||
```
|
||||
"""
|
||||
# update_job() should return None, not True/False
|
||||
result = job_manager.update_job(
|
||||
test_job.job_id,
|
||||
{"status": JobStatus.PENDING.value}
|
||||
)
|
||||
|
||||
# The actual return value is None, not True
|
||||
assert result is None, (
|
||||
"update_job() should return None, not a boolean. "
|
||||
"Code checking 'if not update_job(...)' will always be True!"
|
||||
)
|
||||
|
||||
def test_update_job_on_nonexistent_job(self, job_manager):
|
||||
"""
|
||||
Document update_job() behavior for non-existent jobs.
|
||||
|
||||
In production Firestore, update() on non-existent doc raises NotFound.
|
||||
The key insight for preventing the admin reset bug is understanding
|
||||
that update_job() returns None (not boolean) on success.
|
||||
"""
|
||||
try:
|
||||
result = job_manager.update_job(
|
||||
"nonexistent-job-id-12345",
|
||||
{"status": JobStatus.PENDING.value}
|
||||
)
|
||||
# If no exception raised (emulator quirk), verify return is still None
|
||||
assert result is None, "update_job should return None even on non-existent doc"
|
||||
except Exception:
|
||||
# Exception is the expected behavior in production Firestore
|
||||
pass
|
||||
@@ -1,69 +0,0 @@
|
||||
"""
|
||||
Integration tests for admin job reset using REAL job fixture from production.
|
||||
|
||||
This test validates that update_job() returns None, not a boolean
|
||||
(the root cause of PR #371 bug).
|
||||
|
||||
Uses actual data from production job 984da08b which needed admin reset.
|
||||
|
||||
Run with: scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.tests.emulator.conftest import emulators_running
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
if emulators_running():
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
|
||||
|
||||
class TestAdminResetBugFix:
|
||||
"""
|
||||
Tests that prove the PR #371 bug fix works using real emulator.
|
||||
|
||||
The bug: admin reset endpoint returned 500 because:
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
if not success: # BUG: update_job returns None, not True!
|
||||
raise HTTPException(500)
|
||||
"""
|
||||
|
||||
def test_update_job_returns_none_not_boolean(self):
|
||||
"""
|
||||
Prove that update_job() returns None, not a boolean.
|
||||
|
||||
This is the root cause of the bug. Any code that does
|
||||
'if not update_job(...)' will always be True.
|
||||
"""
|
||||
jm = JobManager()
|
||||
job_id = f"test-{datetime.now(UTC).timestamp()}"
|
||||
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Mark Mallman",
|
||||
title="Minneapolis",
|
||||
)
|
||||
|
||||
try:
|
||||
jm.firestore.create_job(job)
|
||||
|
||||
# THE KEY TEST: update_job returns None
|
||||
result = jm.update_job(job_id, {"status": "pending"})
|
||||
|
||||
assert result is None, (
|
||||
"update_job() must return None. "
|
||||
"Code checking 'if not success:' would always fail!"
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
jm.firestore.delete_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -218,16 +218,16 @@ class TestCountdownStateTransitions:
|
||||
"""
|
||||
Countdown state should survive job state transitions.
|
||||
|
||||
When job moves from GENERATING_VIDEO to ENCODING,
|
||||
When job moves from RENDERING_VIDEO to ENCODING,
|
||||
the lyrics_metadata should remain intact.
|
||||
"""
|
||||
job_id = f"transition-test-{datetime.now(UTC).timestamp()}"
|
||||
|
||||
try:
|
||||
# Create job in GENERATING_VIDEO state (valid predecessor to ENCODING)
|
||||
# Create job in RENDERING_VIDEO state
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.GENERATING_VIDEO,
|
||||
status=JobStatus.RENDERING_VIDEO,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
@@ -241,7 +241,7 @@ class TestCountdownStateTransitions:
|
||||
)
|
||||
firestore_service.create_job(job)
|
||||
|
||||
# Transition to ENCODING (valid transition from GENERATING_VIDEO)
|
||||
# Transition to ENCODING
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.ENCODING,
|
||||
|
||||
@@ -229,12 +229,6 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
mock_job.job_id = "test-job-workers"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it to call workers directly
|
||||
async def mock_start_job_processing(job_id, progress=15, message=""):
|
||||
await mock_worker_service_local.trigger_audio_worker(job_id)
|
||||
await mock_worker_service_local.trigger_lyrics_worker(job_id)
|
||||
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
@@ -251,7 +245,7 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
email_service=mock_email_service,
|
||||
)
|
||||
|
||||
# Verify workers were triggered via start_job_processing
|
||||
# Verify workers were triggered
|
||||
mock_worker_service_local.trigger_audio_worker.assert_called_once_with(mock_job.job_id)
|
||||
mock_worker_service_local.trigger_lyrics_worker.assert_called_once_with(mock_job.job_id)
|
||||
|
||||
@@ -275,9 +269,6 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
mock_job.job_id = "test-job-notifications"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it as simple AsyncMock
|
||||
mock_job_manager.start_job_processing = AsyncMock()
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
@@ -398,12 +389,6 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
mock_job.job_id = "test-job-no-search"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it to call workers directly
|
||||
async def mock_start_job_processing(job_id, progress=15, message=""):
|
||||
await mock_worker_service_local.trigger_audio_worker(job_id)
|
||||
await mock_worker_service_local.trigger_lyrics_worker(job_id)
|
||||
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.audio_search_service.get_audio_search_service') as mock_get_audio, \
|
||||
@@ -427,7 +412,7 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
# Audio search should NOT be called for YouTube URL orders
|
||||
mock_audio_service.search.assert_not_called()
|
||||
|
||||
# Workers should be triggered directly via start_job_processing
|
||||
# Workers should be triggered directly
|
||||
mock_worker_service_local.trigger_audio_worker.assert_called_once()
|
||||
|
||||
|
||||
@@ -563,12 +548,6 @@ class TestMadeForYouWebhookIntegration:
|
||||
mock_job.job_id = "test-job-webhook-flow"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it to call workers directly
|
||||
async def mock_start_job_processing(job_id, progress=15, message=""):
|
||||
await mock_worker_service_local.trigger_audio_worker(job_id)
|
||||
await mock_worker_service_local.trigger_lyrics_worker(job_id)
|
||||
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
@@ -594,7 +573,7 @@ class TestMadeForYouWebhookIntegration:
|
||||
# 2. Verify YouTube download was called
|
||||
mock_youtube_download_service.download.assert_called_once()
|
||||
|
||||
# 3. Verify workers were triggered via start_job_processing
|
||||
# 3. Verify workers were triggered
|
||||
mock_worker_service_local.trigger_audio_worker.assert_called_once()
|
||||
mock_worker_service_local.trigger_lyrics_worker.assert_called_once()
|
||||
|
||||
|
||||
@@ -1,348 +0,0 @@
|
||||
"""
|
||||
Integration tests for the job state machine flow.
|
||||
|
||||
These tests verify complete state transitions without mocking the state machine,
|
||||
ensuring the flow works correctly end-to-end (with mocked external services).
|
||||
|
||||
Created as part of state machine robustness improvements (2026-02-02).
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
|
||||
# Mock Firestore before importing JobManager
|
||||
import sys
|
||||
sys.modules['google.cloud.firestore'] = MagicMock()
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_manager import JobManager, InvalidStateTransitionError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_firestore_service():
|
||||
"""Mock FirestoreService."""
|
||||
with patch('backend.services.job_manager.FirestoreService') as mock:
|
||||
service = Mock()
|
||||
mock.return_value = service
|
||||
yield service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def job_manager(mock_firestore_service):
|
||||
"""Create JobManager instance with mocked Firestore."""
|
||||
return JobManager()
|
||||
|
||||
|
||||
class TestMadeForYouYouTubeFlow:
|
||||
"""
|
||||
Integration tests for Made-For-You YouTube order flow.
|
||||
|
||||
This was the flow that caused jobs 06cfea29 and 984da08b to get stuck.
|
||||
The fix ensures proper state transitions from PENDING -> DOWNLOADING.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_flow_pending_to_downloading(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test the flow from PENDING to DOWNLOADING for YouTube orders.
|
||||
|
||||
This verifies the fix for the bug where jobs got stuck at PENDING
|
||||
because start_job_processing wasn't called properly.
|
||||
"""
|
||||
# Create initial job in PENDING state
|
||||
job = Job(
|
||||
job_id="test-mfy-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/test.mp3",
|
||||
state_data={},
|
||||
)
|
||||
# Access the firestore through the job_manager's instance
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job_status.return_value = True
|
||||
|
||||
# Mock worker service
|
||||
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
|
||||
mock_worker = Mock()
|
||||
mock_worker.trigger_audio_worker = AsyncMock()
|
||||
mock_worker.trigger_lyrics_worker = AsyncMock()
|
||||
mock_get_worker.return_value = mock_worker
|
||||
|
||||
# Call start_job_processing - this is what _handle_made_for_you_order now uses
|
||||
await job_manager.start_job_processing("test-mfy-001")
|
||||
|
||||
# Verify transition to DOWNLOADING was called
|
||||
job_manager.firestore.update_job_status.assert_called()
|
||||
call_kwargs = job_manager.firestore.update_job_status.call_args.kwargs
|
||||
assert call_kwargs['status'] == JobStatus.DOWNLOADING
|
||||
|
||||
# Verify both workers were triggered
|
||||
mock_worker.trigger_audio_worker.assert_called_once_with("test-mfy-001")
|
||||
mock_worker.trigger_lyrics_worker.assert_called_once_with("test-mfy-001")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fails_if_job_stuck_at_wrong_state(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that starting processing fails if job is in wrong state.
|
||||
|
||||
Jobs that are already past PENDING (e.g., DOWNLOADING) shouldn't
|
||||
be able to start processing again - this would indicate a bug.
|
||||
"""
|
||||
# Create job already in DOWNLOADING state
|
||||
job = Job(
|
||||
job_id="test-mfy-002",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING, # Already past PENDING
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/test.mp3",
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
|
||||
# Attempting to start processing should raise InvalidStateTransitionError
|
||||
# because DOWNLOADING -> DOWNLOADING is not a valid transition
|
||||
with pytest.raises(InvalidStateTransitionError):
|
||||
await job_manager.start_job_processing("test-mfy-002")
|
||||
|
||||
|
||||
class TestParallelProcessingCoordination:
|
||||
"""
|
||||
Integration tests for parallel audio + lyrics processing coordination.
|
||||
|
||||
Both workers run in parallel and set flags when complete.
|
||||
Screens worker is triggered when both are done.
|
||||
"""
|
||||
|
||||
def test_audio_complete_updates_state_data(self, job_manager, mock_firestore_service):
|
||||
"""Test that mark_audio_complete updates state_data correctly."""
|
||||
job = Job(
|
||||
job_id="test-parallel-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job.return_value = True
|
||||
|
||||
# Call mark_audio_complete
|
||||
job_manager.mark_audio_complete("test-parallel-001")
|
||||
|
||||
# Verify update_job was called to set the audio_complete flag
|
||||
job_manager.firestore.update_job.assert_called()
|
||||
|
||||
def test_lyrics_complete_updates_state_data(self, job_manager, mock_firestore_service):
|
||||
"""Test that mark_lyrics_complete updates state_data correctly."""
|
||||
job = Job(
|
||||
job_id="test-parallel-002",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job.return_value = True
|
||||
|
||||
# Call mark_lyrics_complete
|
||||
job_manager.mark_lyrics_complete("test-parallel-002")
|
||||
|
||||
# Verify update_job was called to set the lyrics_complete flag
|
||||
job_manager.firestore.update_job.assert_called()
|
||||
|
||||
def test_both_complete_triggers_screens(self, job_manager, mock_firestore_service):
|
||||
"""Test that screens worker is triggered when both complete."""
|
||||
# First call: Job with audio already complete (for update_state_data)
|
||||
# Second call: Job with both complete (for check_parallel_processing_complete)
|
||||
job_audio_only = Job(
|
||||
job_id="test-parallel-003",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True},
|
||||
)
|
||||
job_both_complete = Job(
|
||||
job_id="test-parallel-003",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True, 'lyrics_complete': True},
|
||||
)
|
||||
# Return different jobs on subsequent calls
|
||||
job_manager.firestore.get_job.side_effect = [job_audio_only, job_both_complete]
|
||||
job_manager.firestore.update_job.return_value = True
|
||||
|
||||
# Mock the screens worker trigger
|
||||
with patch.object(job_manager, '_trigger_screens_worker') as mock_trigger:
|
||||
# Now lyrics completes
|
||||
job_manager.mark_lyrics_complete("test-parallel-003")
|
||||
|
||||
# Verify screens worker was triggered (both complete)
|
||||
mock_trigger.assert_called_once_with("test-parallel-003")
|
||||
|
||||
|
||||
class TestStateTransitionValidation:
|
||||
"""
|
||||
Integration tests for state transition validation.
|
||||
|
||||
Ensures invalid transitions are rejected with clear errors.
|
||||
"""
|
||||
|
||||
def test_valid_transition_succeeds(self, job_manager, mock_firestore_service):
|
||||
"""Test that valid transitions succeed."""
|
||||
job = Job(
|
||||
job_id="test-transition-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job_status.return_value = True
|
||||
|
||||
# PENDING -> DOWNLOADING is valid
|
||||
result = job_manager.transition_to_state(
|
||||
job_id="test-transition-001",
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=10,
|
||||
message="Starting processing"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transitions raise exception by default."""
|
||||
job = Job(
|
||||
job_id="test-transition-002",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
|
||||
# PENDING -> COMPLETE is invalid (skips many steps)
|
||||
with pytest.raises(InvalidStateTransitionError) as exc_info:
|
||||
job_manager.transition_to_state(
|
||||
job_id="test-transition-002",
|
||||
new_status=JobStatus.COMPLETE,
|
||||
)
|
||||
|
||||
assert "pending" in str(exc_info.value).lower()
|
||||
assert "complete" in str(exc_info.value).lower()
|
||||
|
||||
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transitions return False when raise_on_invalid=False."""
|
||||
job = Job(
|
||||
job_id="test-transition-003",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
|
||||
# PENDING -> COMPLETE is invalid, but with raise_on_invalid=False
|
||||
result = job_manager.transition_to_state(
|
||||
job_id="test-transition-003",
|
||||
new_status=JobStatus.COMPLETE,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestAdminResetFlow:
|
||||
"""
|
||||
Integration tests for admin reset functionality.
|
||||
|
||||
Admin resets bypass normal state machine validation but must
|
||||
properly clear state_data flags to avoid inconsistent state.
|
||||
"""
|
||||
|
||||
def test_reset_to_pending_clears_parallel_flags(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that resetting to pending clears audio_complete and lyrics_complete.
|
||||
|
||||
This was a bug where the Start button would reset to pending but
|
||||
leave these flags set, causing inconsistent state.
|
||||
"""
|
||||
# This is tested via the admin endpoint, not directly in JobManager
|
||||
# The test in test_admin_job_reset.py covers this
|
||||
pass
|
||||
|
||||
|
||||
class TestConsistencyWithHealthService:
|
||||
"""
|
||||
Integration tests that verify the job health service correctly
|
||||
identifies inconsistencies in real job flows.
|
||||
"""
|
||||
|
||||
def test_detects_stuck_job_scenario(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that the health service would detect the exact bug that
|
||||
caused jobs 06cfea29 and 984da08b to get stuck.
|
||||
"""
|
||||
from backend.services.job_health_service import check_job_consistency
|
||||
|
||||
# Simulate the stuck job scenario:
|
||||
# - Job has input_media_gcs_path (YouTube download completed)
|
||||
# - But status is still PENDING (transition never happened)
|
||||
stuck_job = Job(
|
||||
job_id="test-stuck-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/downloaded.mp3",
|
||||
state_data={},
|
||||
)
|
||||
|
||||
issues = check_job_consistency(stuck_job)
|
||||
|
||||
# Should detect the pending_with_input_media issue
|
||||
assert len(issues) > 0
|
||||
assert any("pending_with_input_media" in issue for issue in issues)
|
||||
|
||||
def test_detects_flag_status_mismatch(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that the health service detects when flags don't match status.
|
||||
"""
|
||||
from backend.services.job_health_service import check_job_consistency
|
||||
|
||||
# Simulate: audio_complete=True but status still PENDING
|
||||
# (would happen if mark_audio_complete ran but transition didn't)
|
||||
inconsistent_job = Job(
|
||||
job_id="test-inconsistent-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True},
|
||||
)
|
||||
|
||||
issues = check_job_consistency(inconsistent_job)
|
||||
|
||||
# Should detect the audio_complete_status_mismatch issue
|
||||
assert len(issues) > 0
|
||||
assert any("audio_complete_status_mismatch" in issue for issue in issues)
|
||||
@@ -84,7 +84,7 @@ class TestResetJobToPending:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -104,7 +104,7 @@ class TestResetJobToPending:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -128,7 +128,7 @@ class TestResetJobToAwaitingAudioSelection:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -149,7 +149,7 @@ class TestResetJobToAwaitingReview:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -162,6 +162,27 @@ class TestResetJobToAwaitingReview:
|
||||
assert data["new_status"] == "awaiting_review"
|
||||
|
||||
|
||||
class TestResetJobToAwaitingInstrumentalSelection:
|
||||
"""Tests for resetting job to AWAITING_INSTRUMENTAL_SELECTION state."""
|
||||
|
||||
def test_reset_to_awaiting_instrumental_selection(self, client, mock_job):
|
||||
"""Test resetting a job to AWAITING_INSTRUMENTAL_SELECTION state."""
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
"/api/admin/jobs/test-job-123/reset",
|
||||
json={"target_state": "awaiting_instrumental_selection"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["new_status"] == "awaiting_instrumental_selection"
|
||||
|
||||
|
||||
class TestResetJobValidation:
|
||||
"""Tests for reset endpoint validation."""
|
||||
|
||||
@@ -247,7 +268,7 @@ class TestResetJobLogging:
|
||||
patch('backend.api.routes.admin.logger') as mock_logger:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -270,7 +291,7 @@ class TestResetJobTimeline:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -319,7 +340,7 @@ class TestResetJobClearsStateData:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -332,57 +353,12 @@ class TestResetJobClearsStateData:
|
||||
data = response.json()
|
||||
assert "cleared_data" in data or "state_data" in str(data)
|
||||
|
||||
def test_reset_to_pending_clears_parallel_processing_flags(self, client):
|
||||
"""Test that resetting to PENDING clears audio_complete and lyrics_complete.
|
||||
|
||||
This is critical for proper re-processing - without clearing these flags,
|
||||
workers may think audio/lyrics processing is already done.
|
||||
"""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime, UTC
|
||||
|
||||
# Create a job with audio_complete and lyrics_complete set
|
||||
mock_job = Job(
|
||||
job_id="test-job-123",
|
||||
status=JobStatus.COMPLETE,
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={
|
||||
"audio_complete": True,
|
||||
"lyrics_complete": True,
|
||||
"audio_search_results": [{"source": "test"}],
|
||||
}
|
||||
)
|
||||
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
"/api/admin/jobs/test-job-123/reset",
|
||||
json={"target_state": "pending"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify the cleared_data includes the parallel processing flags
|
||||
cleared_keys = data.get("cleared_data", [])
|
||||
assert "audio_complete" in cleared_keys, \
|
||||
f"audio_complete should be cleared. Cleared: {cleared_keys}"
|
||||
assert "lyrics_complete" in cleared_keys, \
|
||||
f"lyrics_complete should be cleared. Cleared: {cleared_keys}"
|
||||
|
||||
def test_reset_to_awaiting_review_preserves_audio_data(self, client, mock_job):
|
||||
"""Test that resetting to AWAITING_REVIEW preserves audio stems."""
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -392,15 +368,12 @@ class TestResetJobClearsStateData:
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_rejects_deprecated_awaiting_instrumental_selection(self, client, mock_job):
|
||||
"""Test that awaiting_instrumental_selection is no longer a valid reset target.
|
||||
|
||||
This state was deprecated in Jan 2026 when lyrics review and instrumental
|
||||
selection were combined into a single review step.
|
||||
"""
|
||||
def test_reset_to_instrumental_preserves_review_data(self, client, mock_job):
|
||||
"""Test that resetting to AWAITING_INSTRUMENTAL_SELECTION preserves review data."""
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -408,9 +381,7 @@ class TestResetJobClearsStateData:
|
||||
json={"target_state": "awaiting_instrumental_selection"},
|
||||
)
|
||||
|
||||
# Should be rejected as invalid
|
||||
assert response.status_code == 400
|
||||
assert "invalid" in response.json()["detail"].lower()
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestResetJobToInstrumentalSelected:
|
||||
@@ -425,7 +396,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
# Mock worker service with async method
|
||||
@@ -450,7 +421,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
# Mock worker service
|
||||
@@ -478,7 +449,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
# Mock worker service failure
|
||||
@@ -515,7 +486,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -550,7 +521,7 @@ class TestResetJobClearsErrorState:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -654,7 +625,7 @@ class TestResetIncludesEncodingProgress:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -679,7 +650,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -704,7 +675,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -730,7 +701,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -754,7 +725,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -962,57 +933,3 @@ class TestTriggerWorkerEndpoint:
|
||||
app.dependency_overrides[require_admin] = original_override
|
||||
else:
|
||||
app.dependency_overrides[require_admin] = get_mock_admin
|
||||
|
||||
|
||||
class TestResetEndpointWithRealJobData:
|
||||
"""
|
||||
Tests using real production job data (job 984da08b) as fixture.
|
||||
|
||||
This validates the admin reset endpoint works correctly with realistic
|
||||
state_data from a production job that needed manual admin reset.
|
||||
"""
|
||||
|
||||
# Real state_data from production job 984da08b
|
||||
REAL_JOB_STATE_DATA = {
|
||||
"backing_vocals_analysis": {"has_audible_content": True},
|
||||
"video_progress": {"stage": "running"}, # Stale progress
|
||||
"lyrics_complete": True,
|
||||
"audio_complete": True
|
||||
}
|
||||
|
||||
def test_reset_endpoint_returns_200_with_none_return_value(self, client):
|
||||
"""
|
||||
Test that reset endpoint returns 200 (not 500) when update_job
|
||||
returns None, matching the real API behavior.
|
||||
|
||||
Before fix: endpoint checked 'if not success:' which was always True
|
||||
After fix: endpoint doesn't check return value
|
||||
|
||||
Uses real production job 984da08b data as fixture.
|
||||
"""
|
||||
# Mock job with real production data
|
||||
mock_job = Mock(spec=Job)
|
||||
mock_job.job_id = "984da08b"
|
||||
mock_job.status = JobStatus.COMPLETE
|
||||
mock_job.artist = "Mark Mallman"
|
||||
mock_job.title = "Minneapolis"
|
||||
mock_job.theme_id = "nomad"
|
||||
mock_job.user_email = "madeforyou@nomadkaraoke.com"
|
||||
mock_job.file_urls = {"stems": {}, "lyrics": {}}
|
||||
mock_job.state_data = self.REAL_JOB_STATE_DATA.copy()
|
||||
mock_job.timeline = []
|
||||
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm:
|
||||
mock_jm.return_value.get_job.return_value = mock_job
|
||||
# KEY: return None (matching real API contract - not boolean!)
|
||||
mock_jm.return_value.update_job.return_value = None
|
||||
|
||||
response = client.post(
|
||||
"/api/admin/jobs/984da08b/reset",
|
||||
json={"target_state": "pending"},
|
||||
)
|
||||
|
||||
# This is the bug fix - before it was 500, now 200
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "success"
|
||||
assert response.json()["new_status"] == "pending"
|
||||
@@ -7,7 +7,7 @@ route logic without hitting real cloud services.
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch, Mock
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from fastapi.testclient import TestClient
|
||||
from io import BytesIO
|
||||
|
||||
@@ -131,91 +131,13 @@ class TestHealthRoutes:
|
||||
assert data["queue_length"] == 5
|
||||
|
||||
|
||||
class TestJobConsistencyEndpoint:
|
||||
"""Tests for the /health/job-consistency endpoint."""
|
||||
|
||||
@pytest.fixture
|
||||
def client(self):
|
||||
"""Create test client."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
from backend.main import app
|
||||
return TestClient(app)
|
||||
|
||||
def test_job_consistency_returns_empty_when_no_jobs(self, client):
|
||||
"""Test endpoint returns empty results when no jobs exist."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.list_jobs.return_value = []
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_checked"] == 0
|
||||
assert data["inconsistent_count"] == 0
|
||||
assert data["inconsistent_jobs"] == []
|
||||
|
||||
def test_job_consistency_detects_issues(self, client):
|
||||
"""Test endpoint detects inconsistent jobs."""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime, UTC
|
||||
|
||||
stuck_job = Job(
|
||||
job_id="stuck-123",
|
||||
artist="Test",
|
||||
title="Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True},
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.list_jobs.return_value = [stuck_job]
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_checked"] == 1
|
||||
assert data["inconsistent_count"] == 1
|
||||
assert len(data["inconsistent_jobs"]) == 1
|
||||
assert data["inconsistent_jobs"][0]["job_id"] == "stuck-123"
|
||||
|
||||
def test_job_consistency_filters_by_status(self, client):
|
||||
"""Test endpoint filters by status parameter."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.list_jobs.return_value = []
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency?status=pending")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status_filter"] == "pending"
|
||||
|
||||
def test_job_consistency_rejects_invalid_status(self, client):
|
||||
"""Test endpoint rejects invalid status parameter."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency?status=invalid_status")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
assert "Invalid status" in data["error"]
|
||||
|
||||
|
||||
class TestJobRoutes:
|
||||
"""Tests for jobs.py routes.
|
||||
|
||||
|
||||
Note: These tests verify the route module structure.
|
||||
Full integration tests are in test_api_integration.py.
|
||||
"""
|
||||
|
||||
|
||||
def test_jobs_router_exists(self):
|
||||
"""Test jobs router can be imported."""
|
||||
from backend.api.routes import jobs
|
||||
|
||||
@@ -269,12 +269,10 @@ class TestUrlJobWorkerSequencing:
|
||||
async def test_parallel_worker_triggers_both_workers(self):
|
||||
"""
|
||||
_trigger_workers_parallel should trigger both workers (for uploaded files).
|
||||
Note: This function now lives in jobs.py (direct job creation endpoint).
|
||||
Most code paths use job_manager.start_job_processing() instead.
|
||||
"""
|
||||
from backend.api.routes.jobs import _trigger_workers_parallel
|
||||
from backend.api.routes.file_upload import _trigger_workers_parallel
|
||||
|
||||
with patch('backend.api.routes.jobs.worker_service') as mock_ws:
|
||||
with patch('backend.api.routes.file_upload.worker_service') as mock_ws:
|
||||
mock_ws.trigger_audio_worker = AsyncMock()
|
||||
mock_ws.trigger_lyrics_worker = AsyncMock()
|
||||
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
"""
|
||||
Tests for the job health service.
|
||||
|
||||
Tests consistency checks and worker validation functions.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_health_service import (
|
||||
check_job_consistency,
|
||||
check_job_consistency_detailed,
|
||||
validate_worker_can_run,
|
||||
get_worker_valid_statuses,
|
||||
)
|
||||
|
||||
|
||||
def create_test_job(
|
||||
job_id: str = "test-job-123",
|
||||
status: JobStatus = JobStatus.PENDING,
|
||||
state_data: dict = None,
|
||||
input_media_gcs_path: str = None,
|
||||
) -> Job:
|
||||
"""Helper to create test Job objects."""
|
||||
return Job(
|
||||
job_id=job_id,
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=status,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data=state_data or {},
|
||||
input_media_gcs_path=input_media_gcs_path,
|
||||
)
|
||||
|
||||
|
||||
class TestCheckJobConsistency:
|
||||
"""Tests for check_job_consistency function."""
|
||||
|
||||
def test_healthy_job_has_no_issues(self):
|
||||
"""Test that a properly configured job has no issues."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data={},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
def test_detects_audio_complete_with_pending_status(self):
|
||||
"""Test detection of audio_complete flag with pending status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={'audio_complete': True},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "audio_complete_status_mismatch" in issues[0]
|
||||
assert "audio_complete=True but status=pending" in issues[0]
|
||||
|
||||
def test_detects_lyrics_complete_with_pending_status(self):
|
||||
"""Test detection of lyrics_complete flag with pending status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={'lyrics_complete': True},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "lyrics_complete_status_mismatch" in issues[0]
|
||||
|
||||
def test_detects_screens_complete_with_wrong_status(self):
|
||||
"""Test detection of screens complete but status not in post-screens states."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data={'screens_progress': {'stage': 'complete'}},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "screens_complete_status_mismatch" in issues[0]
|
||||
|
||||
def test_screens_complete_valid_with_awaiting_review(self):
|
||||
"""Test that screens complete is valid with AWAITING_REVIEW status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.AWAITING_REVIEW,
|
||||
state_data={'screens_progress': {'stage': 'complete'}},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
def test_screens_complete_valid_with_complete_status(self):
|
||||
"""Test that screens complete is valid with COMPLETE status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.COMPLETE,
|
||||
state_data={'screens_progress': {'stage': 'complete'}},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
def test_detects_pending_with_input_media_path(self):
|
||||
"""Test detection of job stuck at pending with input_media_gcs_path."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "pending_with_input_media" in issues[0]
|
||||
|
||||
def test_multiple_issues_detected(self):
|
||||
"""Test that multiple issues are detected simultaneously."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={
|
||||
'audio_complete': True,
|
||||
'lyrics_complete': True,
|
||||
},
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 3
|
||||
|
||||
def test_handles_invalid_status_string(self):
|
||||
"""Test handling of invalid status string."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
job.status = "invalid_status" # type: ignore
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "invalid_status" in issues[0]
|
||||
|
||||
def test_handles_empty_state_data(self):
|
||||
"""Test handling of empty or None state_data."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data=None,
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
|
||||
class TestCheckJobConsistencyDetailed:
|
||||
"""Tests for check_job_consistency_detailed function."""
|
||||
|
||||
def test_returns_detailed_info(self):
|
||||
"""Test that detailed info is returned."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={'audio_complete': True},
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
result = check_job_consistency_detailed(job)
|
||||
|
||||
assert result['job_id'] == "test-job-123"
|
||||
assert result['status'] == "pending"
|
||||
assert len(result['issues']) == 2
|
||||
assert result['state_data_summary']['audio_complete'] is True
|
||||
assert result['state_data_summary']['lyrics_complete'] is False
|
||||
assert result['state_data_summary']['has_input_media_gcs_path'] is True
|
||||
assert result['is_healthy'] is False
|
||||
|
||||
def test_healthy_job_returns_is_healthy_true(self):
|
||||
"""Test that healthy job returns is_healthy=True."""
|
||||
job = create_test_job(status=JobStatus.DOWNLOADING)
|
||||
result = check_job_consistency_detailed(job)
|
||||
assert result['is_healthy'] is True
|
||||
assert result['issues'] == []
|
||||
|
||||
|
||||
class TestValidateWorkerCanRun:
|
||||
"""Tests for validate_worker_can_run function."""
|
||||
|
||||
def test_audio_worker_valid_with_downloading_status(self):
|
||||
"""Test audio worker is valid with DOWNLOADING status."""
|
||||
job = create_test_job(status=JobStatus.DOWNLOADING)
|
||||
error = validate_worker_can_run("audio_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_audio_worker_invalid_with_pending_status(self):
|
||||
"""Test audio worker is invalid with PENDING status."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
error = validate_worker_can_run("audio_worker", job)
|
||||
assert error is not None
|
||||
assert "audio_worker called but job status is pending" in error
|
||||
|
||||
def test_lyrics_worker_valid_with_downloading_status(self):
|
||||
"""Test lyrics worker is valid with DOWNLOADING status."""
|
||||
job = create_test_job(status=JobStatus.DOWNLOADING)
|
||||
error = validate_worker_can_run("lyrics_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_screens_worker_valid_with_audio_complete_status(self):
|
||||
"""Test screens worker is valid with AUDIO_COMPLETE status."""
|
||||
job = create_test_job(status=JobStatus.AUDIO_COMPLETE)
|
||||
error = validate_worker_can_run("screens_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_screens_worker_valid_with_lyrics_complete_status(self):
|
||||
"""Test screens worker is valid with LYRICS_COMPLETE status."""
|
||||
job = create_test_job(status=JobStatus.LYRICS_COMPLETE)
|
||||
error = validate_worker_can_run("screens_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_screens_worker_invalid_with_awaiting_review_status(self):
|
||||
"""Test screens worker is invalid with AWAITING_REVIEW status (too late)."""
|
||||
job = create_test_job(status=JobStatus.AWAITING_REVIEW)
|
||||
error = validate_worker_can_run("screens_worker", job)
|
||||
assert error is not None
|
||||
assert "screens_worker called but job status is awaiting_review" in error
|
||||
|
||||
def test_video_worker_valid_with_instrumental_selected_status(self):
|
||||
"""Test video worker is valid with INSTRUMENTAL_SELECTED status."""
|
||||
job = create_test_job(status=JobStatus.INSTRUMENTAL_SELECTED)
|
||||
error = validate_worker_can_run("video_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_video_worker_invalid_with_pending_status(self):
|
||||
"""Test video worker is invalid with PENDING status."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
error = validate_worker_can_run("video_worker", job)
|
||||
assert error is not None
|
||||
assert "video_worker called but job status is pending" in error
|
||||
|
||||
def test_render_video_worker_valid_with_review_complete_status(self):
|
||||
"""Test render video worker is valid with REVIEW_COMPLETE status."""
|
||||
job = create_test_job(status=JobStatus.REVIEW_COMPLETE)
|
||||
error = validate_worker_can_run("render_video_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_unknown_worker_returns_none(self):
|
||||
"""Test that unknown worker names return None (no validation)."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
error = validate_worker_can_run("unknown_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_handles_invalid_status_string(self):
|
||||
"""Test handling of invalid status string."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
job.status = "invalid_status" # type: ignore
|
||||
error = validate_worker_can_run("audio_worker", job)
|
||||
assert error is not None
|
||||
assert "Invalid job status" in error
|
||||
|
||||
|
||||
class TestGetWorkerValidStatuses:
|
||||
"""Tests for get_worker_valid_statuses function."""
|
||||
|
||||
def test_returns_dict_with_all_workers(self):
|
||||
"""Test that all known workers are included."""
|
||||
statuses = get_worker_valid_statuses()
|
||||
assert "audio_worker" in statuses
|
||||
assert "lyrics_worker" in statuses
|
||||
assert "screens_worker" in statuses
|
||||
assert "video_worker" in statuses
|
||||
assert "render_video_worker" in statuses
|
||||
|
||||
def test_audio_worker_includes_downloading(self):
|
||||
"""Test that audio worker includes DOWNLOADING status."""
|
||||
statuses = get_worker_valid_statuses()
|
||||
assert JobStatus.DOWNLOADING in statuses["audio_worker"]
|
||||
|
||||
def test_lyrics_worker_includes_downloading(self):
|
||||
"""Test that lyrics worker includes DOWNLOADING status."""
|
||||
statuses = get_worker_valid_statuses()
|
||||
assert JobStatus.DOWNLOADING in statuses["lyrics_worker"]
|
||||
@@ -5,7 +5,7 @@ Tests the job lifecycle management without requiring actual Firestore connection
|
||||
Uses mocking to isolate the business logic.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch, AsyncMock
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
from datetime import datetime, UTC
|
||||
|
||||
# Mock Firestore before importing JobManager
|
||||
@@ -14,7 +14,6 @@ sys.modules['google.cloud.firestore'] = MagicMock()
|
||||
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.models.job import Job, JobCreate, JobStatus
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -569,288 +568,6 @@ class TestDeleteStateDataKeys:
|
||||
assert result == []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# InvalidStateTransitionError Tests
|
||||
# =============================================================================
|
||||
|
||||
class TestInvalidStateTransitionError:
|
||||
"""Tests for InvalidStateTransitionError exception."""
|
||||
|
||||
def test_exception_stores_all_attributes(self):
|
||||
"""Test exception stores job_id, from_status, to_status, and valid_transitions."""
|
||||
error = InvalidStateTransitionError(
|
||||
message="Test error",
|
||||
job_id="test-123",
|
||||
from_status="pending",
|
||||
to_status="generating_screens",
|
||||
valid_transitions=["downloading", "searching_audio", "failed"]
|
||||
)
|
||||
|
||||
assert error.message == "Test error"
|
||||
assert error.job_id == "test-123"
|
||||
assert error.from_status == "pending"
|
||||
assert error.to_status == "generating_screens"
|
||||
assert error.valid_transitions == ["downloading", "searching_audio", "failed"]
|
||||
assert str(error) == "Test error"
|
||||
|
||||
def test_exception_defaults(self):
|
||||
"""Test exception has sensible defaults."""
|
||||
error = InvalidStateTransitionError(message="Minimal error")
|
||||
|
||||
assert error.message == "Minimal error"
|
||||
assert error.job_id == ""
|
||||
assert error.from_status == ""
|
||||
assert error.to_status == ""
|
||||
assert error.valid_transitions == []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# State Transition Validation Tests (raise_on_invalid behavior)
|
||||
# =============================================================================
|
||||
|
||||
class TestValidateStateTransitionRaisesBehavior:
|
||||
"""Tests for validate_state_transition with raise_on_invalid parameter."""
|
||||
|
||||
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition raises InvalidStateTransitionError by default."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING cannot transition directly to GENERATING_SCREENS
|
||||
with pytest.raises(InvalidStateTransitionError) as exc_info:
|
||||
job_manager.validate_state_transition("test123", JobStatus.GENERATING_SCREENS)
|
||||
|
||||
error = exc_info.value
|
||||
assert error.job_id == "test123"
|
||||
assert error.from_status == JobStatus.PENDING
|
||||
assert error.to_status == "generating_screens"
|
||||
assert "downloading" in error.valid_transitions
|
||||
|
||||
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition returns False when raise_on_invalid=False."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING cannot transition directly to GENERATING_SCREENS
|
||||
result = job_manager.validate_state_transition(
|
||||
"test123",
|
||||
JobStatus.GENERATING_SCREENS,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_valid_transition_returns_true(self, job_manager, mock_firestore_service):
|
||||
"""Test that valid transition returns True."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING can transition to DOWNLOADING
|
||||
result = job_manager.validate_state_transition("test123", JobStatus.DOWNLOADING)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_job_not_found_raises_value_error(self, job_manager, mock_firestore_service):
|
||||
"""Test that missing job raises ValueError when raise_on_invalid=True."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
with pytest.raises(ValueError, match="Job nonexistent not found"):
|
||||
job_manager.validate_state_transition("nonexistent", JobStatus.DOWNLOADING)
|
||||
|
||||
def test_job_not_found_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that missing job returns False when raise_on_invalid=False."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
result = job_manager.validate_state_transition(
|
||||
"nonexistent",
|
||||
JobStatus.DOWNLOADING,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestTransitionToStateRaisesBehavior:
|
||||
"""Tests for transition_to_state with raise_on_invalid parameter."""
|
||||
|
||||
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition raises InvalidStateTransitionError by default."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING cannot transition directly to AWAITING_REVIEW
|
||||
with pytest.raises(InvalidStateTransitionError) as exc_info:
|
||||
job_manager.transition_to_state("test123", JobStatus.AWAITING_REVIEW)
|
||||
|
||||
error = exc_info.value
|
||||
assert error.job_id == "test123"
|
||||
|
||||
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition returns False when raise_on_invalid=False."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
result = job_manager.transition_to_state(
|
||||
"test123",
|
||||
JobStatus.AWAITING_REVIEW,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
# Verify update was NOT called since transition was invalid
|
||||
mock_firestore_service.update_job.assert_not_called()
|
||||
|
||||
def test_valid_transition_updates_job(self, job_manager, mock_firestore_service):
|
||||
"""Test that valid transition updates the job."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
result = job_manager.transition_to_state(
|
||||
"test123",
|
||||
JobStatus.DOWNLOADING,
|
||||
progress=15,
|
||||
message="Starting processing"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
# transition_to_state uses update_job_status internally
|
||||
mock_firestore_service.update_job_status.assert_called()
|
||||
call_kwargs = mock_firestore_service.update_job_status.call_args.kwargs
|
||||
assert call_kwargs['job_id'] == "test123"
|
||||
assert call_kwargs['status'] == JobStatus.DOWNLOADING
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# start_job_processing Tests
|
||||
# =============================================================================
|
||||
|
||||
class TestStartJobProcessing:
|
||||
"""Tests for start_job_processing method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_if_job_not_found(self, job_manager, mock_firestore_service):
|
||||
"""Test raises ValueError if job doesn't exist."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
with pytest.raises(ValueError, match="Job missing-job not found"):
|
||||
await job_manager.start_job_processing("missing-job")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_if_missing_input_media_path(self, job_manager, mock_firestore_service):
|
||||
"""Test raises ValueError if job has no input_media_gcs_path."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path=None # Missing
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="missing input_media_gcs_path"):
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_if_invalid_state_transition(self, job_manager, mock_firestore_service):
|
||||
"""Test raises InvalidStateTransitionError if job can't transition to DOWNLOADING."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.AWAITING_REVIEW, # Can't go back to DOWNLOADING
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/audio.mp3"
|
||||
)
|
||||
|
||||
with pytest.raises(InvalidStateTransitionError):
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transitions_and_triggers_workers(self, job_manager, mock_firestore_service):
|
||||
"""Test successful flow: transitions to DOWNLOADING and triggers workers."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/audio.mp3"
|
||||
)
|
||||
|
||||
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
|
||||
mock_worker = Mock()
|
||||
mock_worker.trigger_audio_worker = AsyncMock()
|
||||
mock_worker.trigger_lyrics_worker = AsyncMock()
|
||||
mock_get_worker.return_value = mock_worker
|
||||
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
# Verify transition was called (uses update_job_status internally)
|
||||
mock_firestore_service.update_job_status.assert_called()
|
||||
call_kwargs = mock_firestore_service.update_job_status.call_args.kwargs
|
||||
assert call_kwargs['status'] == JobStatus.DOWNLOADING
|
||||
|
||||
# Verify both workers were triggered
|
||||
mock_worker.trigger_audio_worker.assert_called_once_with("test123")
|
||||
mock_worker.trigger_lyrics_worker.assert_called_once_with("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workers_triggered_in_parallel(self, job_manager, mock_firestore_service):
|
||||
"""Test that audio and lyrics workers are triggered concurrently."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/audio.mp3"
|
||||
)
|
||||
|
||||
call_order = []
|
||||
|
||||
async def track_audio(*args):
|
||||
call_order.append(('audio_start', datetime.now()))
|
||||
# Simulate some work
|
||||
call_order.append(('audio_end', datetime.now()))
|
||||
|
||||
async def track_lyrics(*args):
|
||||
call_order.append(('lyrics_start', datetime.now()))
|
||||
# Simulate some work
|
||||
call_order.append(('lyrics_end', datetime.now()))
|
||||
|
||||
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
|
||||
mock_worker = Mock()
|
||||
mock_worker.trigger_audio_worker = AsyncMock(side_effect=track_audio)
|
||||
mock_worker.trigger_lyrics_worker = AsyncMock(side_effect=track_lyrics)
|
||||
mock_get_worker.return_value = mock_worker
|
||||
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
# Both workers should have been called
|
||||
assert any('audio_start' in str(c) for c in call_order)
|
||||
assert any('lyrics_start' in str(c) for c in call_order)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -1332,13 +1332,10 @@ class TestHandleMadeForYouOrder:
|
||||
mock_theme_service
|
||||
):
|
||||
"""
|
||||
Orders with YouTube URL should download audio then start processing (no search).
|
||||
Orders with YouTube URL should download audio then trigger workers (no search).
|
||||
|
||||
Note: The download step was added in Jan 2026 to fix job 811ec4e5 failure.
|
||||
See docs/LESSONS-LEARNED.md "YouTube URL Downloads Must Happen Before Workers".
|
||||
|
||||
The handler now uses job_manager.start_job_processing() which handles
|
||||
transitioning to DOWNLOADING and triggering workers.
|
||||
"""
|
||||
from backend.api.routes.users import _handle_made_for_you_order
|
||||
|
||||
@@ -1351,16 +1348,18 @@ class TestHandleMadeForYouOrder:
|
||||
"youtube_url": "https://youtube.com/watch?v=abc123",
|
||||
}
|
||||
|
||||
mock_worker_service = MagicMock()
|
||||
mock_worker_service.trigger_audio_worker = AsyncMock()
|
||||
mock_worker_service.trigger_lyrics_worker = AsyncMock()
|
||||
|
||||
# Mock YouTube download service to return a GCS path
|
||||
mock_youtube_service = MagicMock()
|
||||
mock_youtube_service.download = AsyncMock(
|
||||
return_value="uploads/test-job-123/audio/test.webm"
|
||||
)
|
||||
|
||||
# Mock start_job_processing as an async method
|
||||
mock_job_manager.start_job_processing = AsyncMock()
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service), \
|
||||
patch('backend.services.audio_search_service.get_audio_search_service') as mock_get_audio, \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
patch('backend.api.routes.users.get_theme_service', return_value=mock_theme_service), \
|
||||
@@ -1381,8 +1380,9 @@ class TestHandleMadeForYouOrder:
|
||||
# Audio should be downloaded first
|
||||
mock_youtube_service.download.assert_called_once()
|
||||
|
||||
# start_job_processing should be called (handles transition + workers)
|
||||
mock_job_manager.start_job_processing.assert_called_once_with("test-job-123")
|
||||
# Workers should be triggered after download
|
||||
mock_worker_service.trigger_audio_worker.assert_called_once()
|
||||
mock_worker_service.trigger_lyrics_worker.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stripe_session_marked_processed(
|
||||
@@ -1588,69 +1588,6 @@ class TestMadeForYouYouTubeDownloadContract:
|
||||
f"input_media_gcs_path must be set BEFORE workers are triggered. " \
|
||||
f"Call order was: {call_order}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_youtube_url_calls_start_job_processing_after_download(
|
||||
self, mock_job_manager, mock_email_service, mock_user_service,
|
||||
mock_theme_service, youtube_order_metadata
|
||||
):
|
||||
"""
|
||||
CRITICAL: start_job_processing must be called AFTER download completes.
|
||||
|
||||
The handler uses job_manager.start_job_processing() which:
|
||||
1. Transitions job to DOWNLOADING
|
||||
2. Triggers audio and lyrics workers
|
||||
|
||||
Without start_job_processing, workers would run but all status updates
|
||||
would fail silently because transitions from PENDING to GENERATING_SCREENS
|
||||
or AWAITING_REVIEW are invalid.
|
||||
|
||||
This bug caused made-for-you orders to get stuck at PENDING status even though
|
||||
all processing completed successfully. See job IDs 06cfea29 and 984da08b.
|
||||
"""
|
||||
from backend.api.routes.users import _handle_made_for_you_order
|
||||
|
||||
# Track call order to verify sequence
|
||||
call_order = []
|
||||
|
||||
# Mock YouTubeDownloadService to return a GCS path
|
||||
mock_youtube_service = MagicMock()
|
||||
mock_youtube_service.download = AsyncMock(
|
||||
return_value="uploads/test-job-123/audio/test.webm",
|
||||
side_effect=lambda **kwargs: (
|
||||
call_order.append('youtube_download'),
|
||||
"uploads/test-job-123/audio/test.webm"
|
||||
)[1]
|
||||
)
|
||||
|
||||
# Track when start_job_processing is called
|
||||
mock_job_manager.start_job_processing = AsyncMock(
|
||||
side_effect=lambda job_id: call_order.append('start_job_processing')
|
||||
)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
patch('backend.api.routes.users.get_theme_service', return_value=mock_theme_service), \
|
||||
patch('backend.api.routes.users.get_youtube_download_service', return_value=mock_youtube_service):
|
||||
|
||||
await _handle_made_for_you_order(
|
||||
session_id="sess_123",
|
||||
metadata=youtube_order_metadata,
|
||||
user_service=mock_user_service,
|
||||
email_service=mock_email_service,
|
||||
)
|
||||
|
||||
# CRITICAL: start_job_processing must be called
|
||||
assert 'start_job_processing' in call_order, \
|
||||
f"start_job_processing must be called. Call order was: {call_order}"
|
||||
|
||||
# CRITICAL: start_job_processing must happen AFTER download
|
||||
download_index = call_order.index('youtube_download')
|
||||
processing_index = call_order.index('start_job_processing')
|
||||
|
||||
assert download_index < processing_index, \
|
||||
f"YouTube download must complete BEFORE start_job_processing. " \
|
||||
f"Call order was: {call_order}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_youtube_download_called_with_correct_params(
|
||||
self, mock_job_manager, mock_email_service, mock_user_service,
|
||||
|
||||
@@ -24,7 +24,6 @@ from pathlib import Path
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.config import get_settings
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.workers.registry import worker_registry
|
||||
@@ -289,15 +288,7 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
# Validate job status is appropriate for audio worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("audio_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"[job:{job_id}] {status_error}")
|
||||
job_log.warning(status_error)
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
|
||||
# Create temporary working directory
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_{job_id}_")
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
|
||||
@@ -32,7 +32,6 @@ from backend.models.job import JobStatus
|
||||
from karaoke_gen.utils import sanitize_filename
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.services.lyrics_cache_service import LyricsCacheService
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.workers.style_helper import load_style_config
|
||||
@@ -191,15 +190,7 @@ async def process_lyrics_transcription(job_id: str) -> bool:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
# Validate job status is appropriate for lyrics worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("lyrics_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"[job:{job_id}] {status_error}")
|
||||
job_log.warning(status_error)
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
|
||||
add_span_attribute("artist", job.artist)
|
||||
add_span_attribute("title", job.title)
|
||||
|
||||
|
||||
@@ -36,7 +36,6 @@ from dataclasses import dataclass
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.config import get_settings
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
@@ -99,15 +98,7 @@ async def process_render_video(job_id: str) -> bool:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
# Validate job status is appropriate for render video worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("render_video_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"[job:{job_id}] {status_error}")
|
||||
job_log.warning(status_error)
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
|
||||
job_log.info(f"Starting video render for {job.artist} - {job.title}")
|
||||
logger.info(f"[job:{job_id}] Starting video render (post-review)")
|
||||
|
||||
|
||||
@@ -33,7 +33,6 @@ from pathlib import Path
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.config import get_settings
|
||||
from backend.workers.style_helper import load_style_config, StyleConfig
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
@@ -239,14 +238,6 @@ def _validate_prerequisites(job) -> bool:
|
||||
Returns:
|
||||
True if prerequisites met, False otherwise
|
||||
"""
|
||||
# Validate job status is appropriate for screens worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("screens_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"Job {job.job_id}: {status_error}")
|
||||
# Log warning but don't fail - status validation is a safety net,
|
||||
# the state_data flags below are the authoritative check
|
||||
|
||||
# SAFETY NET: Enforce theme requirement at processing time
|
||||
# This catches any jobs that somehow bypassed JobManager.create_job() validation
|
||||
if not job.theme_id:
|
||||
@@ -271,7 +262,7 @@ def _validate_prerequisites(job) -> bool:
|
||||
if not job.artist or not job.title:
|
||||
logger.error(f"Job {job.job_id}: Missing artist or title")
|
||||
return False
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ from pathlib import Path
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.services.rclone_service import get_rclone_service
|
||||
from backend.services.youtube_service import get_youtube_service
|
||||
from backend.services.encoding_service import get_encoding_service
|
||||
@@ -895,13 +894,6 @@ async def _handle_native_distribution(
|
||||
|
||||
def _validate_prerequisites(job) -> bool:
|
||||
"""Validate that all prerequisites are met for video generation."""
|
||||
# Validate job status is appropriate for video worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("video_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"Job {job.job_id}: {status_error}")
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
# Check instrumental selection
|
||||
instrumental_selection = job.state_data.get('instrumental_selection')
|
||||
if not instrumental_selection:
|
||||
|
||||
+5
-124
@@ -958,134 +958,15 @@ Response:
|
||||
}
|
||||
```
|
||||
|
||||
#### Regenerate Screens
|
||||
|
||||
```http
|
||||
POST /api/admin/jobs/{job_id}/regenerate-screens
|
||||
Authorization: Bearer ADMIN_TOKEN
|
||||
```
|
||||
|
||||
Regenerates title and end screen videos using the **current** artist/title metadata. Use this after editing artist or title fields to update the screens without full reprocessing.
|
||||
|
||||
Requirements:
|
||||
- Job must have audio and lyrics processing complete
|
||||
- Job must be in an allowed state: `complete`, `failed`, `awaiting_review`, `awaiting_instrumental_selection`, `instrumental_selected`, `prep_complete`
|
||||
|
||||
The endpoint:
|
||||
1. Deletes existing screen files from GCS
|
||||
2. Triggers the screens worker to regenerate with current metadata
|
||||
3. Returns immediately (does not wait for completion)
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"job_id": "abc123",
|
||||
"message": "Screens regeneration started",
|
||||
"previous_screens_deleted": true,
|
||||
"worker_triggered": true,
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
Screen generation takes 30-60 seconds. Monitor progress via job timeline or logs.
|
||||
|
||||
#### Restart Job
|
||||
|
||||
```http
|
||||
POST /api/admin/jobs/{job_id}/restart
|
||||
Authorization: Bearer ADMIN_TOKEN
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"preserve_audio_stems": true,
|
||||
"delete_outputs": true
|
||||
}
|
||||
```
|
||||
|
||||
Fully restarts a job from the beginning. Unlike the reset endpoint (which just changes state), restart actually **triggers workers** to begin processing.
|
||||
|
||||
Options:
|
||||
- `preserve_audio_stems` (default: false) - If true, keeps existing audio separation and lyrics. Only regenerates screens and video with current metadata. Good for fixing title/artist typos.
|
||||
- `delete_outputs` (default: true) - Delete existing output files from GCS.
|
||||
|
||||
Behavior by job type:
|
||||
- **YouTube URL jobs**: Re-downloads audio, then triggers audio/lyrics workers
|
||||
- **Audio search jobs**: Transitions to `awaiting_audio_selection` for admin to re-select
|
||||
- **File upload jobs**: Triggers audio/lyrics workers directly
|
||||
|
||||
When `preserve_audio_stems=true`:
|
||||
- Validates audio and lyrics processing completed previously
|
||||
- Clears screens/video/encoding state only
|
||||
- Triggers screens worker immediately
|
||||
|
||||
Allowed states: `pending`, `complete`, `failed`, `awaiting_review`, `awaiting_audio_selection`, `awaiting_instrumental_selection`, `instrumental_selected`, `prep_complete`
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"job_id": "abc123",
|
||||
"message": "Job restarted from complete to downloading",
|
||||
"previous_status": "complete",
|
||||
"new_status": "downloading",
|
||||
"cleared_data": ["audio_complete", "lyrics_complete", "screens_progress", "..."],
|
||||
"deleted_gcs_paths": ["jobs/abc123/screens/title.mov", "..."],
|
||||
"workers_triggered": ["download", "audio", "lyrics"],
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
#### Override Audio Source
|
||||
|
||||
```http
|
||||
POST /api/admin/jobs/{job_id}/override-audio-source
|
||||
Authorization: Bearer ADMIN_TOKEN
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"source_type": "audio_search"
|
||||
}
|
||||
```
|
||||
|
||||
Switches a job's audio source from YouTube URL to audio search mode. Use this when a Made-For-You order was submitted with a YouTube URL but you want to find higher quality audio.
|
||||
|
||||
Currently only supports switching to `audio_search` mode, which:
|
||||
1. Clears existing audio-related state (URL, downloaded file, stems, transcription)
|
||||
2. Performs an audio search using the job's artist/title
|
||||
3. Stores search results in state_data
|
||||
4. Transitions job to `awaiting_audio_selection` with results ready
|
||||
5. Admin can then select an audio source from the search results in the UI
|
||||
|
||||
Allowed states: `pending`, `complete`, `failed`, `awaiting_review`, `awaiting_audio_selection`, `instrumental_selected`, `prep_complete`
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"job_id": "abc123",
|
||||
"message": "Found 5 audio sources - select one in the admin panel.",
|
||||
"previous_source": "youtube",
|
||||
"new_source": "audio_search",
|
||||
"cleared_data": ["audio_complete", "lyrics_complete", "url", "input_media_gcs_path", "..."],
|
||||
"new_status": "awaiting_audio_selection",
|
||||
"search_results_count": 5,
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
If no audio sources found:
|
||||
If triggering fails:
|
||||
```json
|
||||
{
|
||||
"status": "error",
|
||||
"job_id": "abc123",
|
||||
"message": "No audio sources found for: Artist - Title",
|
||||
"previous_source": "youtube",
|
||||
"new_source": "audio_search",
|
||||
"cleared_data": ["..."],
|
||||
"new_status": "failed",
|
||||
"search_results_count": null,
|
||||
"error": "No audio sources found for: Artist - Title"
|
||||
"worker_type": "video",
|
||||
"triggered": false,
|
||||
"message": "Failed to trigger video worker for job abc123",
|
||||
"error": "Job is in wrong state for video processing"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -102,8 +102,6 @@
|
||||
|
||||
**Note**: `awaiting_instrumental_selection` exists for backwards compatibility with historical jobs but is no longer used. Instrumental selection is now part of the combined review (`awaiting_review` → `in_review` → `review_complete`).
|
||||
|
||||
**State Machine Robustness (Feb 2026)**: State transitions are enforced by `STATE_TRANSITIONS` in `models/job.py`. Invalid transitions raise `InvalidStateTransitionError` by default. Use `scripts/generate_state_diagram.py` to generate a Mermaid diagram of valid transitions. The `/health/job-consistency` endpoint detects jobs stuck in invalid states. See `docs/archive/2026-02-02-state-machine-robustness-plan.md` for implementation details.
|
||||
|
||||
## GCS File Structure
|
||||
|
||||
```text
|
||||
|
||||
@@ -45,19 +45,6 @@ When UI removes options, backend must apply sensible defaults. Don't rely on eac
|
||||
### Centralize Job Creation Logic
|
||||
When multiple code paths create jobs (file upload, audio search, webhooks), use a shared service for default resolution. The made-for-you webhook handler diverged from regular job creation, missing CDG/TXT defaults because it didn't call the shared `resolve_cdg_txt_defaults()` function. Fix: Create `job_defaults_service.py` with centralized helpers used by ALL job creation paths.
|
||||
|
||||
### Make State Transitions Loud, Not Silent (Feb 2026)
|
||||
**What happened**: Jobs 06cfea29 and 984da08b got stuck at `pending` forever. The Made-For-You webhook handler triggered workers without first transitioning from PENDING to DOWNLOADING. The state machine returned `False` silently, workers ran, but job status never advanced.
|
||||
|
||||
**Root cause**: `transition_to_state()` returned `False` on invalid transitions, but callers didn't check the return value. Silent failures let bugs hide in production.
|
||||
|
||||
**Fix implemented**:
|
||||
1. **Make failures loud**: `transition_to_state()` now raises `InvalidStateTransitionError` by default (use `raise_on_invalid=False` for soft failures)
|
||||
2. **Centralize the pattern**: Created `start_job_processing()` helper that atomically transitions + triggers workers - all handlers use this instead of implementing independently
|
||||
3. **Runtime consistency checks**: Added `job_health_service.py` with `check_job_consistency()` to detect stuck jobs (audio_complete but status=pending)
|
||||
4. **Worker validation**: Workers now log warnings if triggered when job is in unexpected status
|
||||
|
||||
**Pattern**: When a function can fail, prefer exceptions over boolean returns. Silent failures accumulate into production bugs. Use centralized helpers for multi-step operations (state change + side effects) to prevent divergent implementations.
|
||||
|
||||
### Fix Both Sides of Dual Code Paths
|
||||
When fixing a bug in a system with multiple code paths (e.g., legacy vs orchestrator, local vs cloud), verify ALL paths are fixed. PR #271 fixed the GCE worker to READ `instrumental_selection` but only checked the legacy path which was already SENDING it. The orchestrator path (production default) wasn't sending it. **Pattern**: If a component receives config from multiple callers, check ALL callers when fixing the receiving side. Write integration tests that cover each path.
|
||||
|
||||
@@ -138,22 +125,6 @@ For subprocess without shell, FFmpeg filter paths need: apostrophes escaped as `
|
||||
### Unicode in HTTP Headers
|
||||
Sanitize user input (artist/title) to ASCII for HTTP headers. Smart quotes, em dashes from copy/paste cause encoding errors.
|
||||
|
||||
### Unicode in File Paths (Feb 2026)
|
||||
**What happened**: Job `1eab1172` with Russian title "я куплю тебе дом" (Cyrillic) failed with FFmpeg exit code 183. The temp subtitle file path contained Unicode characters: `temp_subtitles_zivert___я_куплю_тебе_дом_1770150285852.ass`.
|
||||
|
||||
**Root cause**: Python's `str.isalnum()` returns `True` for ALL Unicode alphanumeric characters (Cyrillic, Arabic, Chinese, etc.), not just ASCII. The code used `"".join(c if c.isalnum() else "_" for c in output_prefix)` to sanitize filenames, which allowed Cyrillic letters through because they're "alphanumeric" in Unicode. FFmpeg cannot handle Unicode characters in filter argument paths, causing file I/O errors.
|
||||
|
||||
**Fix**: Replace `c.isalnum()` with `c.isascii() and c.isalnum()` to only allow ASCII alphanumeric characters (a-z, A-Z, 0-9). This ensures temp file paths are safe for FFmpeg and other external tools.
|
||||
|
||||
**Locations fixed** (v0.115.2):
|
||||
- `karaoke_gen/lyrics_transcriber/output/video.py:222` - `generate_video()` temp subtitle path
|
||||
- `karaoke_gen/lyrics_transcriber/output/video.py:282` - `generate_preview_video()` temp subtitle path
|
||||
- `karaoke_gen/utils/remote_cli.py:1970` - output folder name sanitization
|
||||
|
||||
**Pattern**: When sanitizing filenames or paths for external tools, always use ASCII-only checks. Don't assume `isalnum()`, `isalpha()`, or `isdigit()` are ASCII-only - they match all Unicode categories.
|
||||
|
||||
**Test coverage**: Added 10 comprehensive tests covering Cyrillic, Chinese, Arabic, emoji, and accented characters to prevent regression.
|
||||
|
||||
### Fonts in Docker
|
||||
Base Docker images have no fonts. Install `fonts-noto-core`, `fonts-noto-cjk` for video rendering.
|
||||
|
||||
@@ -243,37 +214,6 @@ worker_index = call_order.index('audio_worker')
|
||||
assert download_index < worker_index, "Download must happen BEFORE workers"
|
||||
```
|
||||
|
||||
### Made-For-You Jobs Must Transition to DOWNLOADING Before Workers (Feb 2026)
|
||||
|
||||
**Problem**: Made-for-you orders with YouTube URLs (jobs 06cfea29, 984da08b) got stuck at `pending` status even though all processing completed successfully. Workers ran, `state_data` showed `audio_complete: true`, `lyrics_complete: true`, `screens_progress.stage: complete`, but the `status` field remained `pending`.
|
||||
|
||||
**Root cause**: The `_handle_made_for_you_order` function in `users.py` triggered workers for YouTube URL orders **without first transitioning the job from PENDING to DOWNLOADING**. The job state machine defines strict valid transitions:
|
||||
- `PENDING` → `DOWNLOADING`, `SEARCHING_AUDIO`, `FAILED`, `CANCELLED` (only these are valid)
|
||||
- Workers try: `PENDING` → `GENERATING_SCREENS` → `AWAITING_REVIEW` (INVALID!)
|
||||
|
||||
All status transitions after workers complete fail silently because they're invalid from `PENDING`.
|
||||
|
||||
**Why it was subtle**:
|
||||
1. Workers completed successfully (no errors in logs)
|
||||
2. `state_data` updates worked (they bypass state machine validation)
|
||||
3. Only the `status` field failed to update (uses `transition_to_state` which validates)
|
||||
|
||||
**Fix**:
|
||||
1. Immediate: Add `transition_to_state(job_id, JobStatus.DOWNLOADING)` before triggering workers
|
||||
2. Robustness: Make `transition_to_state()` raise `InvalidStateTransitionError` by default instead of returning `False` silently
|
||||
3. Pattern: Create centralized `job_manager.start_job_processing(job_id)` helper that handles transition + worker triggers atomically
|
||||
|
||||
**Robustness changes** (see `docs/archive/2026-02-02-state-machine-robustness-plan.md`):
|
||||
- `InvalidStateTransitionError` exception with full context (job_id, from_status, to_status, valid_transitions)
|
||||
- `validate_state_transition()` and `transition_to_state()` now have `raise_on_invalid=True` default
|
||||
- `start_job_processing()` centralizes the pattern: validate prerequisites → transition → trigger workers
|
||||
|
||||
**Pattern**: Always verify state machine flow in webhook handlers. Use centralized helpers like `start_job_processing()` instead of duplicating transition + worker logic.
|
||||
|
||||
**Tests added**:
|
||||
- `test_youtube_url_calls_start_job_processing_after_download` - verifies proper sequence
|
||||
- `TestInvalidStateTransitionError`, `TestValidateStateTransitionRaisesBehavior`, `TestStartJobProcessing` in `test_job_manager.py`
|
||||
|
||||
### Use data-testid for E2E
|
||||
Prefer `data-testid` over label/text selectors. They're immune to label changes and won't break when similar fields are added.
|
||||
|
||||
@@ -328,38 +268,6 @@ const debouncedUpdate = useCallback((val: string) => {
|
||||
|
||||
**Testing**: Verify (1) typing multi-digit values doesn't jump cursor, (2) debounce fires after delay, (3) blur syncs immediately, (4) external updates respected when not focused. See `frontend/components/lyrics-review/__tests__/TimeInput.test.tsx` for comprehensive test suite.
|
||||
|
||||
### Mocks Must Match Real Return Values (Feb 2026)
|
||||
|
||||
**Problem**: PR #371 introduced a bug where admin reset always returned 500. The code checked `if not success:` after calling `update_job()`, but `update_job()` returns `None`, not a boolean. Since `not None` is `True`, the error was always raised.
|
||||
|
||||
**Why tests didn't catch it**: All 22 test mocks had:
|
||||
```python
|
||||
mock_jm.update_job.return_value = True # WRONG - real method returns None
|
||||
```
|
||||
|
||||
The mock returned `True`, so `if not True:` was `False` and tests passed. But in production, `if not None:` is `True`, causing 500 errors.
|
||||
|
||||
**Root cause**: Mocks were written based on an assumed API contract (boolean return) rather than the actual contract (void return, raises on error).
|
||||
|
||||
**Fix**:
|
||||
1. Updated all mocks to `return_value = None` with comment explaining why
|
||||
2. Added emulator integration test (`test_admin_reset_integration.py`) that uses real `JobManager`
|
||||
3. Added explicit test `test_update_job_returns_none_not_boolean` to document the API contract
|
||||
|
||||
**Pattern for preventing this**:
|
||||
1. **Mocks should match real signatures**: Check the actual method before setting `return_value`. Void methods should mock `return_value = None`.
|
||||
2. **Add emulator tests for critical paths**: Integration tests with real services catch contract mismatches mocks hide.
|
||||
3. **Document API contracts in tests**: Add explicit tests for non-obvious behaviors (`update_job` returns None, raises on failure).
|
||||
|
||||
**Code pattern**:
|
||||
```python
|
||||
# ❌ WRONG - assumes boolean return
|
||||
mock_jm.update_job.return_value = True
|
||||
|
||||
# ✅ CORRECT - matches real API
|
||||
mock_jm.update_job.return_value = None # Real API: returns None, raises on error
|
||||
```
|
||||
|
||||
### Emulator Tests Catch Real Bugs
|
||||
Firestore emulator tests catch issues (like missing indexes) that unit tests with mocks miss.
|
||||
|
||||
|
||||
@@ -41,10 +41,6 @@
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- **Unicode Filename Fix** (2026-02-04): Fixed FFmpeg failures (exit code 183) when processing songs with non-ASCII characters in titles. Job `1eab1172` with Russian title "я куплю тебе дом" failed because Python's `str.isalnum()` returns True for ALL Unicode alphanumeric characters (Cyrillic, Arabic, Chinese, emoji, etc.), not just ASCII. These Unicode characters in temp subtitle file paths (`temp_subtitles_zivert___я_куплю_тебе_дом_1770150285852.ass`) caused FFmpeg to fail. **Fix**: Changed filename sanitization from `c.isalnum()` to `c.isascii() and c.isalnum()` in three locations: `video.py:222` (generate_video), `video.py:282` (generate_preview_video), and `remote_cli.py:1970` (output folders). Added comprehensive test suite covering Cyrillic, Chinese, Arabic, emoji, and accented characters. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#unicode-in-file-paths-feb-2026).
|
||||
|
||||
- **Admin MFY Restart & Override Tools** (2026-02-04): Added three new admin endpoints for better Made-For-You order management: (1) **Regenerate Screens** (`POST /api/admin/jobs/{job_id}/regenerate-screens`) - regenerates title/end screens with current artist/title metadata without full reprocessing; (2) **Full Restart** (`POST /api/admin/jobs/{job_id}/restart`) - unlike "Reset to Start", this actually triggers workers to run, with option to preserve audio stems for faster restart; (3) **Override Audio Source** (`POST /api/admin/jobs/{job_id}/override-audio-source`) - switch from YouTube URL to audio search mode for higher quality audio. Frontend admin toolbar updated with new buttons (Regen Screens, Full Restart, Audio Search) with confirmation dialogs. **Use case**: Fix MFY orders where customer provided wrong artist/title or low-quality YouTube URL. See [API.md](API.md#regenerate-screens).
|
||||
|
||||
- **Made-For-You YouTube URL Download Fix** (2026-01-28): Fixed bug where made-for-you orders with YouTube URLs failed with "Failed to download audio file". **Root cause**: The webhook handler triggered workers directly without downloading the audio first via `YouTubeDownloadService`. Workers require `input_media_gcs_path` to be set, which only happens after download completes. **Fix**: Handler now downloads YouTube audio to GCS before triggering workers, matching the pattern in `file_upload.py` and `audio_search.py`. Added 4 new contract tests with call-order tracking and AST import validation to prevent regression. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#youtube-url-downloads-must-happen-before-workers-jan-2026).
|
||||
|
||||
- **Lyrics Corrections Now Applied to Video** (2026-01-26): Fixed critical bug where user's lyrics corrections made during human review were being ignored. Videos were rendering with raw transcription instead of corrected lyrics. **Root cause**: The `GET /api/review/{job_id}/correction-data` endpoint was returning the original `corrections.json` instead of `corrections_updated.json` (where user edits are saved). The combined review flow would then overwrite the user's edits when completing the review. **Fix**: The endpoint now checks for `corrections_updated.json` first, matching the behavior of `render_video_worker.py`. Added 35 new tests (backend unit + integration + frontend E2E) to prevent regression. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#always-test-the-full-data-flow-not-just-individual-components).
|
||||
|
||||
@@ -158,30 +158,6 @@ def test_create_job_applies_default_theme(mock_firestore, mock_gcs):
|
||||
assert job.theme_id == "nomad" # Default theme applied
|
||||
```
|
||||
|
||||
**Critical: Mocks Must Match Real API Contracts**
|
||||
|
||||
When mocking methods, ensure the mock's `return_value` matches what the real method returns. A common bug pattern:
|
||||
|
||||
```python
|
||||
# ❌ BAD: Assumes method returns boolean
|
||||
mock_service.update_job.return_value = True
|
||||
|
||||
# Code under test:
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
if not success: # BUG: if real method returns None, this is always True!
|
||||
raise HTTPException(500)
|
||||
```
|
||||
|
||||
The fix:
|
||||
|
||||
```python
|
||||
# ✅ GOOD: Check real method signature first
|
||||
# def update_job(self, job_id: str, updates: Dict) -> None:
|
||||
mock_service.update_job.return_value = None # Match real API
|
||||
```
|
||||
|
||||
**Before writing a mock**, check the actual method's return type. Void methods should mock `return_value = None`. Methods that raise on error (rather than returning False) should have mocks that raise exceptions for error cases. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#mocks-must-match-real-return-values-feb-2026) for a production bug caused by this.
|
||||
|
||||
### Integration Tests (`tests/integration/`)
|
||||
|
||||
**Purpose:** Test CLI workflows and component interactions.
|
||||
|
||||
@@ -1,307 +0,0 @@
|
||||
# State Machine Robustness Plan
|
||||
|
||||
**Date**: 2026-02-02
|
||||
**Status**: ✓ Complete (Weeks 1-3 implemented)
|
||||
**Context**: Jobs 06cfea29 and 984da08b got stuck at `pending` because state transitions failed silently.
|
||||
|
||||
## Problem Analysis
|
||||
|
||||
The root issues that allowed this bug:
|
||||
|
||||
1. **Silent failures**: `transition_to_state()` returns `False` on invalid transitions, but callers don't check it
|
||||
2. **No centralized worker trigger pattern**: Each handler (file_upload.py, users.py, audio_search.py) implements "transition + trigger workers" independently
|
||||
3. **No runtime consistency checks**: Nothing detects when `state_data` says complete but `status` is wrong
|
||||
4. **Workers don't validate prerequisites**: Workers run regardless of job status
|
||||
|
||||
## Proposed Improvements
|
||||
|
||||
### 1. Make State Transition Failures Loud (High Priority)
|
||||
|
||||
**Option A**: Raise exception by default (breaking change, but safest)
|
||||
```python
|
||||
class InvalidStateTransitionError(Exception):
|
||||
"""Raised when an invalid state transition is attempted."""
|
||||
pass
|
||||
|
||||
def transition_to_state(self, job_id: str, new_status: JobStatus, ...,
|
||||
raise_on_invalid: bool = True) -> bool:
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
if raise_on_invalid:
|
||||
raise InvalidStateTransitionError(
|
||||
f"Invalid transition for job {job_id}: {current} -> {new_status}"
|
||||
)
|
||||
return False
|
||||
...
|
||||
```
|
||||
|
||||
**Option B**: Always log + metric, return bool (non-breaking)
|
||||
```python
|
||||
def transition_to_state(...) -> bool:
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
# Log with structured data for alerting
|
||||
logger.error(
|
||||
f"STATE_TRANSITION_FAILED job={job_id} from={current} to={new_status}",
|
||||
extra={"job_id": job_id, "from_status": current, "to_status": new_status}
|
||||
)
|
||||
# Emit metric for monitoring
|
||||
emit_metric("state_transition_failed", 1, {"from": current, "to": new_status})
|
||||
return False
|
||||
...
|
||||
```
|
||||
|
||||
**Recommendation**: Option A with `raise_on_invalid=True` default. Forces callers to handle failures explicitly.
|
||||
|
||||
### 2. Centralize "Start Processing" Pattern (High Priority)
|
||||
|
||||
Create a single function that handles the transition + worker trigger pattern:
|
||||
|
||||
```python
|
||||
# backend/services/job_manager.py
|
||||
|
||||
async def start_job_processing(self, job_id: str, worker_service) -> None:
|
||||
"""
|
||||
Transition job to DOWNLOADING and trigger workers.
|
||||
|
||||
This is the single entry point for starting job processing.
|
||||
All handlers (file_upload, webhook, audio_search) should use this.
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If job can't transition to DOWNLOADING
|
||||
ValueError: If job not found or missing required fields
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
|
||||
if not job.input_media_gcs_path:
|
||||
raise ValueError(f"Job {job_id} missing input_media_gcs_path")
|
||||
|
||||
# This will raise if transition is invalid
|
||||
self.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=15,
|
||||
message="Starting audio and lyrics processing"
|
||||
)
|
||||
|
||||
# Only trigger workers if transition succeeded
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
```
|
||||
|
||||
Then update all handlers to use this:
|
||||
```python
|
||||
# In _handle_made_for_you_order, file_upload, audio_search:
|
||||
await job_manager.start_job_processing(job_id, worker_service)
|
||||
```
|
||||
|
||||
### 3. Add State Consistency Checks (Medium Priority)
|
||||
|
||||
Add a health check that detects inconsistent state:
|
||||
|
||||
```python
|
||||
# backend/services/job_health_service.py
|
||||
|
||||
def check_job_consistency(job: Job) -> List[str]:
|
||||
"""
|
||||
Check for state inconsistencies in a job.
|
||||
Returns list of issues found.
|
||||
"""
|
||||
issues = []
|
||||
state_data = job.state_data or {}
|
||||
|
||||
# Check: audio_complete but status still pending
|
||||
if state_data.get('audio_complete') and job.status == JobStatus.PENDING:
|
||||
issues.append(f"audio_complete=True but status=pending")
|
||||
|
||||
# Check: lyrics_complete but status still pending
|
||||
if state_data.get('lyrics_complete') and job.status == JobStatus.PENDING:
|
||||
issues.append(f"lyrics_complete=True but status=pending")
|
||||
|
||||
# Check: screens_progress.stage=complete but status not awaiting_review
|
||||
screens = state_data.get('screens_progress', {})
|
||||
if screens.get('stage') == 'complete' and job.status not in [
|
||||
JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO, JobStatus.INSTRUMENTAL_SELECTED, ...
|
||||
]:
|
||||
issues.append(f"screens complete but status={job.status}")
|
||||
|
||||
return issues
|
||||
```
|
||||
|
||||
Run this:
|
||||
- In a daily Cloud Scheduler job
|
||||
- After each worker completes
|
||||
- Expose via `/health/job-consistency` endpoint
|
||||
|
||||
### 4. Workers Validate Prerequisites (Medium Priority)
|
||||
|
||||
Workers should verify job is in expected state before processing:
|
||||
|
||||
```python
|
||||
# backend/workers/screens_worker.py
|
||||
|
||||
async def generate_screens(job_id: str) -> bool:
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
# Validate job is in expected state for this worker
|
||||
valid_statuses = [JobStatus.DOWNLOADING, JobStatus.AUDIO_COMPLETE, JobStatus.LYRICS_COMPLETE]
|
||||
if job.status not in [s.value for s in valid_statuses]:
|
||||
logger.error(
|
||||
f"[job:{job_id}] screens_worker called but job status is {job.status}, "
|
||||
f"expected one of {valid_statuses}. This indicates a bug in the trigger logic."
|
||||
)
|
||||
# Option: return False, or try to recover, or alert
|
||||
return False
|
||||
|
||||
# ... rest of worker
|
||||
```
|
||||
|
||||
### 5. Add Integration Tests for State Flow (Medium Priority)
|
||||
|
||||
Add tests that verify the complete state machine flow without mocking transitions:
|
||||
|
||||
```python
|
||||
# backend/tests/integration/test_state_machine_flow.py
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_made_for_you_youtube_flow_state_transitions():
|
||||
"""
|
||||
Verify complete state flow for made-for-you YouTube orders.
|
||||
Uses real JobManager (mocked Firestore) to validate state machine.
|
||||
"""
|
||||
# Create job
|
||||
job = job_manager.create_job(...)
|
||||
assert job.status == JobStatus.PENDING
|
||||
|
||||
# Simulate YouTube download complete
|
||||
job_manager.update_job(job_id, {'input_media_gcs_path': 'test/path'})
|
||||
|
||||
# Start processing (this should transition to DOWNLOADING)
|
||||
await job_manager.start_job_processing(job_id, mock_worker_service)
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
assert job.status == JobStatus.DOWNLOADING, \
|
||||
f"Expected DOWNLOADING after start_job_processing, got {job.status}"
|
||||
|
||||
# ... continue through full flow
|
||||
```
|
||||
|
||||
### 6. State Machine Diagram Generation (Low Priority)
|
||||
|
||||
Auto-generate a Mermaid diagram from `STATE_TRANSITIONS`:
|
||||
|
||||
```python
|
||||
# scripts/generate_state_diagram.py
|
||||
|
||||
def generate_mermaid():
|
||||
lines = ["stateDiagram-v2"]
|
||||
for from_state, to_states in STATE_TRANSITIONS.items():
|
||||
for to_state in to_states:
|
||||
lines.append(f" {from_state} --> {to_state}")
|
||||
return "\n".join(lines)
|
||||
```
|
||||
|
||||
Add to `docs/ARCHITECTURE.md` and regenerate on changes.
|
||||
|
||||
### 7. Fix Admin Reset Buttons (High Priority)
|
||||
|
||||
The admin dashboard has reset buttons that bypass state machine validation and have several issues:
|
||||
|
||||
**Current buttons:**
|
||||
- "Start" → reset to `pending`
|
||||
- "Audio" → reset to `awaiting_audio_selection`
|
||||
- "Lyrics" → reset to `awaiting_review`
|
||||
- "Inst." → reset to `awaiting_instrumental_selection`
|
||||
- "Reprocess" → reset to `instrumental_selected`
|
||||
|
||||
**Issues identified:**
|
||||
|
||||
1. **"Start" button leaves job stuck**: Resets to `pending` but doesn't trigger workers or clear `audio_complete`/`lyrics_complete` flags. Job sits indefinitely.
|
||||
|
||||
2. **"Lyrics" and "Inst." are obsolete**: Combined review flow (Jan 2026) merged these into a single step. Should be replaced with single "Review" button that resets to `awaiting_review`.
|
||||
|
||||
3. **Missing state_data keys in clear list for `pending`**:
|
||||
```python
|
||||
# Should also clear:
|
||||
"audio_complete", "lyrics_complete"
|
||||
```
|
||||
|
||||
4. **State machine bypass**: Admin resets use `update_job()` directly, not `transition_to_state()`. This is intentional for admin overrides but should be documented.
|
||||
|
||||
**Fixes needed:**
|
||||
|
||||
```python
|
||||
# backend/api/routes/admin.py
|
||||
|
||||
# 1. Update STATE_DATA_CLEAR_KEYS for "pending"
|
||||
STATE_DATA_CLEAR_KEYS = {
|
||||
"pending": [
|
||||
# ... existing keys ...
|
||||
"audio_complete", # NEW - clear parallel processing flag
|
||||
"lyrics_complete", # NEW - clear parallel processing flag
|
||||
],
|
||||
# ...
|
||||
}
|
||||
|
||||
# 2. Remove awaiting_instrumental_selection from ALLOWED_RESET_STATES
|
||||
ALLOWED_RESET_STATES = {
|
||||
"pending",
|
||||
"awaiting_audio_selection",
|
||||
"awaiting_review", # Combined review (lyrics + instrumental)
|
||||
# "awaiting_instrumental_selection", # REMOVED - deprecated
|
||||
"instrumental_selected",
|
||||
}
|
||||
|
||||
# 3. After reset to pending, trigger audio search if job has artist/title
|
||||
# or provide clear guidance to admin about next steps
|
||||
```
|
||||
|
||||
```tsx
|
||||
// frontend/app/admin/jobs/page.tsx
|
||||
|
||||
// Replace "Lyrics" and "Inst." buttons with single "Review" button:
|
||||
{[
|
||||
{ state: "pending", icon: RotateCcw, label: "Start" },
|
||||
{ state: "awaiting_audio_selection", icon: Music, label: "Audio" },
|
||||
{ state: "awaiting_review", icon: Mic, label: "Review" }, // Combined
|
||||
// REMOVED: { state: "awaiting_instrumental_selection", ... }
|
||||
{ state: "instrumental_selected", icon: RefreshCw, label: "Reprocess" },
|
||||
]}
|
||||
```
|
||||
|
||||
## Implementation Priority
|
||||
|
||||
1. **Week 1** ✓ DONE:
|
||||
- ✓ Make `transition_to_state` raise by default (`InvalidStateTransitionError`, `raise_on_invalid=True`)
|
||||
- ✓ Create `start_job_processing()` helper in `job_manager.py`
|
||||
- ✓ Update `_handle_made_for_you_order` to use it
|
||||
- ✓ Added 15 unit tests covering exception behavior and new helper
|
||||
|
||||
2. **Week 1.5** ✓ DONE (Admin Button Fixes):
|
||||
- ✓ Add `audio_complete`, `lyrics_complete` to STATE_DATA_CLEAR_KEYS for "pending"
|
||||
- ✓ Remove `awaiting_instrumental_selection` from ALLOWED_RESET_STATES
|
||||
- ✓ Update frontend to replace "Lyrics"/"Inst." with single "Review" button
|
||||
- ✓ Add tests for admin reset endpoint (`test_reset_to_pending_clears_parallel_processing_flags`, `test_rejects_deprecated_awaiting_instrumental_selection`)
|
||||
|
||||
3. **Week 2** ✓ DONE:
|
||||
- ✓ Update `file_upload.py` and `audio_search.py` to use `start_job_processing()`
|
||||
- ✓ Fixed double-transition bug in YouTube URL upload flow
|
||||
- ✓ Removed unused `_trigger_workers_parallel` helper from file_upload.py and audio_search.py
|
||||
- ✓ Add worker prerequisite validation (`validate_worker_can_run()` in all workers)
|
||||
- ✓ Add state consistency health check (`job_health_service.py` with `check_job_consistency()`)
|
||||
- ✓ Added 26 tests for job_health_service
|
||||
|
||||
4. **Week 3** ✓ DONE:
|
||||
- ✓ Add integration tests for state flow (`tests/integration/test_state_machine_flow.py` - 11 tests)
|
||||
- ✓ Generate state machine diagram script (`scripts/generate_state_diagram.py`)
|
||||
- ✓ Expose `/health/job-consistency` endpoint for admin dashboard
|
||||
- TODO: Add monitoring/alerting on `STATE_TRANSITION_FAILED` logs (requires Cloud Logging alert setup)
|
||||
|
||||
## Quick Wins (Can Do Now)
|
||||
|
||||
1. Add explicit error logging when transition fails (already exists but add job_id to structured logging)
|
||||
2. Create Cloud Logging alert for "Invalid state transition" errors
|
||||
3. Add consistency check to admin dashboard "stuck jobs" view
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import { useEffect, useState, useCallback, Suspense } from "react"
|
||||
import { useRouter, useSearchParams } from "next/navigation"
|
||||
import { adminApi, api, Job, FileInfo, JobFilesResponse, JobUpdateRequest, JobResetResponse, DeleteOutputsResponse, RegenerateScreensResponse, RestartJobResponse, OverrideAudioSourceResponse } from "@/lib/api"
|
||||
import { adminApi, api, Job, FileInfo, JobFilesResponse, JobUpdateRequest, JobResetResponse, DeleteOutputsResponse, ClearWorkersResponse, TriggerWorkerResponse } from "@/lib/api"
|
||||
import { useAdminSettings } from "@/lib/admin-settings"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Input } from "@/components/ui/input"
|
||||
@@ -182,21 +182,11 @@ function AdminJobsPageContent() {
|
||||
const [deleteOutputsResult, setDeleteOutputsResult] = useState<DeleteOutputsResponse | null>(null)
|
||||
const [deleteOutputsResultOpen, setDeleteOutputsResultOpen] = useState(false)
|
||||
|
||||
// Note: clearingWorkers and triggeringWorker states removed
|
||||
// These low-level escape hatches are replaced by Full Restart and Regen Screens
|
||||
// Clear workers state
|
||||
const [clearingWorkers, setClearingWorkers] = useState(false)
|
||||
|
||||
// Regenerate screens state
|
||||
const [regeneratingScreens, setRegeneratingScreens] = useState(false)
|
||||
const [regenerateScreensDialogOpen, setRegenerateScreensDialogOpen] = useState(false)
|
||||
|
||||
// Restart job state
|
||||
const [restartDialogOpen, setRestartDialogOpen] = useState(false)
|
||||
const [restarting, setRestarting] = useState(false)
|
||||
const [restartPreserveStems, setRestartPreserveStems] = useState(true)
|
||||
|
||||
// Override audio source state
|
||||
const [overrideAudioDialogOpen, setOverrideAudioDialogOpen] = useState(false)
|
||||
const [overridingAudio, setOverridingAudio] = useState(false)
|
||||
// Trigger worker state
|
||||
const [triggeringWorker, setTriggeringWorker] = useState(false)
|
||||
|
||||
const loadJobs = useCallback(async () => {
|
||||
try {
|
||||
@@ -430,113 +420,74 @@ function AdminJobsPageContent() {
|
||||
}
|
||||
}
|
||||
|
||||
// Note: handleClearWorkers and handleTriggerWorker removed
|
||||
// These low-level escape hatches are replaced by Full Restart and Regen Screens
|
||||
|
||||
// Handle regenerate screens - regenerates title/end screens with current metadata
|
||||
const handleRegenerateScreens = async () => {
|
||||
// Handle clear workers - clears worker progress to allow re-execution
|
||||
const handleClearWorkers = async () => {
|
||||
if (!selectedJobId) return
|
||||
|
||||
try {
|
||||
setRegeneratingScreens(true)
|
||||
const result = await adminApi.regenerateScreens(selectedJobId)
|
||||
setClearingWorkers(true)
|
||||
const result = await adminApi.clearWorkers(selectedJobId)
|
||||
|
||||
if (result.worker_triggered) {
|
||||
if (result.cleared_keys.length > 0) {
|
||||
toast({
|
||||
title: "Screen Regeneration Started",
|
||||
description: "Title and end screens are being regenerated with current artist/title. This may take 30-60 seconds.",
|
||||
title: "Workers Cleared",
|
||||
description: `${result.message}: ${result.cleared_keys.join(", ")}`,
|
||||
})
|
||||
} else {
|
||||
toast({
|
||||
title: "Regeneration Failed",
|
||||
description: result.error || result.message,
|
||||
title: "No Progress Keys",
|
||||
description: result.message,
|
||||
})
|
||||
}
|
||||
|
||||
// Refresh job details
|
||||
loadJobDetail(selectedJobId)
|
||||
loadLogs(selectedJobId)
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: "Error",
|
||||
description: err.message || "Failed to clear workers",
|
||||
variant: "destructive",
|
||||
})
|
||||
} finally {
|
||||
setClearingWorkers(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle trigger worker - manually triggers a worker for processing
|
||||
const handleTriggerWorker = async (workerType: string = "video") => {
|
||||
if (!selectedJobId) return
|
||||
|
||||
try {
|
||||
setTriggeringWorker(true)
|
||||
const result = await adminApi.triggerWorker(selectedJobId, workerType)
|
||||
|
||||
if (result.triggered) {
|
||||
toast({
|
||||
title: `${workerType.charAt(0).toUpperCase() + workerType.slice(1)} Worker Triggered`,
|
||||
description: `Job ${selectedJobId} is now processing. The ${workerType} worker will begin generating the video.`,
|
||||
})
|
||||
} else {
|
||||
toast({
|
||||
title: "Trigger Failed",
|
||||
description: result.error
|
||||
? `${result.message}: ${result.error}`
|
||||
: result.message,
|
||||
variant: "destructive",
|
||||
})
|
||||
}
|
||||
|
||||
setRegenerateScreensDialogOpen(false)
|
||||
// Refresh job details
|
||||
loadJobDetail(selectedJobId)
|
||||
loadLogs(selectedJobId)
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: "Error",
|
||||
description: err.message || "Failed to regenerate screens",
|
||||
description: err.message || "Failed to trigger worker",
|
||||
variant: "destructive",
|
||||
})
|
||||
} finally {
|
||||
setRegeneratingScreens(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle restart job - fully restarts a job with worker triggering
|
||||
const handleRestartJob = async () => {
|
||||
if (!selectedJobId) return
|
||||
|
||||
try {
|
||||
setRestarting(true)
|
||||
const result = await adminApi.restartJob(selectedJobId, {
|
||||
preserve_audio_stems: restartPreserveStems,
|
||||
delete_outputs: true,
|
||||
})
|
||||
|
||||
if (result.status === "success" || result.status === "partial") {
|
||||
toast({
|
||||
title: "Job Restarted",
|
||||
description: `${result.message}. Workers triggered: ${result.workers_triggered.join(", ") || "none (waiting for action)"}`,
|
||||
})
|
||||
}
|
||||
|
||||
if (result.error) {
|
||||
toast({
|
||||
title: "Warning",
|
||||
description: result.error,
|
||||
variant: "destructive",
|
||||
})
|
||||
}
|
||||
|
||||
setRestartDialogOpen(false)
|
||||
// Refresh job details
|
||||
loadJobDetail(selectedJobId)
|
||||
loadLogs(selectedJobId)
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: "Error",
|
||||
description: err.message || "Failed to restart job",
|
||||
variant: "destructive",
|
||||
})
|
||||
} finally {
|
||||
setRestarting(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle override audio source - switches to audio search mode
|
||||
const handleOverrideAudioSource = async () => {
|
||||
if (!selectedJobId) return
|
||||
|
||||
try {
|
||||
setOverridingAudio(true)
|
||||
const result = await adminApi.overrideAudioSource(selectedJobId, {
|
||||
source_type: "audio_search",
|
||||
})
|
||||
|
||||
toast({
|
||||
title: "Audio Source Changed",
|
||||
description: result.message,
|
||||
})
|
||||
|
||||
setOverrideAudioDialogOpen(false)
|
||||
// Refresh job details
|
||||
loadJobDetail(selectedJobId)
|
||||
loadLogs(selectedJobId)
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: "Error",
|
||||
description: err.message || "Failed to override audio source",
|
||||
variant: "destructive",
|
||||
})
|
||||
} finally {
|
||||
setOverridingAudio(false)
|
||||
setTriggeringWorker(false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -545,7 +496,7 @@ function AdminJobsPageContent() {
|
||||
const info: Record<string, { label: string; description: string; icon: any }> = {
|
||||
pending: {
|
||||
label: "Pending",
|
||||
description: "Restart from the beginning. Clears all processing data including audio/lyrics progress.",
|
||||
description: "Restart from the beginning. Clears all processing data.",
|
||||
icon: RotateCcw,
|
||||
},
|
||||
awaiting_audio_selection: {
|
||||
@@ -554,10 +505,15 @@ function AdminJobsPageContent() {
|
||||
icon: Music,
|
||||
},
|
||||
awaiting_review: {
|
||||
label: "Combined Review",
|
||||
description: "Re-review lyrics AND re-select instrumental. Preserves audio stems.",
|
||||
label: "Lyrics Review",
|
||||
description: "Re-review lyrics. Preserves audio stems.",
|
||||
icon: Mic,
|
||||
},
|
||||
awaiting_instrumental_selection: {
|
||||
label: "Instrumental",
|
||||
description: "Re-select instrumental. Preserves lyrics review.",
|
||||
icon: Sliders,
|
||||
},
|
||||
instrumental_selected: {
|
||||
label: "Reprocess Video",
|
||||
description: "Re-encode and re-distribute with current settings. Triggers video worker automatically.",
|
||||
@@ -858,9 +814,10 @@ function AdminJobsPageContent() {
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wide mr-1">Reset to:</span>
|
||||
{[
|
||||
// Note: "pending" (Start) removed - use "Full Restart" button instead
|
||||
{ state: "pending", icon: RotateCcw, label: "Start" },
|
||||
{ state: "awaiting_audio_selection", icon: Music, label: "Audio" },
|
||||
{ state: "awaiting_review", icon: Mic, label: "Review" }, // Combined lyrics + instrumental review
|
||||
{ state: "awaiting_review", icon: Mic, label: "Lyrics" },
|
||||
{ state: "awaiting_instrumental_selection", icon: Sliders, label: "Inst." },
|
||||
{ state: "instrumental_selected", icon: RefreshCw, label: "Reprocess" },
|
||||
].map(({ state, icon: Icon, label }) => (
|
||||
<Tooltip key={state}>
|
||||
@@ -907,27 +864,21 @@ function AdminJobsPageContent() {
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
{/* Admin Actions */}
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 px-2 text-xs border-blue-500 text-blue-600 hover:bg-blue-50 dark:hover:bg-blue-950"
|
||||
onClick={() => setRegenerateScreensDialogOpen(true)}
|
||||
disabled={regeneratingScreens}
|
||||
className="h-7 px-2 text-xs"
|
||||
onClick={handleClearWorkers}
|
||||
disabled={clearingWorkers}
|
||||
>
|
||||
{regeneratingScreens ? (
|
||||
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
|
||||
) : (
|
||||
<RefreshCw className="w-3 h-3 mr-1" />
|
||||
)}
|
||||
Regen Screens
|
||||
<RefreshCw className={`w-3 h-3 mr-1 ${clearingWorkers ? "animate-spin" : ""}`} />
|
||||
Clear Workers
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p className="font-medium">Regenerate Title/End Screens</p>
|
||||
<p className="text-xs text-muted-foreground">Use after editing artist/title to update screens</p>
|
||||
Clear worker progress markers to allow re-execution
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
@@ -936,52 +887,23 @@ function AdminJobsPageContent() {
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 px-2 text-xs border-purple-500 text-purple-600 hover:bg-purple-50 dark:hover:bg-purple-950"
|
||||
onClick={() => {
|
||||
setRestartPreserveStems(true)
|
||||
setRestartDialogOpen(true)
|
||||
}}
|
||||
disabled={restarting}
|
||||
className={`h-7 px-2 text-xs ${selectedJob.status === "instrumental_selected" ? "border-green-500 text-green-600 hover:bg-green-50 dark:hover:bg-green-950" : ""}`}
|
||||
onClick={() => handleTriggerWorker("video")}
|
||||
disabled={triggeringWorker}
|
||||
>
|
||||
{restarting ? (
|
||||
{triggeringWorker ? (
|
||||
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
|
||||
) : (
|
||||
<RotateCcw className="w-3 h-3 mr-1" />
|
||||
<Play className="w-3 h-3 mr-1" />
|
||||
)}
|
||||
Full Restart
|
||||
Trigger
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p className="font-medium">Full Restart with Worker Triggering</p>
|
||||
<p className="text-xs text-muted-foreground">Unlike Reset, this actually triggers workers to run</p>
|
||||
Manually trigger video worker processing
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 px-2 text-xs border-amber-500 text-amber-600 hover:bg-amber-50 dark:hover:bg-amber-950"
|
||||
onClick={() => setOverrideAudioDialogOpen(true)}
|
||||
disabled={overridingAudio}
|
||||
>
|
||||
{overridingAudio ? (
|
||||
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
|
||||
) : (
|
||||
<Music className="w-3 h-3 mr-1" />
|
||||
)}
|
||||
Audio Search
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p className="font-medium">Switch to Audio Search</p>
|
||||
<p className="text-xs text-muted-foreground">Override YouTube URL with audio search for better quality</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
<Separator orientation="vertical" className="h-5 mx-1" />
|
||||
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
@@ -1716,7 +1638,8 @@ function AdminJobsPageContent() {
|
||||
<span className="font-medium">What happens next:</span>{" "}
|
||||
{resetResult.new_status === "pending" && "Job will need to go through audio search, download, and all processing steps."}
|
||||
{resetResult.new_status === "awaiting_audio_selection" && "User can select from the cached audio search results."}
|
||||
{resetResult.new_status === "awaiting_review" && "User can review lyrics and select instrumental track."}
|
||||
{resetResult.new_status === "awaiting_review" && "User can review and correct the lyrics."}
|
||||
{resetResult.new_status === "awaiting_instrumental_selection" && "User can select the instrumental track."}
|
||||
{resetResult.new_status === "instrumental_selected" && (
|
||||
resetResult.worker_triggered
|
||||
? "Video generation is now in progress."
|
||||
@@ -1744,134 +1667,6 @@ function AdminJobsPageContent() {
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{/* Regenerate Screens Confirmation Dialog */}
|
||||
<AlertDialog open={regenerateScreensDialogOpen} onOpenChange={setRegenerateScreensDialogOpen}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle className="flex items-center gap-2">
|
||||
<RefreshCw className="w-5 h-5 text-blue-500" />
|
||||
Regenerate Title/End Screens
|
||||
</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
This will regenerate the title and end screen videos using the <strong>current</strong> artist and title metadata.
|
||||
<br /><br />
|
||||
<strong>Current values:</strong>
|
||||
<ul className="list-disc list-inside mt-2 space-y-1">
|
||||
<li>Artist: <code className="bg-muted px-1 rounded">{selectedJob?.artist || "(not set)"}</code></li>
|
||||
<li>Title: <code className="bg-muted px-1 rounded">{selectedJob?.title || "(not set)"}</code></li>
|
||||
</ul>
|
||||
<br />
|
||||
Screen generation takes 30-60 seconds. Progress will be visible in the job timeline.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
onClick={handleRegenerateScreens}
|
||||
disabled={regeneratingScreens}
|
||||
className="bg-blue-500 hover:bg-blue-600"
|
||||
>
|
||||
{regeneratingScreens && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
|
||||
Regenerate Screens
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
{/* Restart Job Confirmation Dialog */}
|
||||
<AlertDialog open={restartDialogOpen} onOpenChange={setRestartDialogOpen}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle className="flex items-center gap-2">
|
||||
<RotateCcw className="w-5 h-5 text-purple-500" />
|
||||
Full Restart Job
|
||||
</AlertDialogTitle>
|
||||
<AlertDialogDescription asChild>
|
||||
<div>
|
||||
<p>Unlike "Reset to Start", this will actually <strong>trigger workers</strong> to begin processing.</p>
|
||||
<br />
|
||||
<div className="flex items-center space-x-2 p-3 bg-muted rounded-lg">
|
||||
<input
|
||||
type="checkbox"
|
||||
id="preserveStems"
|
||||
checked={restartPreserveStems}
|
||||
onChange={(e) => setRestartPreserveStems(e.target.checked)}
|
||||
className="h-4 w-4 rounded border-gray-300"
|
||||
/>
|
||||
<label htmlFor="preserveStems" className="text-sm font-medium">
|
||||
Preserve audio stems (faster)
|
||||
</label>
|
||||
</div>
|
||||
<br />
|
||||
{restartPreserveStems ? (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
<strong>Quick restart:</strong> Keeps existing audio separation and lyrics. Only regenerates screens and video with current metadata. Good for fixing title/artist typos.
|
||||
</p>
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
<strong>Full restart:</strong> Re-downloads audio (if YouTube), re-separates stems, re-transcribes lyrics. Use when audio source needs to change.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
onClick={handleRestartJob}
|
||||
disabled={restarting}
|
||||
className="bg-purple-500 hover:bg-purple-600"
|
||||
>
|
||||
{restarting && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
|
||||
Restart Job
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
{/* Override Audio Source Confirmation Dialog */}
|
||||
<AlertDialog open={overrideAudioDialogOpen} onOpenChange={setOverrideAudioDialogOpen}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle className="flex items-center gap-2">
|
||||
<Music className="w-5 h-5 text-amber-500" />
|
||||
Switch to Audio Search
|
||||
</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
This will clear the current audio source and switch to audio search mode.
|
||||
<br /><br />
|
||||
<strong>What happens:</strong>
|
||||
<ul className="list-disc list-inside mt-2 space-y-1">
|
||||
<li>YouTube URL and downloaded audio will be cleared</li>
|
||||
<li>All processing state will be reset</li>
|
||||
<li>Job will move to "Awaiting Audio Selection"</li>
|
||||
<li>You can then search for and select higher quality audio</li>
|
||||
</ul>
|
||||
<br />
|
||||
<strong>Current source:</strong>{" "}
|
||||
{selectedJob?.url ? (
|
||||
<span className="text-muted-foreground">YouTube URL</span>
|
||||
) : selectedJob?.input_media_gcs_path ? (
|
||||
<span className="text-muted-foreground">Uploaded file</span>
|
||||
) : (
|
||||
<span className="text-muted-foreground">Audio search</span>
|
||||
)}
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
onClick={handleOverrideAudioSource}
|
||||
disabled={overridingAudio}
|
||||
className="bg-amber-500 hover:bg-amber-600"
|
||||
>
|
||||
{overridingAudio && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
|
||||
Switch to Audio Search
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</div>
|
||||
</TooltipProvider>
|
||||
)
|
||||
|
||||
@@ -16,8 +16,8 @@ import { createEmailHelper, isEmailTestingAvailable } from '../helpers/email-tes
|
||||
* 4. Create Job - Search for test song via UI
|
||||
* 5. Audio Selection - Select audio source via UI dialog
|
||||
* 6. Wait for Processing - Monitor via UI status updates
|
||||
* 7. Combined Review - Open review UI, preview video, proceed to instrumental
|
||||
* 8. Instrumental Selection - Select instrumental and submit (same page via hash nav)
|
||||
* 7. Lyrics Review - Open review UI, interact, submit via UI
|
||||
* 8. Instrumental Selection - Open selection UI, interact, submit via UI
|
||||
* 9. Wait for Completion - Monitor via UI
|
||||
* 10. Verify Downloads - Check download links work
|
||||
* 11. Verify Distribution - Check YouTube/Dropbox/GDrive indicators
|
||||
@@ -540,15 +540,11 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
console.log('STEP 6 COMPLETE: Lyrics processing finished');
|
||||
|
||||
// =========================================================================
|
||||
// STEP 7: Combined Lyrics Review & Instrumental Selection (via UI - REAL USER FLOW)
|
||||
// After the UI redesign, lyrics review and instrumental selection are combined:
|
||||
// 1. User opens lyrics review page
|
||||
// 2. User clicks "Preview Video" to open modal
|
||||
// 3. User clicks "Proceed to Instrumental Review" which navigates to instrumental selection
|
||||
// 4. User selects instrumental and clicks "Confirm & Continue"
|
||||
// STEP 7: Lyrics Review (via UI - REAL USER FLOW)
|
||||
// The minimal path: scroll to bottom -> click "Preview Video" -> wait for video -> click "Complete Review"
|
||||
// =========================================================================
|
||||
console.log('\n========================================');
|
||||
console.log('STEP 7: Combined Lyrics Review & Instrumental Selection');
|
||||
console.log('STEP 7: Lyrics Review (UI Interaction)');
|
||||
console.log('========================================');
|
||||
|
||||
// Click the Review Lyrics button/link
|
||||
@@ -625,7 +621,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
const errorText = await errorAlert.textContent();
|
||||
console.log(` WARNING: Preview error: ${errorText}`);
|
||||
await reviewPage.screenshot({ path: 'test-results/07e-preview-error.png', fullPage: true });
|
||||
// Continue anyway - we can still proceed to instrumental even if preview failed
|
||||
// Continue anyway - we can still click Complete Review even if preview failed
|
||||
} else if (await videoElement.isVisible({ timeout: 10000 }).catch(() => false)) {
|
||||
console.log(' Video element visible in modal');
|
||||
// Give the video a moment to buffer/load
|
||||
@@ -636,93 +632,31 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/07f-video-state.png', fullPage: true });
|
||||
|
||||
// Click "Proceed to Instrumental Review" button in the modal
|
||||
// This button saves corrections and navigates to the instrumental selection UI
|
||||
const proceedBtn = reviewPage.getByRole('button', { name: /proceed to instrumental/i });
|
||||
await expect(proceedBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found "Proceed to Instrumental Review" button');
|
||||
// Click "Complete Review" button in the modal
|
||||
// This button has variant="contained", color="success", and endIcon={<CloudUpload />}
|
||||
const completeReviewBtn = reviewPage.getByRole('button', { name: /complete review/i });
|
||||
await expect(completeReviewBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found "Complete Review" button');
|
||||
|
||||
await proceedBtn.click();
|
||||
console.log(' Clicked "Proceed to Instrumental Review" button');
|
||||
await completeReviewBtn.click();
|
||||
console.log(' Clicked "Complete Review" button');
|
||||
|
||||
// Wait for the navigation to instrumental selection (happens via hash change)
|
||||
// The page navigates to #/{jobId}/instrumental
|
||||
await reviewPage.waitForTimeout(3000);
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/07g-after-proceed.png', fullPage: true });
|
||||
|
||||
// =========================================================================
|
||||
// STEP 8: Instrumental Selection (continuation of combined flow)
|
||||
// The page has navigated to the instrumental selection UI via hash routing
|
||||
// =========================================================================
|
||||
console.log('\n========================================');
|
||||
console.log('STEP 8: Instrumental Selection');
|
||||
console.log('========================================');
|
||||
|
||||
// Wait for the instrumental selection UI to load
|
||||
// The InstrumentalSelector component renders selection options
|
||||
console.log(' Waiting for instrumental selection UI to load...');
|
||||
|
||||
// Wait for either the selection options or the loading indicator to appear
|
||||
try {
|
||||
await reviewPage.waitForSelector('.selection-option, .selection-panel, [class*="selection"]', { timeout: 30000 });
|
||||
console.log(' Instrumental selection UI loaded');
|
||||
} catch {
|
||||
// If selector not found, check for loading state
|
||||
const loadingState = reviewPage.getByText(/loading instrumental/i);
|
||||
if (await loadingState.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
console.log(' Waiting for instrumental analysis to complete...');
|
||||
await expect(loadingState).not.toBeVisible({ timeout: 60000 });
|
||||
}
|
||||
}
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/08a-instrumental-opened.png', fullPage: true });
|
||||
console.log(' Instrumental selection UI opened');
|
||||
|
||||
// Select "Clean" instrumental option
|
||||
// The options have class "selection-option" and contain labels with "Clean" or "With Backing"
|
||||
const cleanOption = reviewPage.locator('.selection-option:has-text("Clean")').first();
|
||||
if (await cleanOption.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await cleanOption.click();
|
||||
console.log(' Selected "Clean" instrumental option');
|
||||
} else {
|
||||
// If we can't find "Clean" specifically, the first option might already be selected
|
||||
console.log(' Clean option not found - using default selection');
|
||||
}
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/08b-clean-selected.png', fullPage: true });
|
||||
|
||||
// Click the submit button: "✓ Confirm & Continue" (id="submit-btn")
|
||||
const submitBtn = reviewPage.locator('#submit-btn');
|
||||
await expect(submitBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found submit button');
|
||||
|
||||
await submitBtn.click();
|
||||
console.log(' Clicked "Confirm & Continue" button');
|
||||
|
||||
// Wait for submission to complete - the page shows a success screen then redirects
|
||||
// In cloud mode, after success it redirects to /app after a countdown
|
||||
// Wait for the submission to process
|
||||
// The handleSubmitToServer function calls apiClient.submitCorrections() then window.close()
|
||||
await reviewPage.waitForTimeout(5000);
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/08c-instrumental-submitted.png', fullPage: true });
|
||||
await reviewPage.screenshot({ path: 'test-results/07g-after-complete-review.png', fullPage: true });
|
||||
|
||||
// The page redirects to /app after completion, or may close
|
||||
// Wait a bit for the redirect/close to happen
|
||||
// The review page may close automatically after submission (window.close())
|
||||
// If it's still open, close it manually
|
||||
if (!reviewPage.isClosed()) {
|
||||
// Wait for potential redirect or success screen
|
||||
try {
|
||||
await reviewPage.waitForURL(/\/app/, { timeout: 10000 });
|
||||
console.log(' Redirected to /app after instrumental selection');
|
||||
} catch {
|
||||
// If no redirect, close manually
|
||||
await reviewPage.close();
|
||||
console.log(' Closed review UI');
|
||||
}
|
||||
await reviewPage.close();
|
||||
console.log(' Closed review UI');
|
||||
} else {
|
||||
console.log(' Review UI closed automatically after submission');
|
||||
}
|
||||
|
||||
// Refresh main app
|
||||
// Refresh main app to see updated status
|
||||
await page.bringToFront();
|
||||
const refreshBtn = page.getByRole('button', { name: /refresh/i });
|
||||
if (await refreshBtn.isVisible().catch(() => false)) {
|
||||
@@ -730,10 +664,121 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
await page.waitForTimeout(3000);
|
||||
}
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/08d-after-instrumental.png', fullPage: true }).catch(() => {
|
||||
// reviewPage may be closed already, that's OK
|
||||
});
|
||||
await page.screenshot({ path: 'test-results/08d-main-after-instrumental.png' });
|
||||
await page.screenshot({ path: 'test-results/07h-after-review-refresh.png' });
|
||||
console.log('STEP 7 COMPLETE: Lyrics review submitted via UI');
|
||||
|
||||
// =========================================================================
|
||||
// STEP 8: Wait for Video Rendering & Instrumental Selection (via UI - REAL USER FLOW)
|
||||
// The instrumental UI has: selection-option elements, submit button with id="submit-btn"
|
||||
// =========================================================================
|
||||
console.log('\n========================================');
|
||||
console.log('STEP 8: Wait for Video Rendering & Instrumental Selection');
|
||||
console.log('========================================');
|
||||
|
||||
// Wait for instrumental selection to become available
|
||||
console.log(' Waiting for video rendering and instrumental selection...');
|
||||
|
||||
let foundInstrumentalStatus = false;
|
||||
const renderStartTime = Date.now();
|
||||
|
||||
while (Date.now() - renderStartTime < TIMEOUTS.videoRendering) {
|
||||
// Refresh
|
||||
if (await refreshBtn.isVisible().catch(() => false)) {
|
||||
await refreshBtn.click();
|
||||
await page.waitForTimeout(2000);
|
||||
}
|
||||
|
||||
// Check job card status
|
||||
const statusText = await jobCard.textContent() || '';
|
||||
console.log(` Status: ${statusText.substring(0, 80)}...`);
|
||||
|
||||
// Look for instrumental selection UI availability
|
||||
// Users ALWAYS have to go through the instrumental review UI - there's no skip
|
||||
if (statusText.toLowerCase().includes('select instrumental') ||
|
||||
statusText.toLowerCase().includes('awaiting_instrumental') ||
|
||||
(statusText.toLowerCase().includes('action needed') && statusText.toLowerCase().includes('instrumental'))) {
|
||||
foundInstrumentalStatus = true;
|
||||
console.log(' Job ready for instrumental selection');
|
||||
break;
|
||||
}
|
||||
|
||||
if (statusText.toLowerCase().includes('failed')) {
|
||||
throw new Error('Job failed during video rendering');
|
||||
}
|
||||
|
||||
await page.waitForTimeout(10000);
|
||||
}
|
||||
|
||||
if (!foundInstrumentalStatus) {
|
||||
throw new Error('Timeout waiting for instrumental selection - users must always complete this step');
|
||||
}
|
||||
|
||||
{
|
||||
// Click the Select Instrumental button/link
|
||||
const instrumentalLink = jobCard.getByRole('link', { name: /select instrumental/i });
|
||||
await expect(instrumentalLink).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
|
||||
const instrumentalUrl = await instrumentalLink.getAttribute('href');
|
||||
console.log(` Instrumental URL: ${instrumentalUrl}`);
|
||||
|
||||
// Open instrumental UI in new page
|
||||
const instrumentalPage = await context.newPage();
|
||||
// Handle both absolute and relative URLs
|
||||
const fullUrl = instrumentalUrl?.startsWith('http') ? instrumentalUrl : `${PROD_URL}${instrumentalUrl}`;
|
||||
// Use 'load' instead of 'networkidle' - instrumental page has ongoing audio requests
|
||||
// that prevent networkidle from ever being reached
|
||||
await instrumentalPage.goto(fullUrl, { waitUntil: 'load' });
|
||||
await instrumentalPage.waitForTimeout(3000);
|
||||
|
||||
await instrumentalPage.screenshot({ path: 'test-results/08a-instrumental-opened.png', fullPage: true });
|
||||
console.log(' Instrumental selection UI opened');
|
||||
|
||||
// Wait for the page to fully load - the instrumental UI renders selection options
|
||||
await instrumentalPage.waitForSelector('.selection-option, .selection-panel', { timeout: 30000 });
|
||||
|
||||
// Select "Clean" instrumental option
|
||||
// The options have class "selection-option" and contain labels with "Clean" or "With Backing"
|
||||
const cleanOption = instrumentalPage.locator('.selection-option:has-text("Clean")').first();
|
||||
if (await cleanOption.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await cleanOption.click();
|
||||
console.log(' Selected "Clean" instrumental option');
|
||||
} else {
|
||||
// If we can't find "Clean" specifically, the first option might already be selected
|
||||
console.log(' Clean option not found - using default selection');
|
||||
}
|
||||
|
||||
await instrumentalPage.screenshot({ path: 'test-results/08b-clean-selected.png', fullPage: true });
|
||||
|
||||
// Click the submit button: "✓ Confirm & Continue" (id="submit-btn")
|
||||
const submitBtn = instrumentalPage.locator('#submit-btn');
|
||||
await expect(submitBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found submit button');
|
||||
|
||||
await submitBtn.click();
|
||||
console.log(' Clicked "Confirm & Continue" button');
|
||||
|
||||
// Wait for submission to complete - the page shows a success screen then auto-closes
|
||||
await instrumentalPage.waitForTimeout(5000);
|
||||
|
||||
await instrumentalPage.screenshot({ path: 'test-results/08c-instrumental-submitted.png', fullPage: true });
|
||||
|
||||
// The page may auto-close after submission
|
||||
if (!instrumentalPage.isClosed()) {
|
||||
await instrumentalPage.close();
|
||||
console.log(' Closed instrumental UI');
|
||||
} else {
|
||||
console.log(' Instrumental UI closed automatically after submission');
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh main app
|
||||
await page.bringToFront();
|
||||
if (await refreshBtn.isVisible().catch(() => false)) {
|
||||
await refreshBtn.click();
|
||||
await page.waitForTimeout(3000);
|
||||
}
|
||||
|
||||
await page.screenshot({ path: 'test-results/08d-after-instrumental.png' });
|
||||
console.log('STEP 8 COMPLETE: Instrumental selection handled via UI');
|
||||
|
||||
// =========================================================================
|
||||
|
||||
@@ -190,51 +190,43 @@ test.describe('Lyrics Review Only - Focused Test', () => {
|
||||
|
||||
await page.screenshot({ path: 'test-results/focused-06-video-state.png', fullPage: true });
|
||||
|
||||
// Click "Proceed to Instrumental Review" button in the modal
|
||||
// (Previously "Complete Review" - changed with combined review UI)
|
||||
console.log('6. Looking for "Proceed to Instrumental Review" button...');
|
||||
// Click "Complete Review" button in the modal
|
||||
console.log('6. Looking for "Complete Review" button...');
|
||||
|
||||
// List all buttons in the modal
|
||||
const modalButtons = await previewModal.getByRole('button').allTextContents();
|
||||
console.log(` Modal buttons: ${modalButtons.join(', ')}`);
|
||||
|
||||
const proceedBtn = page.getByRole('button', { name: /proceed to instrumental/i });
|
||||
const completeReviewBtn = page.getByRole('button', { name: /complete review/i });
|
||||
|
||||
if (await proceedBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
console.log(' Found "Proceed to Instrumental Review" button - clicking...');
|
||||
await proceedBtn.click();
|
||||
console.log(' Clicked "Proceed to Instrumental Review" button');
|
||||
if (await completeReviewBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
console.log(' Found "Complete Review" button - clicking...');
|
||||
await completeReviewBtn.click();
|
||||
console.log(' Clicked "Complete Review" button');
|
||||
} else {
|
||||
// Try to find any submit-like button (for backwards compatibility)
|
||||
const submitBtn = previewModal.getByRole('button', { name: /submit|confirm|done|save|complete/i });
|
||||
// Try to find any submit-like button
|
||||
const submitBtn = previewModal.getByRole('button', { name: /submit|confirm|done|save/i });
|
||||
if (await submitBtn.isVisible().catch(() => false)) {
|
||||
const btnText = await submitBtn.textContent();
|
||||
console.log(` Found alternative submit button: "${btnText}" - clicking...`);
|
||||
await submitBtn.click();
|
||||
} else {
|
||||
console.log(' Could not find Proceed to Instrumental or similar button');
|
||||
await page.screenshot({ path: 'test-results/focused-07-no-proceed-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Proceed to Instrumental Review button');
|
||||
console.log(' Could not find Complete Review or similar button');
|
||||
await page.screenshot({ path: 'test-results/focused-07-no-complete-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Complete Review button');
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for navigation to instrumental selection
|
||||
console.log('7. Waiting for navigation to instrumental selection...');
|
||||
// Wait for the submission to process
|
||||
console.log('7. Waiting for submission to complete...');
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
await page.screenshot({ path: 'test-results/focused-08-after-proceed.png', fullPage: true });
|
||||
|
||||
// Note: This focused test only covers the lyrics review portion.
|
||||
// The instrumental selection happens next on the same page via hash navigation.
|
||||
// For full flow testing, use happy-path-real-user.spec.ts
|
||||
await page.screenshot({ path: 'test-results/focused-08-after-submit.png', fullPage: true });
|
||||
|
||||
console.log('');
|
||||
console.log('========================================');
|
||||
console.log('FOCUSED TEST COMPLETE');
|
||||
console.log('========================================');
|
||||
console.log('NOTE: This test completes the lyrics review step only.');
|
||||
console.log(' The instrumental selection UI should now be visible on this page.');
|
||||
console.log(' For the full flow, use happy-path-real-user.spec.ts');
|
||||
});
|
||||
|
||||
test('Add Reference Lyrics via API', async ({ page, context, request }) => {
|
||||
|
||||
@@ -256,28 +256,27 @@ async function testLyricsReview(page: any, context: any, jobId: string) {
|
||||
console.log(' WARNING: No video element found');
|
||||
}
|
||||
|
||||
// Click "Proceed to Instrumental Review" button (previously "Complete Review")
|
||||
console.log('10. Clicking "Proceed to Instrumental Review" button...');
|
||||
// Click "Complete Review" button
|
||||
console.log('10. Clicking "Complete Review" button...');
|
||||
const modalButtons = await previewModal.getByRole('button').allTextContents();
|
||||
console.log(` Modal buttons: ${modalButtons.join(', ')}`);
|
||||
|
||||
const proceedBtn = reviewPage.getByRole('button', { name: /proceed to instrumental/i });
|
||||
const completeReviewBtn = reviewPage.getByRole('button', { name: /complete review/i });
|
||||
|
||||
if (await proceedBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await proceedBtn.click();
|
||||
console.log(' Clicked Proceed to Instrumental Review button');
|
||||
if (await completeReviewBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await completeReviewBtn.click();
|
||||
console.log(' Clicked Complete Review button');
|
||||
} else {
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-09-no-proceed-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Proceed to Instrumental Review button');
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-09-no-complete-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Complete Review button');
|
||||
}
|
||||
|
||||
// Wait for navigation to instrumental selection (happens via hash change on same page)
|
||||
console.log('11. Waiting for navigation to instrumental selection...');
|
||||
// Wait for submission
|
||||
console.log('11. Waiting for submission...');
|
||||
await reviewPage.waitForTimeout(5000);
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-10-after-proceed.png', fullPage: true });
|
||||
console.log(' Lyrics review step completed - now on instrumental selection!');
|
||||
console.log(' NOTE: With combined UI, instrumental selection is on the same page.');
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-10-after-submit.png', fullPage: true });
|
||||
console.log(' Lyrics review completed successfully!');
|
||||
|
||||
await reviewPage.close();
|
||||
}
|
||||
|
||||
@@ -1454,59 +1454,6 @@ export const adminApi = {
|
||||
return handleResponse(response);
|
||||
},
|
||||
|
||||
/**
|
||||
* Regenerate title and end screens with current artist/title metadata (admin only).
|
||||
* Use when you've edited artist/title and need screens to reflect the new metadata.
|
||||
*/
|
||||
async regenerateScreens(jobId: string): Promise<RegenerateScreensResponse> {
|
||||
const response = await fetch(
|
||||
`${API_BASE_URL}/api/admin/jobs/${jobId}/regenerate-screens`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: getAuthHeaders()
|
||||
}
|
||||
);
|
||||
return handleResponse(response);
|
||||
},
|
||||
|
||||
/**
|
||||
* Fully restart a job from the beginning (admin only).
|
||||
* Unlike reset (which just changes state), restart actually triggers workers.
|
||||
*/
|
||||
async restartJob(jobId: string, options: RestartJobRequest): Promise<RestartJobResponse> {
|
||||
const response = await fetch(
|
||||
`${API_BASE_URL}/api/admin/jobs/${jobId}/restart`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...getAuthHeaders()
|
||||
},
|
||||
body: JSON.stringify(options),
|
||||
}
|
||||
);
|
||||
return handleResponse(response);
|
||||
},
|
||||
|
||||
/**
|
||||
* Override the audio source for a job (admin only).
|
||||
* Switch from YouTube URL to audio search mode.
|
||||
*/
|
||||
async overrideAudioSource(jobId: string, request: OverrideAudioSourceRequest): Promise<OverrideAudioSourceResponse> {
|
||||
const response = await fetch(
|
||||
`${API_BASE_URL}/api/admin/jobs/${jobId}/override-audio-source`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...getAuthHeaders()
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
}
|
||||
);
|
||||
return handleResponse(response);
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Rate Limits API
|
||||
// =========================================================================
|
||||
@@ -1745,46 +1692,6 @@ export interface ClearWorkersResponse {
|
||||
cleared_keys: string[];
|
||||
}
|
||||
|
||||
export interface RegenerateScreensResponse {
|
||||
status: string;
|
||||
job_id: string;
|
||||
message: string;
|
||||
previous_screens_deleted: boolean;
|
||||
worker_triggered: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface RestartJobRequest {
|
||||
preserve_audio_stems: boolean;
|
||||
delete_outputs: boolean;
|
||||
}
|
||||
|
||||
export interface RestartJobResponse {
|
||||
status: string;
|
||||
job_id: string;
|
||||
message: string;
|
||||
previous_status: string;
|
||||
new_status: string;
|
||||
cleared_data: string[];
|
||||
deleted_gcs_paths: string[];
|
||||
workers_triggered: string[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface OverrideAudioSourceRequest {
|
||||
source_type: "audio_search";
|
||||
}
|
||||
|
||||
export interface OverrideAudioSourceResponse {
|
||||
status: string;
|
||||
job_id: string;
|
||||
message: string;
|
||||
previous_source: string;
|
||||
new_source: string;
|
||||
cleared_data: string[];
|
||||
new_status: string;
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Lyrics Review API endpoints
|
||||
// ==========================================================================
|
||||
|
||||
@@ -37,7 +37,7 @@ from config import (
|
||||
DiskSizes,
|
||||
)
|
||||
from modules import database, storage as storage_module, artifact_registry, secrets
|
||||
from modules import cloud_tasks, cloud_run, monitoring, networking, runner_manager
|
||||
from modules import cloud_tasks, cloud_run, monitoring, networking
|
||||
from modules.iam import backend_sa, github_actions_sa, claude_automation_sa, worker_sas
|
||||
from compute import encoding_worker_vm, github_runners
|
||||
|
||||
@@ -249,18 +249,8 @@ encoding_worker_instance = encoding_worker_vm.create_encoding_worker_vm(
|
||||
encoding_worker_firewall = encoding_worker_vm.create_encoding_worker_firewall()
|
||||
|
||||
# GitHub Runners (CI/CD self-hosted runners)
|
||||
# Create Cloud NAT for outbound internet access (no external IPs needed)
|
||||
github_runners_router, github_runners_nat = github_runners.create_cloud_nat()
|
||||
|
||||
# Create runner VMs (Spot instances for 60-91% cost savings)
|
||||
github_runner_vms = github_runners.create_github_runners(
|
||||
github_runner_sa, all_secrets["github-runner-pat"], github_runners_nat
|
||||
)
|
||||
|
||||
# Create runner manager (auto-start/stop VMs based on CI activity)
|
||||
runner_manager_resources = runner_manager.create_runner_manager_resources(
|
||||
all_secrets["github-webhook-secret"],
|
||||
all_secrets["github-runner-pat"],
|
||||
github_runner_sa, all_secrets["github-runner-pat"]
|
||||
)
|
||||
|
||||
|
||||
@@ -344,8 +334,3 @@ pulumi.export("encoding_worker_vm_name", encoding_worker_instance.name)
|
||||
# GitHub runners
|
||||
pulumi.export("github_runner_vm_names", [vm.name for vm in github_runner_vms])
|
||||
pulumi.export("github_runner_service_account", github_runner_sa.email)
|
||||
pulumi.export("github_runners_nat", github_runners_nat.name)
|
||||
|
||||
# GitHub runner manager (auto-start/stop)
|
||||
pulumi.export("runner_manager_function_url", runner_manager_resources["function"].url)
|
||||
pulumi.export("runner_manager_scheduler_job", runner_manager_resources["scheduler_job"].name)
|
||||
@@ -11,12 +11,11 @@ from .encoding_worker_vm import (
|
||||
create_encoding_worker_ip,
|
||||
create_encoding_worker_firewall,
|
||||
)
|
||||
from .github_runners import create_github_runners, create_cloud_nat
|
||||
from .github_runners import create_github_runners
|
||||
|
||||
__all__ = [
|
||||
"create_encoding_worker_vm",
|
||||
"create_encoding_worker_ip",
|
||||
"create_encoding_worker_firewall",
|
||||
"create_github_runners",
|
||||
"create_cloud_nat",
|
||||
]
|
||||
@@ -1,82 +1,30 @@
|
||||
"""
|
||||
GitHub Actions Self-Hosted Runners on GCP.
|
||||
GitHub Actions self-hosted runner VMs.
|
||||
|
||||
Creates Spot/Preemptible VMs that register as self-hosted runners
|
||||
at the organization level. Uses Spot instances for 60-91% cost savings.
|
||||
|
||||
Resources created:
|
||||
- Cloud Router and Cloud NAT (for outbound internet without external IPs)
|
||||
- N Compute Engine VMs (Spot instances)
|
||||
- Each VM runs the GitHub Actions runner agent
|
||||
|
||||
The VMs are managed by the runner_manager Cloud Function which:
|
||||
- Starts VMs when CI jobs are queued (via GitHub webhook)
|
||||
- Stops VMs when idle for too long (via Cloud Scheduler)
|
||||
Manages multiple VM instances that serve as self-hosted runners
|
||||
for GitHub Actions CI/CD workflows.
|
||||
"""
|
||||
|
||||
import pulumi
|
||||
from pulumi_gcp import compute, secretmanager, serviceaccount
|
||||
from pulumi_gcp import compute, serviceaccount, secretmanager
|
||||
|
||||
from config import REGION, ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
|
||||
from config import ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
|
||||
from .startup_scripts import read_script
|
||||
|
||||
|
||||
def create_cloud_nat() -> tuple[compute.Router, compute.RouterNat]:
|
||||
"""
|
||||
Create Cloud Router and Cloud NAT for outbound internet access.
|
||||
|
||||
This allows VMs without external IPs to access the internet,
|
||||
which saves costs and improves security.
|
||||
|
||||
Returns:
|
||||
tuple: (router, nat) resources
|
||||
"""
|
||||
# Create Cloud Router
|
||||
router = compute.Router(
|
||||
"github-runners-router",
|
||||
name="github-runners-router",
|
||||
region=REGION,
|
||||
network="default",
|
||||
description="Router for GitHub Actions runner NAT",
|
||||
)
|
||||
|
||||
# Create Cloud NAT
|
||||
nat = compute.RouterNat(
|
||||
"github-runners-nat",
|
||||
name="github-runners-nat",
|
||||
router=router.name,
|
||||
region=REGION,
|
||||
nat_ip_allocate_option="AUTO_ONLY",
|
||||
source_subnetwork_ip_ranges_to_nat="ALL_SUBNETWORKS_ALL_IP_RANGES",
|
||||
log_config=compute.RouterNatLogConfigArgs(
|
||||
enable=True,
|
||||
filter="ERRORS_ONLY",
|
||||
),
|
||||
opts=pulumi.ResourceOptions(depends_on=[router]),
|
||||
)
|
||||
|
||||
return router, nat
|
||||
|
||||
|
||||
def create_github_runners(
|
||||
service_account: serviceaccount.Account,
|
||||
runner_pat_secret: secretmanager.Secret,
|
||||
nat: compute.RouterNat,
|
||||
) -> list[compute.Instance]:
|
||||
"""
|
||||
Create GitHub Actions self-hosted runner VM instances.
|
||||
|
||||
Creates NUM_GITHUB_RUNNERS Spot VMs, each configured as a GitHub Actions
|
||||
Creates NUM_GITHUB_RUNNERS VMs, each configured as a GitHub Actions
|
||||
self-hosted runner for the nomadkaraoke organization (available to all repos).
|
||||
|
||||
Uses Spot instances (preemptible) for 60-91% cost savings.
|
||||
VMs start in TERMINATED state and are started by the runner_manager
|
||||
Cloud Function when CI jobs are queued.
|
||||
|
||||
Args:
|
||||
service_account: The GitHub runner service account.
|
||||
runner_pat_secret: The Secret Manager secret containing the GitHub PAT.
|
||||
nat: Cloud NAT for outbound internet access (VMs have no external IPs).
|
||||
|
||||
Returns:
|
||||
list[compute.Instance]: List of runner VM instances.
|
||||
@@ -85,91 +33,33 @@ def create_github_runners(
|
||||
|
||||
runner_vms = []
|
||||
for i in range(1, NUM_GITHUB_RUNNERS + 1):
|
||||
instance_name = f"github-runner-{i}"
|
||||
|
||||
vm = compute.Instance(
|
||||
instance_name,
|
||||
name=instance_name,
|
||||
f"github-runner-{i}",
|
||||
name=f"github-runner-{i}",
|
||||
machine_type=MachineTypes.GITHUB_RUNNER,
|
||||
zone=ZONE,
|
||||
# Boot disk configuration
|
||||
boot_disk=compute.InstanceBootDiskArgs(
|
||||
initialize_params=compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=DiskSizes.GITHUB_RUNNER,
|
||||
type="pd-ssd", # SSD for faster Docker builds
|
||||
type="pd-ssd",
|
||||
),
|
||||
auto_delete=True,
|
||||
),
|
||||
# Network configuration - no external IP (uses Cloud NAT)
|
||||
# This saves ~$3/month per VM and improves security
|
||||
network_interfaces=[
|
||||
compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
# No access_configs = no external IP
|
||||
# Outbound internet via Cloud NAT
|
||||
)
|
||||
],
|
||||
# Service account for GCP API access
|
||||
network_interfaces=[compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[compute.InstanceNetworkInterfaceAccessConfigArgs()],
|
||||
)],
|
||||
service_account=compute.InstanceServiceAccountArgs(
|
||||
email=service_account.email,
|
||||
scopes=["https://www.googleapis.com/auth/cloud-platform"],
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
# Spot/Preemptible instance configuration (60-91% cheaper)
|
||||
# Preemptible VMs may be terminated at any time but cost much less.
|
||||
# The startup script re-registers the runner on boot.
|
||||
scheduling=compute.InstanceSchedulingArgs(
|
||||
preemptible=True,
|
||||
automatic_restart=False, # Required for preemptible
|
||||
on_host_maintenance="TERMINATE", # Required for preemptible
|
||||
),
|
||||
# Metadata for startup script
|
||||
metadata={
|
||||
"github-runner-pat-secret": "github-runner-pat",
|
||||
"github-org": "nomadkaraoke",
|
||||
"runner-labels": "self-hosted,linux,x64,gcp,large-disk",
|
||||
},
|
||||
metadata_startup_script=startup_script,
|
||||
# Labels for organization and cost tracking
|
||||
labels={
|
||||
"purpose": "github-runner",
|
||||
"managed-by": "pulumi",
|
||||
},
|
||||
tags=["github-runner"],
|
||||
# Allow stopping for updates
|
||||
allow_stopping_for_update=True,
|
||||
# Deletion protection disabled (runners are ephemeral)
|
||||
deletion_protection=False,
|
||||
opts=pulumi.ResourceOptions(
|
||||
depends_on=[runner_pat_secret, nat],
|
||||
# Runners can be replaced without affecting other infrastructure
|
||||
delete_before_replace=True,
|
||||
depends_on=[runner_pat_secret],
|
||||
),
|
||||
)
|
||||
|
||||
runner_vms.append(vm)
|
||||
|
||||
return runner_vms
|
||||
|
||||
|
||||
def create_instance_group_for_restart(
|
||||
instances: list[compute.Instance],
|
||||
) -> compute.InstanceGroup:
|
||||
"""
|
||||
Create an unmanaged instance group for the runner VMs.
|
||||
|
||||
This enables bulk operations like restarting all runners.
|
||||
|
||||
Args:
|
||||
instances: List of runner VM instances.
|
||||
|
||||
Returns:
|
||||
compute.InstanceGroup: The instance group.
|
||||
"""
|
||||
return compute.InstanceGroup(
|
||||
"github-runners-group",
|
||||
name="github-runners",
|
||||
description="Self-hosted GitHub Actions runners",
|
||||
zone=ZONE,
|
||||
instances=[instance.self_link for instance in instances],
|
||||
)
|
||||
@@ -87,29 +87,11 @@ class QueueConfigs:
|
||||
|
||||
|
||||
# Number of self-hosted GitHub Action runners
|
||||
# Reduced from 20 to 3 - runners auto-scale with start/stop management
|
||||
NUM_GITHUB_RUNNERS = 3
|
||||
NUM_GITHUB_RUNNERS = 20
|
||||
|
||||
# Google Drive folder ID for validator (public share folder)
|
||||
GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
|
||||
|
||||
|
||||
class RunnerManagerConfig:
|
||||
"""Configuration for GitHub runner auto-start/stop management."""
|
||||
|
||||
FUNCTION_NAME = "github-runner-manager"
|
||||
FUNCTION_MEMORY = "256M"
|
||||
FUNCTION_TIMEOUT = 300 # 5 minutes
|
||||
IDLE_TIMEOUT_HOURS = 1 # Shorter than Aquarius (3h) since we have more CI activity
|
||||
IDLE_CHECK_SCHEDULE = "*/15 * * * *" # Every 15 minutes
|
||||
|
||||
|
||||
class SecretNames:
|
||||
"""Secret Manager secret names."""
|
||||
|
||||
GITHUB_RUNNER_PAT = "github-runner-pat"
|
||||
GITHUB_WEBHOOK_SECRET = "github-webhook-secret"
|
||||
|
||||
# GitHub repository for runner registration
|
||||
GITHUB_REPO_OWNER = "nomadkaraoke"
|
||||
GITHUB_REPO_NAME = "karaoke-gen"
|
||||
@@ -1,417 +0,0 @@
|
||||
"""
|
||||
GitHub Actions Runner Manager Cloud Function.
|
||||
|
||||
Handles two types of triggers:
|
||||
1. GitHub Webhook (workflow_job.queued) - Starts runner VMs when CI jobs are queued
|
||||
2. Cloud Scheduler (every 15 min) - Stops idle runner VMs after timeout
|
||||
|
||||
Environment variables:
|
||||
- GCP_PROJECT: GCP project ID
|
||||
- GCP_ZONE: Zone where runner VMs are located
|
||||
- WEBHOOK_SECRET_NAME: Secret Manager secret name for webhook verification
|
||||
- RUNNER_PAT_SECRET_NAME: Secret Manager secret name for GitHub PAT
|
||||
- IDLE_TIMEOUT_HOURS: Hours of inactivity before stopping runners (default: 1)
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import functions_framework
|
||||
from flask import Request
|
||||
from google.cloud import compute_v1, secretmanager
|
||||
|
||||
# Configuration from environment
|
||||
PROJECT_ID = os.environ.get("GCP_PROJECT", "nomadkaraoke")
|
||||
ZONE = os.environ.get("GCP_ZONE", "us-central1-a")
|
||||
WEBHOOK_SECRET_NAME = os.environ.get("WEBHOOK_SECRET_NAME", "github-webhook-secret")
|
||||
RUNNER_PAT_SECRET_NAME = os.environ.get("RUNNER_PAT_SECRET_NAME", "github-runner-pat")
|
||||
IDLE_TIMEOUT_HOURS = int(os.environ.get("IDLE_TIMEOUT_HOURS", "1"))
|
||||
RUNNER_PREFIX = "github-runner-"
|
||||
NUM_RUNNERS = 3
|
||||
|
||||
# Lazy-loaded clients and secrets
|
||||
_compute_client = None
|
||||
_secret_client = None
|
||||
_webhook_secret = None
|
||||
_github_pat = None
|
||||
|
||||
|
||||
def get_compute_client() -> compute_v1.InstancesClient:
|
||||
"""Get or create the Compute Engine client."""
|
||||
global _compute_client
|
||||
if _compute_client is None:
|
||||
_compute_client = compute_v1.InstancesClient()
|
||||
return _compute_client
|
||||
|
||||
|
||||
def get_secret_client() -> secretmanager.SecretManagerServiceClient:
|
||||
"""Get or create the Secret Manager client."""
|
||||
global _secret_client
|
||||
if _secret_client is None:
|
||||
_secret_client = secretmanager.SecretManagerServiceClient()
|
||||
return _secret_client
|
||||
|
||||
|
||||
def get_webhook_secret() -> str:
|
||||
"""Get the webhook secret from Secret Manager (cached)."""
|
||||
global _webhook_secret
|
||||
if _webhook_secret is None:
|
||||
client = get_secret_client()
|
||||
name = f"projects/{PROJECT_ID}/secrets/{WEBHOOK_SECRET_NAME}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
_webhook_secret = response.payload.data.decode("utf-8").strip()
|
||||
return _webhook_secret
|
||||
|
||||
|
||||
def get_github_pat() -> str:
|
||||
"""Get the GitHub PAT from Secret Manager (cached)."""
|
||||
global _github_pat
|
||||
if _github_pat is None:
|
||||
client = get_secret_client()
|
||||
name = f"projects/{PROJECT_ID}/secrets/{RUNNER_PAT_SECRET_NAME}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
_github_pat = response.payload.data.decode("utf-8").strip()
|
||||
return _github_pat
|
||||
|
||||
|
||||
def verify_webhook_signature(request: Request) -> bool:
|
||||
"""
|
||||
Verify the GitHub webhook signature.
|
||||
|
||||
GitHub sends a signature in the X-Hub-Signature-256 header.
|
||||
We compute the expected signature and compare.
|
||||
"""
|
||||
signature_header = request.headers.get("X-Hub-Signature-256")
|
||||
if not signature_header:
|
||||
print("Missing X-Hub-Signature-256 header")
|
||||
return False
|
||||
|
||||
# Get the raw body for signature verification
|
||||
body = request.get_data()
|
||||
secret = get_webhook_secret()
|
||||
|
||||
# Compute expected signature
|
||||
expected_signature = (
|
||||
"sha256="
|
||||
+ hmac.new(
|
||||
secret.encode("utf-8"),
|
||||
body,
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
)
|
||||
|
||||
# Constant-time comparison to prevent timing attacks
|
||||
return hmac.compare_digest(expected_signature, signature_header)
|
||||
|
||||
|
||||
def get_runner_instances() -> list[compute_v1.Instance]:
|
||||
"""Get all GitHub runner instances."""
|
||||
client = get_compute_client()
|
||||
instances = []
|
||||
|
||||
for i in range(1, NUM_RUNNERS + 1):
|
||||
instance_name = f"{RUNNER_PREFIX}{i}"
|
||||
try:
|
||||
instance = client.get(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
)
|
||||
instances.append(instance)
|
||||
except Exception as e:
|
||||
print(f"Error getting instance {instance_name}: {e}")
|
||||
|
||||
return instances
|
||||
|
||||
|
||||
def start_runners() -> dict:
|
||||
"""
|
||||
Start any TERMINATED runner VMs.
|
||||
|
||||
Returns a dict with lists of started, already_running, and failed instances.
|
||||
"""
|
||||
client = get_compute_client()
|
||||
result = {"started": [], "already_running": [], "failed": []}
|
||||
|
||||
for i in range(1, NUM_RUNNERS + 1):
|
||||
instance_name = f"{RUNNER_PREFIX}{i}"
|
||||
try:
|
||||
instance = client.get(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
)
|
||||
|
||||
if instance.status == "TERMINATED":
|
||||
print(f"Starting {instance_name}...")
|
||||
operation = client.start(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
)
|
||||
# Wait for the operation to complete (timeout after 5 min)
|
||||
try:
|
||||
operation.result(timeout=300)
|
||||
print(f"{instance_name} started successfully")
|
||||
result["started"].append(instance_name)
|
||||
except Exception as op_error:
|
||||
print(f"Start operation failed for {instance_name}: {op_error}")
|
||||
result["failed"].append(instance_name)
|
||||
elif instance.status == "RUNNING":
|
||||
result["already_running"].append(instance_name)
|
||||
else:
|
||||
print(f"{instance_name} is in state {instance.status}")
|
||||
result["already_running"].append(instance_name)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error with {instance_name}: {e}")
|
||||
result["failed"].append(instance_name)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def update_instance_metadata(instance_name: str, key: str, value: str) -> None:
|
||||
"""Update a single metadata key on an instance."""
|
||||
client = get_compute_client()
|
||||
|
||||
# Get current instance to get fingerprint
|
||||
instance = client.get(project=PROJECT_ID, zone=ZONE, instance=instance_name)
|
||||
current_metadata = instance.metadata
|
||||
|
||||
# Build new items list, replacing or adding the key
|
||||
new_items = []
|
||||
found = False
|
||||
for item in current_metadata.items:
|
||||
if item.key == key:
|
||||
new_items.append(compute_v1.Items(key=key, value=value))
|
||||
found = True
|
||||
else:
|
||||
new_items.append(item)
|
||||
|
||||
if not found:
|
||||
new_items.append(compute_v1.Items(key=key, value=value))
|
||||
|
||||
# Update metadata
|
||||
metadata = compute_v1.Metadata(
|
||||
fingerprint=current_metadata.fingerprint,
|
||||
items=new_items,
|
||||
)
|
||||
|
||||
client.set_metadata(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
metadata_resource=metadata,
|
||||
)
|
||||
|
||||
|
||||
def get_instance_metadata(instance: compute_v1.Instance, key: str) -> str | None:
|
||||
"""Get a metadata value from an instance."""
|
||||
if not instance.metadata or not instance.metadata.items:
|
||||
return None
|
||||
|
||||
for item in instance.metadata.items:
|
||||
if item.key == key:
|
||||
return item.value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def check_github_for_pending_jobs() -> bool:
|
||||
"""
|
||||
Check GitHub API for any pending/queued jobs for our runners.
|
||||
|
||||
Returns True if there are jobs waiting for our self-hosted runners.
|
||||
"""
|
||||
import urllib.request
|
||||
|
||||
pat = get_github_pat()
|
||||
org = "nomadkaraoke"
|
||||
|
||||
# Check for queued workflow runs in karaoke-gen repo
|
||||
url = f"https://api.github.com/repos/{org}/karaoke-gen/actions/runs?status=queued"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {pat}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
if data.get("total_count", 0) > 0:
|
||||
print(f"Found {data['total_count']} queued runs in karaoke-gen")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error checking GitHub API: {e}")
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def check_and_stop_idle_runners() -> dict:
|
||||
"""
|
||||
Check for idle runners and stop them if idle for IDLE_TIMEOUT_HOURS.
|
||||
|
||||
A runner is considered idle if:
|
||||
1. It's RUNNING
|
||||
2. No jobs are pending/queued on GitHub
|
||||
3. last-activity metadata is older than IDLE_TIMEOUT_HOURS
|
||||
|
||||
Returns a dict with stopped and kept instances.
|
||||
"""
|
||||
result = {"stopped": [], "kept": [], "errors": []}
|
||||
|
||||
# First check if there are any pending jobs
|
||||
if check_github_for_pending_jobs():
|
||||
print("Jobs are pending - keeping all runners active")
|
||||
# Update last-activity on all running instances
|
||||
for instance in get_runner_instances():
|
||||
if instance.status == "RUNNING":
|
||||
result["kept"].append(instance.name)
|
||||
# Refresh activity timestamp since there are pending jobs
|
||||
try:
|
||||
update_instance_metadata(
|
||||
instance.name,
|
||||
"last-activity",
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error updating metadata for {instance.name}: {e}")
|
||||
return result
|
||||
|
||||
# No pending jobs - check each instance for idle timeout
|
||||
client = get_compute_client()
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for instance in get_runner_instances():
|
||||
if instance.status != "RUNNING":
|
||||
continue
|
||||
|
||||
# Get last-activity timestamp
|
||||
last_activity_str = get_instance_metadata(instance, "last-activity")
|
||||
|
||||
if not last_activity_str:
|
||||
# No activity recorded - set it now and don't stop
|
||||
print(f"{instance.name}: No last-activity metadata, setting now")
|
||||
try:
|
||||
update_instance_metadata(
|
||||
instance.name,
|
||||
"last-activity",
|
||||
now.isoformat(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error setting metadata for {instance.name}: {e}")
|
||||
result["kept"].append(instance.name)
|
||||
continue
|
||||
|
||||
try:
|
||||
last_activity = datetime.fromisoformat(last_activity_str.replace("Z", "+00:00"))
|
||||
idle_hours = (now - last_activity).total_seconds() / 3600
|
||||
|
||||
if idle_hours >= IDLE_TIMEOUT_HOURS:
|
||||
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, stopping...")
|
||||
client.stop(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance.name,
|
||||
)
|
||||
result["stopped"].append(instance.name)
|
||||
else:
|
||||
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, keeping...")
|
||||
result["kept"].append(instance.name)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing {instance.name}: {e}")
|
||||
result["errors"].append(instance.name)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@functions_framework.http
|
||||
def handle_request(request: Request):
|
||||
"""
|
||||
Main entry point for the Cloud Function.
|
||||
|
||||
Handles:
|
||||
1. Cloud Scheduler triggers (action=check_idle parameter)
|
||||
2. GitHub webhook events (workflow_job.queued)
|
||||
"""
|
||||
# Check if this is a scheduler trigger (idle check)
|
||||
action = request.args.get("action")
|
||||
if action == "check_idle":
|
||||
print("Scheduler trigger: checking for idle runners")
|
||||
result = check_and_stop_idle_runners()
|
||||
return json.dumps(result), 200, {"Content-Type": "application/json"}
|
||||
|
||||
# Otherwise, this should be a GitHub webhook
|
||||
# Verify the webhook signature
|
||||
if not verify_webhook_signature(request):
|
||||
print("Webhook signature verification failed")
|
||||
return "Unauthorized", 401
|
||||
|
||||
# Get the event type
|
||||
event = request.headers.get("X-GitHub-Event")
|
||||
if not event:
|
||||
return "Missing X-GitHub-Event header", 400
|
||||
|
||||
# We only care about workflow_job events
|
||||
if event != "workflow_job":
|
||||
return "OK - event ignored", 200
|
||||
|
||||
# Parse the payload
|
||||
try:
|
||||
payload = request.get_json()
|
||||
except Exception as e:
|
||||
print(f"Error parsing JSON: {e}")
|
||||
return "Invalid JSON", 400
|
||||
|
||||
action = payload.get("action")
|
||||
job = payload.get("workflow_job", {})
|
||||
labels = job.get("labels", [])
|
||||
|
||||
print(f"Received workflow_job.{action} event")
|
||||
print(f"Job labels: {labels}")
|
||||
|
||||
# Check if this job requires our self-hosted runners
|
||||
if "self-hosted" not in labels:
|
||||
print("Job doesn't require self-hosted runners, ignoring")
|
||||
return "OK - not for self-hosted runners", 200
|
||||
|
||||
# Handle different actions
|
||||
if action == "queued":
|
||||
# Job is queued - ensure runners are started
|
||||
print("Job queued - starting runners if needed")
|
||||
result = start_runners()
|
||||
return json.dumps(result), 200, {"Content-Type": "application/json"}
|
||||
|
||||
elif action == "in_progress":
|
||||
# Job started - update activity timestamp on all running instances
|
||||
print("Job in progress - updating activity timestamps")
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
for instance in get_runner_instances():
|
||||
if instance.status == "RUNNING":
|
||||
try:
|
||||
update_instance_metadata(instance.name, "last-activity", now)
|
||||
except Exception as e:
|
||||
print(f"Error updating {instance.name}: {e}")
|
||||
return "OK", 200
|
||||
|
||||
elif action == "completed":
|
||||
# Job completed - update activity timestamp
|
||||
print("Job completed - updating activity timestamps")
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
for instance in get_runner_instances():
|
||||
if instance.status == "RUNNING":
|
||||
try:
|
||||
update_instance_metadata(instance.name, "last-activity", now)
|
||||
except Exception as e:
|
||||
print(f"Error updating {instance.name}: {e}")
|
||||
return "OK", 200
|
||||
|
||||
return "OK", 200
|
||||
@@ -1,4 +0,0 @@
|
||||
functions-framework==3.*
|
||||
google-cloud-compute==1.*
|
||||
google-cloud-secret-manager==2.*
|
||||
flask>=2.0.0
|
||||
@@ -13,7 +13,6 @@ from . import cloud_run
|
||||
from . import monitoring
|
||||
from . import iam
|
||||
from . import networking
|
||||
from . import runner_manager
|
||||
|
||||
__all__ = [
|
||||
"database",
|
||||
@@ -25,5 +24,4 @@ __all__ = [
|
||||
"monitoring",
|
||||
"iam",
|
||||
"networking",
|
||||
"runner_manager",
|
||||
]
|
||||
@@ -73,11 +73,7 @@ def create_database() -> dict:
|
||||
firestore.IndexFieldArgs(field_path="made_for_you", order="ASCENDING"),
|
||||
firestore.IndexFieldArgs(field_path="created_at", order="DESCENDING"),
|
||||
],
|
||||
opts=pulumi.ResourceOptions(
|
||||
depends_on=[firestore_db],
|
||||
# Import existing index created before Pulumi managed it
|
||||
import_="projects/nomadkaraoke/databases/(default)/collectionGroups/jobs/indexes/CICAgOi36pgK",
|
||||
),
|
||||
opts=pulumi.ResourceOptions(depends_on=[firestore_db]),
|
||||
)
|
||||
|
||||
# Sessions: query by user_email + is_active, order by created_at
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
"""
|
||||
GitHub Actions Runner Manager - Auto-start and auto-stop self-hosted runners.
|
||||
|
||||
Creates infrastructure to automatically manage runner lifecycle:
|
||||
1. Cloud Function (Gen2) - Handles GitHub webhooks and idle checks
|
||||
2. Cloud Scheduler - Triggers idle checks every 15 minutes
|
||||
3. IAM bindings - Permissions to start/stop Compute Engine instances
|
||||
|
||||
Architecture:
|
||||
- GitHub sends workflow_job webhooks to the Cloud Function
|
||||
- When a job is queued, the function starts any TERMINATED runner VMs
|
||||
- Cloud Scheduler triggers idle checks every 15 minutes
|
||||
- If no jobs are pending and runners have been idle, they're stopped
|
||||
|
||||
Manual Setup Required:
|
||||
1. Create a webhook secret:
|
||||
python -c "import secrets; print(secrets.token_hex(32))" | \
|
||||
gcloud secrets versions add github-webhook-secret --data-file=-
|
||||
|
||||
2. Configure GitHub org webhook at:
|
||||
https://github.com/organizations/nomadkaraoke/settings/hooks
|
||||
- Payload URL: <Cloud Function URL from pulumi output>
|
||||
- Content type: application/json
|
||||
- Secret: <value from step 1>
|
||||
- Events: Workflow jobs
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pulumi
|
||||
import pulumi_gcp as gcp
|
||||
from config import (
|
||||
PROJECT_ID,
|
||||
REGION,
|
||||
ZONE,
|
||||
RunnerManagerConfig,
|
||||
SecretNames,
|
||||
)
|
||||
|
||||
|
||||
def create_runner_manager_service_account() -> gcp.serviceaccount.Account:
|
||||
"""Create service account for the runner manager Cloud Function."""
|
||||
return gcp.serviceaccount.Account(
|
||||
"runner-manager-sa",
|
||||
account_id="runner-manager",
|
||||
display_name="GitHub Runner Manager",
|
||||
description="Service account for the GitHub Actions runner manager Cloud Function",
|
||||
)
|
||||
|
||||
|
||||
def grant_runner_manager_permissions(
|
||||
service_account: gcp.serviceaccount.Account,
|
||||
webhook_secret: gcp.secretmanager.Secret,
|
||||
pat_secret: gcp.secretmanager.Secret,
|
||||
) -> dict:
|
||||
"""Grant permissions to the runner manager service account."""
|
||||
bindings = {}
|
||||
|
||||
# Grant Compute Instance Admin - needed to start/stop VMs
|
||||
bindings["compute_admin"] = gcp.projects.IAMMember(
|
||||
"runner-manager-compute-admin",
|
||||
project=PROJECT_ID,
|
||||
role="roles/compute.instanceAdmin.v1",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Secret Manager accessor for webhook secret
|
||||
bindings["webhook_secret_accessor"] = gcp.secretmanager.SecretIamMember(
|
||||
"runner-manager-webhook-secret-access",
|
||||
secret_id=webhook_secret.id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Secret Manager accessor for GitHub PAT (for checking pending jobs)
|
||||
bindings["pat_secret_accessor"] = gcp.secretmanager.SecretIamMember(
|
||||
"runner-manager-pat-secret-access",
|
||||
secret_id=pat_secret.id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant logging writer
|
||||
bindings["logging_writer"] = gcp.projects.IAMMember(
|
||||
"runner-manager-logging-writer",
|
||||
project=PROJECT_ID,
|
||||
role="roles/logging.logWriter",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
return bindings
|
||||
|
||||
|
||||
def create_function_storage_bucket() -> gcp.storage.Bucket:
|
||||
"""Create a GCS bucket for Cloud Function source code."""
|
||||
return gcp.storage.Bucket(
|
||||
"runner-manager-source-bucket",
|
||||
name=f"{PROJECT_ID}-runner-manager-source",
|
||||
location=REGION,
|
||||
uniform_bucket_level_access=True,
|
||||
lifecycle_rules=[
|
||||
gcp.storage.BucketLifecycleRuleArgs(
|
||||
action=gcp.storage.BucketLifecycleRuleActionArgs(type="Delete"),
|
||||
condition=gcp.storage.BucketLifecycleRuleConditionArgs(age=7),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.BucketObject:
|
||||
"""Create a GCS object with the Cloud Function source code."""
|
||||
source_dir = Path(__file__).parent.parent / "functions" / "runner_manager"
|
||||
|
||||
return gcp.storage.BucketObject(
|
||||
"runner-manager-source",
|
||||
bucket=bucket.name,
|
||||
name="runner-manager-source.zip",
|
||||
source=pulumi.FileArchive(str(source_dir)),
|
||||
)
|
||||
|
||||
|
||||
def create_cloud_function(
|
||||
service_account: gcp.serviceaccount.Account,
|
||||
bucket: gcp.storage.Bucket,
|
||||
source_archive: gcp.storage.BucketObject,
|
||||
permissions: dict,
|
||||
) -> gcp.cloudfunctionsv2.Function:
|
||||
"""Create the Cloud Function (Gen2) for runner management."""
|
||||
return gcp.cloudfunctionsv2.Function(
|
||||
"runner-manager-function",
|
||||
name=RunnerManagerConfig.FUNCTION_NAME,
|
||||
location=REGION,
|
||||
description="Manages GitHub Actions runner VM lifecycle (auto-start/stop)",
|
||||
build_config=gcp.cloudfunctionsv2.FunctionBuildConfigArgs(
|
||||
runtime="python312",
|
||||
entry_point="handle_request",
|
||||
source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceArgs(
|
||||
storage_source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceStorageSourceArgs(
|
||||
bucket=bucket.name,
|
||||
object=source_archive.name,
|
||||
),
|
||||
),
|
||||
),
|
||||
service_config=gcp.cloudfunctionsv2.FunctionServiceConfigArgs(
|
||||
available_memory=RunnerManagerConfig.FUNCTION_MEMORY,
|
||||
timeout_seconds=RunnerManagerConfig.FUNCTION_TIMEOUT,
|
||||
service_account_email=service_account.email,
|
||||
environment_variables={
|
||||
"GCP_PROJECT": PROJECT_ID,
|
||||
"GCP_ZONE": ZONE,
|
||||
"WEBHOOK_SECRET_NAME": SecretNames.GITHUB_WEBHOOK_SECRET,
|
||||
"RUNNER_PAT_SECRET_NAME": SecretNames.GITHUB_RUNNER_PAT,
|
||||
"IDLE_TIMEOUT_HOURS": str(RunnerManagerConfig.IDLE_TIMEOUT_HOURS),
|
||||
},
|
||||
min_instance_count=0, # Scale to zero when not in use
|
||||
max_instance_count=5, # Allow some concurrency for multiple webhooks
|
||||
),
|
||||
opts=pulumi.ResourceOptions(
|
||||
depends_on=list(permissions.values()),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def allow_unauthenticated_invocation(
|
||||
function: gcp.cloudfunctionsv2.Function,
|
||||
) -> gcp.cloudfunctionsv2.FunctionIamMember:
|
||||
"""
|
||||
Allow unauthenticated invocations of the Cloud Function.
|
||||
|
||||
Required for GitHub webhooks since GitHub doesn't support GCP auth.
|
||||
Security is provided by webhook signature verification.
|
||||
"""
|
||||
return gcp.cloudfunctionsv2.FunctionIamMember(
|
||||
"runner-manager-public-invoker",
|
||||
project=PROJECT_ID,
|
||||
location=REGION,
|
||||
cloud_function=function.name,
|
||||
role="roles/cloudfunctions.invoker",
|
||||
member="allUsers",
|
||||
)
|
||||
|
||||
|
||||
def create_scheduler_service_account() -> gcp.serviceaccount.Account:
|
||||
"""Create service account for Cloud Scheduler to invoke the function."""
|
||||
return gcp.serviceaccount.Account(
|
||||
"runner-manager-scheduler-sa",
|
||||
account_id="runner-manager-scheduler",
|
||||
display_name="Runner Manager Scheduler",
|
||||
description="Service account for Cloud Scheduler to trigger runner idle checks",
|
||||
)
|
||||
|
||||
|
||||
def grant_scheduler_invoker_permission(
|
||||
scheduler_sa: gcp.serviceaccount.Account,
|
||||
function: gcp.cloudfunctionsv2.Function,
|
||||
) -> gcp.cloudfunctionsv2.FunctionIamMember:
|
||||
"""Grant the scheduler service account permission to invoke the function."""
|
||||
return gcp.cloudfunctionsv2.FunctionIamMember(
|
||||
"runner-manager-scheduler-invoker",
|
||||
project=PROJECT_ID,
|
||||
location=REGION,
|
||||
cloud_function=function.name,
|
||||
role="roles/cloudfunctions.invoker",
|
||||
member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
|
||||
def create_idle_check_scheduler(
|
||||
scheduler_sa: gcp.serviceaccount.Account,
|
||||
function: gcp.cloudfunctionsv2.Function,
|
||||
) -> gcp.cloudscheduler.Job:
|
||||
"""Create Cloud Scheduler job for idle runner checks."""
|
||||
return gcp.cloudscheduler.Job(
|
||||
"runner-manager-idle-check",
|
||||
name="runner-manager-idle-check",
|
||||
description="Triggers idle check for GitHub Actions runners every 15 minutes",
|
||||
schedule=RunnerManagerConfig.IDLE_CHECK_SCHEDULE,
|
||||
time_zone="UTC",
|
||||
region=REGION,
|
||||
http_target=gcp.cloudscheduler.JobHttpTargetArgs(
|
||||
uri=function.url.apply(lambda url: f"{url}?action=check_idle"),
|
||||
http_method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body=base64.b64encode(json.dumps({"trigger": "scheduler"}).encode()).decode(),
|
||||
oidc_token=gcp.cloudscheduler.JobHttpTargetOidcTokenArgs(
|
||||
service_account_email=scheduler_sa.email,
|
||||
audience=function.url,
|
||||
),
|
||||
),
|
||||
retry_config=gcp.cloudscheduler.JobRetryConfigArgs(
|
||||
retry_count=2,
|
||||
max_retry_duration="60s",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_runner_manager_resources(
|
||||
webhook_secret: gcp.secretmanager.Secret,
|
||||
pat_secret: gcp.secretmanager.Secret,
|
||||
) -> dict:
|
||||
"""
|
||||
Create all resources for the GitHub Actions runner manager.
|
||||
|
||||
Args:
|
||||
webhook_secret: The webhook secret for signature verification.
|
||||
pat_secret: The GitHub PAT secret for API calls.
|
||||
|
||||
Returns:
|
||||
dict: Dictionary of all created resources.
|
||||
"""
|
||||
# Create service accounts
|
||||
function_sa = create_runner_manager_service_account()
|
||||
scheduler_sa = create_scheduler_service_account()
|
||||
|
||||
# Grant permissions to function service account
|
||||
permissions = grant_runner_manager_permissions(
|
||||
function_sa,
|
||||
webhook_secret,
|
||||
pat_secret,
|
||||
)
|
||||
|
||||
# Create source storage bucket and archive
|
||||
bucket = create_function_storage_bucket()
|
||||
source_archive = create_function_source_archive(bucket)
|
||||
|
||||
# Create the Cloud Function
|
||||
function = create_cloud_function(
|
||||
function_sa,
|
||||
bucket,
|
||||
source_archive,
|
||||
permissions,
|
||||
)
|
||||
|
||||
# Allow unauthenticated invocation (GitHub webhooks)
|
||||
public_access = allow_unauthenticated_invocation(function)
|
||||
|
||||
# Grant scheduler permission to invoke function
|
||||
scheduler_invoker = grant_scheduler_invoker_permission(scheduler_sa, function)
|
||||
|
||||
# Create the idle check scheduler job
|
||||
scheduler_job = create_idle_check_scheduler(scheduler_sa, function)
|
||||
|
||||
return {
|
||||
"function_service_account": function_sa,
|
||||
"scheduler_service_account": scheduler_sa,
|
||||
"permissions": permissions,
|
||||
"source_bucket": bucket,
|
||||
"source_archive": source_archive,
|
||||
"function": function,
|
||||
"public_access": public_access,
|
||||
"scheduler_invoker": scheduler_invoker,
|
||||
"scheduler_job": scheduler_job,
|
||||
}
|
||||
@@ -47,7 +47,6 @@ def create_secrets() -> dict[str, secretmanager.Secret]:
|
||||
|
||||
# GitHub
|
||||
"github-runner-pat",
|
||||
"github-webhook-secret", # For runner manager webhook verification
|
||||
|
||||
# E2E Testing
|
||||
"e2e-bypass-key",
|
||||
|
||||
@@ -219,8 +219,7 @@ class VideoGenerator:
|
||||
# Create a temporary copy of the ASS file with a unique filename
|
||||
import time
|
||||
|
||||
# Only allow ASCII alphanumeric characters to prevent FFmpeg path issues with Unicode
|
||||
safe_prefix = "".join(c if c.isascii() and c.isalnum() else "_" for c in output_prefix)
|
||||
safe_prefix = "".join(c if c.isalnum() else "_" for c in output_prefix)
|
||||
timestamp = int(time.time() * 1000)
|
||||
temp_ass_path = os.path.join(self.cache_dir, f"temp_subtitles_{safe_prefix}_{timestamp}.ass")
|
||||
import shutil
|
||||
@@ -280,8 +279,7 @@ class VideoGenerator:
|
||||
import time
|
||||
import shutil
|
||||
|
||||
# Only allow ASCII alphanumeric characters to prevent FFmpeg path issues with Unicode
|
||||
safe_prefix = "".join(c if c.isascii() and c.isalnum() else "_" for c in output_prefix)
|
||||
safe_prefix = "".join(c if c.isalnum() else "_" for c in output_prefix)
|
||||
timestamp = int(time.time() * 1000)
|
||||
temp_ass_path = os.path.join(self.cache_dir, f"temp_preview_subtitles_{safe_prefix}_{timestamp}.ass")
|
||||
|
||||
|
||||
@@ -1965,14 +1965,10 @@ class JobMonitor:
|
||||
folder_name = f"{brand_code} - {artist} - {title}"
|
||||
else:
|
||||
folder_name = f"{artist} - {title}"
|
||||
|
||||
# Sanitize folder name - only allow ASCII alphanumeric to prevent filesystem issues
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
# Guard against empty or separator-only folder name (e.g., if title/artist were entirely non-ASCII)
|
||||
if not folder_name or not any(c.isascii() and c.isalnum() for c in folder_name):
|
||||
folder_name = f"job_{job_id}"
|
||||
|
||||
|
||||
# Sanitize folder name
|
||||
folder_name = "".join(c for c in folder_name if c.isalnum() or c in " -_").strip()
|
||||
|
||||
output_dir = Path(self.config.output_dir) / folder_name
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.115.2"
|
||||
version = "0.114.15"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
"""Unit tests for remote_cli Unicode folder name sanitization.
|
||||
|
||||
Tests the fix for job 1eab1172 where Cyrillic characters in folder names
|
||||
could cause filesystem issues.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestRemoteCliFolderNameSanitization:
|
||||
"""Test folder name sanitization logic for Unicode characters."""
|
||||
|
||||
@pytest.mark.parametrize("artist,title,expected_has,expected_not_has", [
|
||||
("zivert", "я куплю тебе дом", ["zivert"], ["я", "куплю", "тебе", "дом", " - ", "_"]), # Cyrillic removed, only artist remains
|
||||
("artist", "中文标题", ["artist"], ["中", "文", "标", "题", " - ", "_"]), # Chinese removed
|
||||
("artist", "العربية", ["artist"], ["ا", "ل", "ع", "ر", "ب", " - ", "_"]), # Arabic removed
|
||||
("artist", "título", ["artist", " - ", "tulo"], ["í"]), # Accented (í removed, t+ulo remain)
|
||||
("Artist", "Normal Title", ["Artist", " - ", "Normal", "Title"], []), # ASCII preserved
|
||||
("Test", "Song (2024)", ["Test", " - ", "Song", "2024"], ["(", ")"]), # Parens removed
|
||||
])
|
||||
def test_folder_name_sanitization_logic(self, artist, title, expected_has, expected_not_has):
|
||||
"""Test the folder name sanitization logic directly."""
|
||||
# This is the exact logic from remote_cli.py:1970
|
||||
folder_name = f"{artist} - {title}"
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
# Verify folder name is ASCII only
|
||||
assert folder_name.isascii(), f"Folder name should be ASCII-only: {folder_name}"
|
||||
|
||||
# Verify expected parts are present
|
||||
for expected in expected_has:
|
||||
assert expected in folder_name, \
|
||||
f"Expected '{expected}' to be in folder name: {folder_name}"
|
||||
|
||||
# Verify unwanted parts are removed
|
||||
for not_expected in expected_not_has:
|
||||
assert not_expected not in folder_name, \
|
||||
f"Did not expect '{not_expected}' in folder name: {folder_name}"
|
||||
|
||||
def test_folder_name_brand_code_logic(self):
|
||||
"""Test folder name with brand code prefix."""
|
||||
brand_code = "BRAND-001"
|
||||
artist = "Artist"
|
||||
title = "Title"
|
||||
|
||||
folder_name = f"{brand_code} - {artist} - {title}"
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
assert folder_name.startswith("BRAND-001 - ")
|
||||
assert "Artist" in folder_name
|
||||
assert "Title" in folder_name
|
||||
|
||||
def test_folder_name_strips_whitespace(self):
|
||||
"""Test that leading/trailing whitespace is stripped."""
|
||||
artist = " Artist "
|
||||
title = " Title "
|
||||
|
||||
folder_name = f"{artist} - {title}"
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
assert not folder_name.startswith(" ")
|
||||
assert not folder_name.endswith(" ")
|
||||
assert "Artist" in folder_name
|
||||
assert "Title" in folder_name
|
||||
|
||||
def test_cyrillic_real_world_case_job_1eab1172(self):
|
||||
"""Test the exact case from failed job 1eab1172 - Russian title."""
|
||||
artist = "zivert"
|
||||
title = "я куплю тебе дом" # Russian: "I will buy you a house"
|
||||
|
||||
folder_name = f"{artist} - {title}"
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
# Should be ASCII-only
|
||||
assert folder_name.isascii()
|
||||
|
||||
# Should contain the artist name
|
||||
assert "zivert" in folder_name
|
||||
|
||||
# Cyrillic characters should be removed
|
||||
assert "я" not in folder_name
|
||||
assert "куплю" not in folder_name
|
||||
assert "тебе" not in folder_name
|
||||
assert "дом" not in folder_name
|
||||
|
||||
def test_emoji_and_special_unicode(self):
|
||||
"""Test that emojis and special Unicode characters are removed."""
|
||||
artist = "Artist 😀"
|
||||
title = "Song 🎵"
|
||||
|
||||
folder_name = f"{artist} - {title}"
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
assert folder_name.isascii()
|
||||
assert "😀" not in folder_name
|
||||
assert "🎵" not in folder_name
|
||||
assert "Artist" in folder_name
|
||||
assert "Song" in folder_name
|
||||
|
||||
def test_empty_folder_name_fallback(self):
|
||||
"""Test that if sanitization results in empty or separator-only string, fallback to job_id."""
|
||||
job_id = "abc123xyz"
|
||||
|
||||
# Test 1: Emoji with separator results in "-" after sanitization
|
||||
artist = "😀😀😀"
|
||||
title = "🎵🎵🎵"
|
||||
folder_name = f"{artist} - {title}"
|
||||
folder_name = "".join(c for c in folder_name if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
# After sanitization, only "-" remains (spaces/hyphens allowed but emoji removed)
|
||||
assert folder_name == "-"
|
||||
|
||||
# Apply the fallback logic (updated to check for alphanumeric content)
|
||||
if not folder_name or not any(c.isascii() and c.isalnum() for c in folder_name):
|
||||
folder_name = f"job_{job_id}"
|
||||
|
||||
# Verify fallback is used for separator-only result
|
||||
assert folder_name == "job_abc123xyz"
|
||||
|
||||
# Test 2: Truly empty result (all characters removed, no separator)
|
||||
artist2 = "😀"
|
||||
title2 = "🎵"
|
||||
folder_name2 = f"{artist2}{title2}" # No separator, just emoji
|
||||
folder_name2 = "".join(c for c in folder_name2 if (c.isascii() and c.isalnum()) or c in " -_").strip()
|
||||
|
||||
# Should be empty
|
||||
assert folder_name2 == ""
|
||||
|
||||
# Apply fallback
|
||||
if not folder_name2 or not any(c.isascii() and c.isalnum() for c in folder_name2):
|
||||
folder_name2 = f"job_{job_id}"
|
||||
|
||||
assert folder_name2 == "job_abc123xyz"
|
||||
@@ -1,283 +0,0 @@
|
||||
"""Unit tests for VideoGenerator Unicode filename handling.
|
||||
|
||||
Tests the fix for job 1eab1172 where Cyrillic characters in temp subtitle
|
||||
filenames caused FFmpeg to fail with exit code 183.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from karaoke_gen.lyrics_transcriber.output.video import VideoGenerator
|
||||
|
||||
|
||||
class TestVideoGeneratorUnicodeSanitization:
|
||||
"""Test that VideoGenerator properly sanitizes Unicode characters in temp filenames."""
|
||||
|
||||
@pytest.fixture
|
||||
def video_generator(self):
|
||||
"""Create a VideoGenerator instance for testing."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
output_dir = os.path.join(temp_dir, "output")
|
||||
cache_dir = os.path.join(temp_dir, "cache")
|
||||
os.makedirs(output_dir)
|
||||
os.makedirs(cache_dir)
|
||||
|
||||
styles = {
|
||||
"karaoke": {
|
||||
"background_color": "black",
|
||||
"font_path": None,
|
||||
}
|
||||
}
|
||||
|
||||
generator = VideoGenerator(
|
||||
output_dir=output_dir,
|
||||
cache_dir=cache_dir,
|
||||
video_resolution=(1920, 1080),
|
||||
styles=styles,
|
||||
logger=MagicMock(),
|
||||
)
|
||||
|
||||
yield generator
|
||||
|
||||
def test_generate_video_sanitizes_cyrillic_characters(self, video_generator):
|
||||
"""Test that Cyrillic characters are replaced with underscores in temp filename.
|
||||
|
||||
This was the bug in job 1eab1172 with Russian title "я куплю тебе дом".
|
||||
"""
|
||||
output_prefix = "zivert - я куплю тебе дом"
|
||||
|
||||
# Create mock ASS and audio files
|
||||
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
|
||||
ass_path = ass_file.name
|
||||
ass_file.write(b"[Script Info]\nTitle: Test\n")
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
|
||||
audio_path = audio_file.name
|
||||
|
||||
try:
|
||||
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
|
||||
# This should not raise an exception
|
||||
video_generator.generate_video(ass_path, audio_path, output_prefix)
|
||||
|
||||
# Verify FFmpeg was called
|
||||
assert mock_ffmpeg.called
|
||||
|
||||
# Get the temp file path that was created
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
ass_filter_arg = None
|
||||
for i, arg in enumerate(cmd):
|
||||
if arg == "-vf":
|
||||
ass_filter_arg = cmd[i + 1]
|
||||
break
|
||||
|
||||
assert ass_filter_arg is not None, "Should have ASS filter in FFmpeg command"
|
||||
|
||||
# Extract the temp ASS path from the filter
|
||||
# Format: ass=/path/to/temp_subtitles_...:fontsdir=...
|
||||
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
|
||||
|
||||
# Verify the temp filename only contains ASCII characters
|
||||
temp_filename = os.path.basename(temp_ass_path)
|
||||
assert temp_filename.isascii(), f"Temp filename should be ASCII-only: {temp_filename}"
|
||||
|
||||
# Verify Cyrillic characters were replaced with underscores
|
||||
assert "я" not in temp_filename
|
||||
assert "куплю" not in temp_filename
|
||||
assert "дом" not in temp_filename
|
||||
assert "_" in temp_filename # Should have underscores as replacements
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
if os.path.exists(ass_path):
|
||||
os.unlink(ass_path)
|
||||
if os.path.exists(audio_path):
|
||||
os.unlink(audio_path)
|
||||
|
||||
def test_generate_preview_video_sanitizes_unicode(self, video_generator):
|
||||
"""Test that preview video generation also sanitizes Unicode characters."""
|
||||
output_prefix = "artist - 中文标题" # Chinese characters
|
||||
|
||||
# Create mock ASS and audio files
|
||||
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
|
||||
ass_path = ass_file.name
|
||||
ass_file.write(b"[Script Info]\nTitle: Test\n")
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
|
||||
audio_path = audio_file.name
|
||||
|
||||
try:
|
||||
# Mock the LocalPreviewEncodingService
|
||||
with patch('backend.services.local_preview_encoding_service.LocalPreviewEncodingService') as mock_service_class:
|
||||
mock_service = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.success = True
|
||||
mock_service.encode_preview.return_value = mock_result
|
||||
mock_service_class.return_value = mock_service
|
||||
|
||||
# This should not raise an exception
|
||||
video_generator.generate_preview_video(ass_path, audio_path, output_prefix)
|
||||
|
||||
# Verify the service was called
|
||||
assert mock_service.encode_preview.called
|
||||
|
||||
# Get the config that was passed
|
||||
config = mock_service.encode_preview.call_args[0][0]
|
||||
temp_ass_path = config.ass_path
|
||||
|
||||
# Verify the temp filename only contains ASCII characters
|
||||
temp_filename = os.path.basename(temp_ass_path)
|
||||
assert temp_filename.isascii(), f"Temp filename should be ASCII-only: {temp_filename}"
|
||||
|
||||
# Verify Chinese characters were replaced
|
||||
assert "中文标题" not in temp_filename
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
if os.path.exists(ass_path):
|
||||
os.unlink(ass_path)
|
||||
if os.path.exists(audio_path):
|
||||
os.unlink(audio_path)
|
||||
|
||||
@pytest.mark.parametrize("output_prefix,unicode_chars,should_contain_ascii", [
|
||||
("artist - я куплю тебе дом", ["я", "к", "у", "п", "л", "ю", "т", "е", "б", "д", "о", "м"], "artist"), # Cyrillic
|
||||
("artist - 中文标题", ["中", "文", "标", "题"], "artist"), # Chinese
|
||||
("artist - العربية", ["ا", "ل", "ع", "ر", "ب", "ي", "ة"], "artist"), # Arabic
|
||||
("artist - 😀🎵", ["😀", "🎵"], "artist"), # Emoji
|
||||
("artist - título", ["í"], "tulo"), # Spanish with accent - 't' and 'ulo' should remain
|
||||
("normal title", [], "normal_title"), # ASCII-only (control)
|
||||
])
|
||||
def test_various_unicode_characters_sanitized(self, video_generator, output_prefix, unicode_chars, should_contain_ascii):
|
||||
"""Test that various Unicode character sets are properly sanitized."""
|
||||
import re
|
||||
|
||||
# Create mock ASS and audio files
|
||||
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
|
||||
ass_path = ass_file.name
|
||||
ass_file.write(b"[Script Info]\nTitle: Test\n")
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
|
||||
audio_path = audio_file.name
|
||||
|
||||
try:
|
||||
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
|
||||
video_generator.generate_video(ass_path, audio_path, output_prefix)
|
||||
|
||||
# Get the temp file path
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
ass_filter_arg = None
|
||||
for i, arg in enumerate(cmd):
|
||||
if arg == "-vf":
|
||||
ass_filter_arg = cmd[i + 1]
|
||||
break
|
||||
|
||||
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
|
||||
temp_filename = os.path.basename(temp_ass_path)
|
||||
|
||||
# Verify filename only contains ASCII
|
||||
assert temp_filename.isascii(), f"Should be ASCII-only: {temp_filename}"
|
||||
|
||||
# Verify all Unicode characters were removed
|
||||
for unicode_char in unicode_chars:
|
||||
assert unicode_char not in temp_filename, \
|
||||
f"Unicode character '{unicode_char}' should not be in filename: {temp_filename}"
|
||||
|
||||
# Verify ASCII parts were preserved
|
||||
if should_contain_ascii:
|
||||
assert should_contain_ascii in temp_filename, \
|
||||
f"Expected '{should_contain_ascii}' to be in filename: {temp_filename}"
|
||||
|
||||
# Verify it matches the general pattern (starts with temp_subtitles_, ends with .ass)
|
||||
assert temp_filename.startswith("temp_subtitles_"), \
|
||||
f"Filename should start with 'temp_subtitles_': {temp_filename}"
|
||||
assert temp_filename.endswith(".ass"), \
|
||||
f"Filename should end with '.ass': {temp_filename}"
|
||||
|
||||
finally:
|
||||
if os.path.exists(ass_path):
|
||||
os.unlink(ass_path)
|
||||
if os.path.exists(audio_path):
|
||||
os.unlink(audio_path)
|
||||
|
||||
def test_sanitization_preserves_ascii_alphanumeric(self, video_generator):
|
||||
"""Test that ASCII alphanumeric characters are preserved."""
|
||||
output_prefix = "ABC123xyz"
|
||||
|
||||
# Create mock ASS and audio files
|
||||
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
|
||||
ass_path = ass_file.name
|
||||
ass_file.write(b"[Script Info]\nTitle: Test\n")
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
|
||||
audio_path = audio_file.name
|
||||
|
||||
try:
|
||||
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
|
||||
video_generator.generate_video(ass_path, audio_path, output_prefix)
|
||||
|
||||
# Get the temp file path
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
ass_filter_arg = None
|
||||
for i, arg in enumerate(cmd):
|
||||
if arg == "-vf":
|
||||
ass_filter_arg = cmd[i + 1]
|
||||
break
|
||||
|
||||
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
|
||||
temp_filename = os.path.basename(temp_ass_path)
|
||||
|
||||
# Verify ASCII alphanumeric characters are preserved
|
||||
assert "ABC123xyz" in temp_filename
|
||||
|
||||
finally:
|
||||
if os.path.exists(ass_path):
|
||||
os.unlink(ass_path)
|
||||
if os.path.exists(audio_path):
|
||||
os.unlink(audio_path)
|
||||
|
||||
def test_special_characters_replaced_with_underscores(self, video_generator):
|
||||
"""Test that special characters (spaces, hyphens, etc.) are replaced."""
|
||||
output_prefix = "artist - title (version) [2024]"
|
||||
|
||||
# Create mock ASS and audio files
|
||||
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
|
||||
ass_path = ass_file.name
|
||||
ass_file.write(b"[Script Info]\nTitle: Test\n")
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
|
||||
audio_path = audio_file.name
|
||||
|
||||
try:
|
||||
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
|
||||
video_generator.generate_video(ass_path, audio_path, output_prefix)
|
||||
|
||||
# Get the temp file path
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
ass_filter_arg = None
|
||||
for i, arg in enumerate(cmd):
|
||||
if arg == "-vf":
|
||||
ass_filter_arg = cmd[i + 1]
|
||||
break
|
||||
|
||||
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
|
||||
temp_filename = os.path.basename(temp_ass_path)
|
||||
|
||||
# Verify special characters were replaced
|
||||
assert " " not in temp_filename
|
||||
assert "-" not in temp_filename
|
||||
assert "(" not in temp_filename
|
||||
assert ")" not in temp_filename
|
||||
assert "[" not in temp_filename
|
||||
assert "]" not in temp_filename
|
||||
|
||||
# But alphanumeric should remain
|
||||
assert "artist" in temp_filename
|
||||
assert "title" in temp_filename
|
||||
assert "version" in temp_filename
|
||||
assert "2024" in temp_filename
|
||||
|
||||
finally:
|
||||
if os.path.exists(ass_path):
|
||||
os.unlink(ass_path)
|
||||
if os.path.exists(audio_path):
|
||||
os.unlink(audio_path)
|
||||
Reference in new issue
Block a user