mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-21 21:38:07 +09:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef48b726d8 | ||
|
|
5901137724 | ||
|
|
b0cbd28c8d | ||
|
|
c8d3fa46c4 | ||
|
|
6ebf076576 |
No files matched your search
@@ -1012,8 +1012,8 @@ class TriggerWorkerResponse(BaseModel):
|
||||
ALLOWED_RESET_STATES = {
|
||||
"pending",
|
||||
"awaiting_audio_selection",
|
||||
"awaiting_review",
|
||||
"awaiting_instrumental_selection",
|
||||
"awaiting_review", # Combined review (lyrics + instrumental selection)
|
||||
# Note: awaiting_instrumental_selection removed - deprecated by combined review flow (Jan 2026)
|
||||
"instrumental_selected", # Re-run video generation with same settings
|
||||
}
|
||||
|
||||
@@ -1034,6 +1034,9 @@ STATE_DATA_CLEAR_KEYS = {
|
||||
"render_progress",
|
||||
"screens_progress",
|
||||
"encoding_progress",
|
||||
# Parallel processing flags - must clear to allow workers to run again
|
||||
"audio_complete",
|
||||
"lyrics_complete",
|
||||
],
|
||||
"awaiting_audio_selection": [
|
||||
"audio_selection",
|
||||
@@ -1046,6 +1049,7 @@ STATE_DATA_CLEAR_KEYS = {
|
||||
"encoding_progress",
|
||||
],
|
||||
"awaiting_review": [
|
||||
# Combined review - user reviews lyrics AND selects instrumental in one step
|
||||
"review_complete",
|
||||
"corrected_lyrics",
|
||||
"instrumental_selection",
|
||||
@@ -1054,13 +1058,7 @@ STATE_DATA_CLEAR_KEYS = {
|
||||
"screens_progress",
|
||||
"encoding_progress",
|
||||
],
|
||||
"awaiting_instrumental_selection": [
|
||||
"instrumental_selection",
|
||||
"video_progress",
|
||||
"render_progress",
|
||||
"screens_progress",
|
||||
"encoding_progress",
|
||||
],
|
||||
# Note: awaiting_instrumental_selection removed - deprecated by combined review flow (Jan 2026)
|
||||
"instrumental_selected": [
|
||||
# Clear video/encoding/distribution state to allow re-processing
|
||||
# Preserves: instrumental_selection, lyrics_metadata, review data
|
||||
@@ -1148,14 +1146,8 @@ async def reset_job(
|
||||
}
|
||||
|
||||
# Perform the update with state_data clearing
|
||||
# We need to set the cleared keys to DELETE_FIELD
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to reset job. Please try again."
|
||||
)
|
||||
# Note: update_job() raises exceptions on failure, returns None on success
|
||||
job_manager.update_job(job_id, updates)
|
||||
|
||||
# Clear the state data keys and error state using direct Firestore update
|
||||
from google.cloud.firestore_v1 import DELETE_FIELD, ArrayUnion
|
||||
|
||||
@@ -236,14 +236,6 @@ def extract_request_metadata(request: Request, created_from: str = "audio_search
|
||||
}
|
||||
|
||||
|
||||
async def _trigger_workers_parallel(job_id: str) -> None:
|
||||
"""Trigger both audio and lyrics workers in parallel."""
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
|
||||
def _extract_gcs_path(filepath: str) -> str:
|
||||
"""
|
||||
Extract the path portion from a GCS path or gs:// URL.
|
||||
@@ -434,21 +426,31 @@ async def _download_and_start_processing(
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp files: {e}")
|
||||
|
||||
# Update job with GCS path and transition to DOWNLOADING
|
||||
# Update job with GCS path
|
||||
job_manager.update_job(job_id, {
|
||||
'input_media_gcs_path': audio_gcs_path,
|
||||
'filename': filename,
|
||||
})
|
||||
|
||||
|
||||
# Transition from DOWNLOADING_AUDIO to DOWNLOADING and trigger workers
|
||||
# Note: Can't use start_job_processing() here because that expects PENDING status,
|
||||
# but we're in DOWNLOADING_AUDIO state from the audio search flow.
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=15,
|
||||
message="Audio downloaded, starting processing"
|
||||
)
|
||||
|
||||
# Trigger workers
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
# Trigger workers in background
|
||||
async def trigger_workers():
|
||||
worker_service = get_worker_service()
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
background_tasks.add_task(trigger_workers)
|
||||
|
||||
return {
|
||||
'selected_index': selection_index,
|
||||
|
||||
@@ -256,19 +256,6 @@ ALLOWED_IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp'}
|
||||
ALLOWED_FONT_EXTENSIONS = {'.ttf', '.otf', '.woff', '.woff2'}
|
||||
|
||||
|
||||
async def _trigger_workers_parallel(job_id: str) -> None:
|
||||
"""
|
||||
Trigger both audio and lyrics workers in parallel.
|
||||
|
||||
FastAPI's BackgroundTasks runs async tasks sequentially, so we use
|
||||
asyncio.gather to ensure both workers start at the same time.
|
||||
"""
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
|
||||
async def _trigger_audio_worker_only(job_id: str) -> None:
|
||||
"""
|
||||
Trigger only the audio worker.
|
||||
@@ -755,18 +742,17 @@ async def upload_and_create_job(
|
||||
if style_assets:
|
||||
logger.info(f"Style assets: {list(style_assets.keys())}")
|
||||
|
||||
# Transition job to DOWNLOADING state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Files uploaded, preparing to process"
|
||||
# Use centralized start_job_processing which handles:
|
||||
# 1. Validates job exists and has input_media_gcs_path
|
||||
# 2. Transitions to DOWNLOADING (raises InvalidStateTransitionError if invalid)
|
||||
# 3. Triggers audio + lyrics workers in parallel
|
||||
background_tasks.add_task(
|
||||
job_manager.start_job_processing,
|
||||
job_id,
|
||||
5, # progress
|
||||
"Files uploaded, preparing to process"
|
||||
)
|
||||
|
||||
# Trigger workers in parallel using asyncio.gather
|
||||
# (FastAPI's BackgroundTasks runs async tasks sequentially)
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
# Build distribution services info for response
|
||||
distribution_services: Dict[str, Any] = {}
|
||||
|
||||
@@ -1305,17 +1291,17 @@ async def mark_uploads_complete(
|
||||
|
||||
logger.info(f"Validated uploads for job {job_id}: {body.uploaded_files}")
|
||||
|
||||
# Transition job to DOWNLOADING state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Files uploaded, preparing to process"
|
||||
# Use centralized start_job_processing which handles:
|
||||
# 1. Validates job exists and has input_media_gcs_path
|
||||
# 2. Transitions to DOWNLOADING (raises InvalidStateTransitionError if invalid)
|
||||
# 3. Triggers audio + lyrics workers in parallel
|
||||
background_tasks.add_task(
|
||||
job_manager.start_job_processing,
|
||||
job_id,
|
||||
5, # progress
|
||||
"Files uploaded, preparing to process"
|
||||
)
|
||||
|
||||
# Trigger workers in parallel
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
# Get distribution services info for response
|
||||
credential_manager = get_credential_manager()
|
||||
distribution_services: Dict[str, Any] = {}
|
||||
@@ -1588,14 +1574,6 @@ async def create_job_from_url(
|
||||
if title:
|
||||
logger.info(f" Title: {title}")
|
||||
|
||||
# Transition job to DOWNLOADING state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Starting audio download from URL"
|
||||
)
|
||||
|
||||
# For YouTube URLs, download audio NOW using YouTubeDownloadService
|
||||
# This uses remote flacfetch (if configured) to avoid bot detection on Cloud Run
|
||||
if _is_youtube_url(body.url):
|
||||
@@ -1617,14 +1595,16 @@ async def create_job_from_url(
|
||||
|
||||
logger.info(f"YouTube audio downloaded to GCS: {audio_gcs_path}")
|
||||
|
||||
# Now trigger both workers in parallel (audio already downloaded)
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=10,
|
||||
message="Audio downloaded, starting processing"
|
||||
# Use centralized start_job_processing which handles:
|
||||
# 1. Validates job exists and has input_media_gcs_path
|
||||
# 2. Transitions PENDING → DOWNLOADING (raises if invalid)
|
||||
# 3. Triggers audio + lyrics workers in parallel
|
||||
background_tasks.add_task(
|
||||
job_manager.start_job_processing,
|
||||
job_id,
|
||||
10, # progress
|
||||
"Audio downloaded, starting processing"
|
||||
)
|
||||
background_tasks.add_task(_trigger_workers_parallel, job_id)
|
||||
|
||||
except YouTubeDownloadError as e:
|
||||
logger.error(f"YouTube download failed: {e}")
|
||||
@@ -1634,8 +1614,14 @@ async def create_job_from_url(
|
||||
detail=f"YouTube download failed: {e}"
|
||||
)
|
||||
else:
|
||||
# For non-YouTube URLs, trigger ONLY audio worker first
|
||||
# For non-YouTube URLs, transition to DOWNLOADING and trigger audio worker
|
||||
# The audio worker will download the URL and trigger lyrics worker after
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=5,
|
||||
message="Starting audio download from URL"
|
||||
)
|
||||
background_tasks.add_task(_trigger_audio_worker_only, job_id)
|
||||
|
||||
return CreateJobFromUrlResponse(
|
||||
|
||||
@@ -16,6 +16,9 @@ from backend.services.encoding_service import get_encoding_service
|
||||
from backend.services.spacy_preloader import get_preloaded_model, is_model_preloaded
|
||||
from backend.services.nltk_preloader import get_preloaded_cmudict, is_cmudict_preloaded
|
||||
from backend.services.langfuse_preloader import get_preloaded_langfuse_handler, is_langfuse_preloaded, is_langfuse_configured
|
||||
from backend.services.job_health_service import check_job_consistency_detailed
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.models.job import JobStatus
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -430,6 +433,69 @@ async def preload_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health/job-consistency")
|
||||
async def job_consistency_check(
|
||||
status: Optional[str] = None,
|
||||
limit: int = 50
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Check job consistency across the system.
|
||||
|
||||
This endpoint identifies jobs with state inconsistencies that may indicate
|
||||
bugs in the state machine or trigger logic. Use this to find stuck jobs
|
||||
or jobs in invalid states.
|
||||
|
||||
Args:
|
||||
status: Optional filter by job status (e.g., "pending", "downloading")
|
||||
limit: Maximum number of jobs to check (default 50)
|
||||
|
||||
Returns:
|
||||
Dictionary with:
|
||||
- total_checked: Number of jobs checked
|
||||
- inconsistent_count: Number of jobs with issues
|
||||
- inconsistent_jobs: List of job details with their issues
|
||||
- summary: Summary of issue types found
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
|
||||
# Query jobs to check
|
||||
query_params = {}
|
||||
if status:
|
||||
try:
|
||||
job_status = JobStatus(status)
|
||||
query_params['status'] = job_status
|
||||
except ValueError:
|
||||
return {
|
||||
"error": f"Invalid status: {status}",
|
||||
"valid_statuses": [s.value for s in JobStatus]
|
||||
}
|
||||
|
||||
# Get jobs from Firestore
|
||||
jobs = job_manager.list_jobs(limit=limit, **query_params)
|
||||
|
||||
# Check each job for consistency
|
||||
inconsistent_jobs = []
|
||||
issue_summary = {}
|
||||
|
||||
for job in jobs:
|
||||
result = check_job_consistency_detailed(job)
|
||||
if not result['is_healthy']:
|
||||
inconsistent_jobs.append(result)
|
||||
# Count issue types
|
||||
for issue in result['issues']:
|
||||
issue_type = issue.split(':')[0] if ':' in issue else issue
|
||||
issue_summary[issue_type] = issue_summary.get(issue_type, 0) + 1
|
||||
|
||||
return {
|
||||
"total_checked": len(jobs),
|
||||
"inconsistent_count": len(inconsistent_jobs),
|
||||
"inconsistent_jobs": inconsistent_jobs,
|
||||
"summary": issue_summary,
|
||||
"status_filter": status,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/readiness")
|
||||
async def readiness_check() -> Dict[str, str]:
|
||||
"""Readiness check endpoint for Cloud Run."""
|
||||
|
||||
@@ -65,6 +65,7 @@ from backend.services.youtube_download_service import (
|
||||
get_youtube_download_service,
|
||||
YouTubeDownloadError,
|
||||
)
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -476,14 +477,12 @@ async def _handle_made_for_you_order(
|
||||
"""
|
||||
from backend.models.job import JobCreate, JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.worker_service import get_worker_service
|
||||
from backend.services.audio_search_service import (
|
||||
get_audio_search_service,
|
||||
NoResultsError,
|
||||
AudioSearchError,
|
||||
)
|
||||
from backend.services.storage_service import StorageService
|
||||
import asyncio
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
@@ -501,7 +500,6 @@ async def _handle_made_for_you_order(
|
||||
|
||||
try:
|
||||
job_manager = JobManager()
|
||||
worker_service = get_worker_service()
|
||||
storage_service = StorageService()
|
||||
|
||||
# Apply default theme (Nomad) - same as audio_search endpoint
|
||||
@@ -602,13 +600,22 @@ async def _handle_made_for_you_order(
|
||||
'filename': os.path.basename(audio_gcs_path),
|
||||
})
|
||||
|
||||
logger.info(f"Job {job_id}: YouTube audio downloaded to {audio_gcs_path}, triggering workers")
|
||||
logger.info(f"Job {job_id}: YouTube audio downloaded to {audio_gcs_path}")
|
||||
|
||||
# Now trigger both workers in parallel (audio already downloaded)
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
# Use centralized helper that handles transition + worker triggers
|
||||
# This ensures consistent state machine flow and raises on invalid transitions
|
||||
try:
|
||||
await job_manager.start_job_processing(job_id)
|
||||
except InvalidStateTransitionError as e:
|
||||
logger.error(f"Job {job_id}: Failed to start processing - invalid state transition: {e}")
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.FAILED,
|
||||
progress=0,
|
||||
message=f"Failed to start processing: {str(e)}",
|
||||
raise_on_invalid=False # Don't raise if already failed
|
||||
)
|
||||
raise
|
||||
|
||||
except YouTubeDownloadError as e:
|
||||
logger.error(f"Job {job_id}: YouTube download failed: {e}")
|
||||
|
||||
@@ -64,3 +64,39 @@ class IPBlockedError(Exception):
|
||||
self.message = message
|
||||
self.ip_address = ip_address
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class InvalidStateTransitionError(Exception):
|
||||
"""
|
||||
Raised when an invalid job state transition is attempted.
|
||||
|
||||
This exception is raised by JobManager.transition_to_state() when the
|
||||
requested transition is not allowed by the state machine defined in
|
||||
STATE_TRANSITIONS.
|
||||
|
||||
Example: Trying to transition from PENDING to GENERATING_SCREENS is invalid
|
||||
because PENDING can only transition to DOWNLOADING, SEARCHING_AUDIO, FAILED,
|
||||
or CANCELLED.
|
||||
|
||||
Attributes:
|
||||
message: Human-readable error message
|
||||
job_id: The job ID that failed to transition
|
||||
from_status: Current status of the job
|
||||
to_status: Attempted target status
|
||||
valid_transitions: List of valid transitions from current status
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
job_id: str = "",
|
||||
from_status: str = "",
|
||||
to_status: str = "",
|
||||
valid_transitions: list = None
|
||||
):
|
||||
self.message = message
|
||||
self.job_id = job_id
|
||||
self.from_status = from_status
|
||||
self.to_status = to_status
|
||||
self.valid_transitions = valid_transitions or []
|
||||
super().__init__(message)
|
||||
@@ -0,0 +1,265 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate a Mermaid state diagram from the job state machine.
|
||||
|
||||
This script reads the STATE_TRANSITIONS dictionary from models/job.py
|
||||
and generates a Mermaid diagram that can be embedded in documentation.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_state_diagram.py > docs/state-machine.mmd
|
||||
# Or to update ARCHITECTURE.md directly:
|
||||
python scripts/generate_state_diagram.py --update-docs
|
||||
|
||||
Created as part of state machine robustness improvements (2026-02-02).
|
||||
"""
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
# Add backend to path for imports
|
||||
backend_dir = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(backend_dir))
|
||||
|
||||
# Import the state machine data
|
||||
# Note: Must be run from an environment where karaoke_gen is installed
|
||||
try:
|
||||
from models.job import STATE_TRANSITIONS, JobStatus
|
||||
except ImportError:
|
||||
# Fallback: define the state machine inline for documentation purposes
|
||||
from enum import Enum
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
SEARCHING_AUDIO = "searching_audio"
|
||||
AWAITING_AUDIO_SELECTION = "awaiting_audio_selection"
|
||||
DOWNLOADING_AUDIO = "downloading_audio"
|
||||
DOWNLOADING = "downloading"
|
||||
SEPARATING_STAGE1 = "separating_stage1"
|
||||
SEPARATING_STAGE2 = "separating_stage2"
|
||||
AUDIO_COMPLETE = "audio_complete"
|
||||
TRANSCRIBING = "transcribing"
|
||||
CORRECTING = "correcting"
|
||||
LYRICS_COMPLETE = "lyrics_complete"
|
||||
GENERATING_SCREENS = "generating_screens"
|
||||
APPLYING_PADDING = "applying_padding"
|
||||
AWAITING_REVIEW = "awaiting_review"
|
||||
IN_REVIEW = "in_review"
|
||||
REVIEW_COMPLETE = "review_complete"
|
||||
RENDERING_VIDEO = "rendering_video"
|
||||
AWAITING_INSTRUMENTAL_SELECTION = "awaiting_instrumental_selection"
|
||||
INSTRUMENTAL_SELECTED = "instrumental_selected"
|
||||
GENERATING_VIDEO = "generating_video"
|
||||
ENCODING = "encoding"
|
||||
PACKAGING = "packaging"
|
||||
UPLOADING = "uploading"
|
||||
NOTIFYING = "notifying"
|
||||
COMPLETE = "complete"
|
||||
PREP_COMPLETE = "prep_complete"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
QUEUED = "queued"
|
||||
PROCESSING = "processing"
|
||||
ERROR = "error"
|
||||
READY_FOR_FINALIZATION = "ready_for_finalization"
|
||||
FINALIZING = "finalizing"
|
||||
|
||||
STATE_TRANSITIONS = {
|
||||
JobStatus.PENDING: [JobStatus.DOWNLOADING, JobStatus.SEARCHING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.SEARCHING_AUDIO: [JobStatus.AWAITING_AUDIO_SELECTION, JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED],
|
||||
JobStatus.AWAITING_AUDIO_SELECTION: [JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.DOWNLOADING_AUDIO: [JobStatus.DOWNLOADING, JobStatus.FAILED],
|
||||
JobStatus.DOWNLOADING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING, JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
JobStatus.SEPARATING_STAGE1: [JobStatus.SEPARATING_STAGE2, JobStatus.FAILED],
|
||||
JobStatus.SEPARATING_STAGE2: [JobStatus.AUDIO_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.AUDIO_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
JobStatus.TRANSCRIBING: [JobStatus.CORRECTING, JobStatus.FAILED],
|
||||
JobStatus.CORRECTING: [JobStatus.LYRICS_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.LYRICS_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
JobStatus.GENERATING_SCREENS: [JobStatus.APPLYING_PADDING, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.APPLYING_PADDING: [JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.AWAITING_REVIEW: [JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.IN_REVIEW: [JobStatus.REVIEW_COMPLETE, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.REVIEW_COMPLETE: [JobStatus.RENDERING_VIDEO, JobStatus.FAILED],
|
||||
JobStatus.RENDERING_VIDEO: [JobStatus.INSTRUMENTAL_SELECTED, JobStatus.FAILED],
|
||||
JobStatus.INSTRUMENTAL_SELECTED: [JobStatus.GENERATING_VIDEO, JobStatus.FAILED],
|
||||
JobStatus.GENERATING_VIDEO: [JobStatus.ENCODING, JobStatus.FAILED],
|
||||
JobStatus.ENCODING: [JobStatus.PACKAGING, JobStatus.COMPLETE, JobStatus.PREP_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.PACKAGING: [JobStatus.UPLOADING, JobStatus.COMPLETE, JobStatus.PREP_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
}
|
||||
|
||||
|
||||
def generate_mermaid() -> str:
|
||||
"""
|
||||
Generate a Mermaid state diagram from STATE_TRANSITIONS.
|
||||
|
||||
Returns:
|
||||
Mermaid diagram as a string
|
||||
"""
|
||||
lines = ["stateDiagram-v2"]
|
||||
lines.append(" direction TB")
|
||||
lines.append("")
|
||||
|
||||
# Group states by category for better visualization
|
||||
categories = {
|
||||
"Initial": [JobStatus.PENDING],
|
||||
"Audio Search": [
|
||||
JobStatus.SEARCHING_AUDIO,
|
||||
JobStatus.AWAITING_AUDIO_SELECTION,
|
||||
JobStatus.DOWNLOADING_AUDIO,
|
||||
],
|
||||
"Processing": [
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.SEPARATING_STAGE1,
|
||||
JobStatus.SEPARATING_STAGE2,
|
||||
JobStatus.AUDIO_COMPLETE,
|
||||
JobStatus.TRANSCRIBING,
|
||||
JobStatus.CORRECTING,
|
||||
JobStatus.LYRICS_COMPLETE,
|
||||
],
|
||||
"Screens": [
|
||||
JobStatus.GENERATING_SCREENS,
|
||||
JobStatus.APPLYING_PADDING,
|
||||
],
|
||||
"Review": [
|
||||
JobStatus.AWAITING_REVIEW,
|
||||
JobStatus.IN_REVIEW,
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
],
|
||||
"Video": [
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
JobStatus.INSTRUMENTAL_SELECTED,
|
||||
JobStatus.GENERATING_VIDEO,
|
||||
JobStatus.ENCODING,
|
||||
JobStatus.PACKAGING,
|
||||
],
|
||||
"Distribution": [
|
||||
JobStatus.UPLOADING,
|
||||
JobStatus.NOTIFYING,
|
||||
],
|
||||
"Terminal": [
|
||||
JobStatus.COMPLETE,
|
||||
JobStatus.PREP_COMPLETE,
|
||||
JobStatus.FAILED,
|
||||
JobStatus.CANCELLED,
|
||||
],
|
||||
}
|
||||
|
||||
# Add state definitions with notes for key states
|
||||
lines.append(" %% State definitions")
|
||||
key_states = {
|
||||
JobStatus.PENDING: "Job created",
|
||||
JobStatus.AWAITING_AUDIO_SELECTION: "⚠️ USER ACTION",
|
||||
JobStatus.AWAITING_REVIEW: "⚠️ USER ACTION",
|
||||
JobStatus.COMPLETE: "✅ Success",
|
||||
JobStatus.FAILED: "❌ Error",
|
||||
}
|
||||
|
||||
for status, note in key_states.items():
|
||||
lines.append(f" {status.value}: {status.value}")
|
||||
lines.append(f" note right of {status.value}: {note}")
|
||||
|
||||
lines.append("")
|
||||
lines.append(" %% Transitions")
|
||||
|
||||
# Generate transitions
|
||||
for from_status, to_statuses in STATE_TRANSITIONS.items():
|
||||
for to_status in to_statuses:
|
||||
# Skip deprecated statuses
|
||||
if to_status in [JobStatus.QUEUED, JobStatus.PROCESSING, JobStatus.ERROR,
|
||||
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING]:
|
||||
continue
|
||||
if from_status in [JobStatus.QUEUED, JobStatus.PROCESSING, JobStatus.ERROR,
|
||||
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING]:
|
||||
continue
|
||||
|
||||
lines.append(f" {from_status.value} --> {to_status.value}")
|
||||
|
||||
# Add start and end markers
|
||||
lines.append("")
|
||||
lines.append(" %% Start and end")
|
||||
lines.append(" [*] --> pending")
|
||||
lines.append(" complete --> [*]")
|
||||
lines.append(" prep_complete --> [*]")
|
||||
lines.append(" failed --> [*]")
|
||||
lines.append(" cancelled --> [*]")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_markdown_section() -> str:
|
||||
"""
|
||||
Generate a markdown section with the state diagram.
|
||||
|
||||
Returns:
|
||||
Markdown string with the diagram
|
||||
"""
|
||||
diagram = generate_mermaid()
|
||||
|
||||
return f"""## Job State Machine
|
||||
|
||||
The following diagram shows all valid state transitions for a karaoke generation job.
|
||||
|
||||
Key states:
|
||||
- **PENDING**: Initial state when job is created
|
||||
- **AWAITING_AUDIO_SELECTION**: Human selects audio source (for artist+title search)
|
||||
- **AWAITING_REVIEW**: Human reviews lyrics and selects instrumental
|
||||
- **COMPLETE**: All processing finished successfully
|
||||
|
||||
```mermaid
|
||||
{diagram}
|
||||
```
|
||||
|
||||
### Parallel Processing
|
||||
|
||||
Jobs use parallel processing for audio separation and lyrics transcription:
|
||||
|
||||
1. **DOWNLOADING** triggers both:
|
||||
- Audio worker → SEPARATING_STAGE1 → SEPARATING_STAGE2 → AUDIO_COMPLETE
|
||||
- Lyrics worker → TRANSCRIBING → CORRECTING → LYRICS_COMPLETE
|
||||
|
||||
2. When BOTH complete, the screens worker is triggered automatically.
|
||||
|
||||
### Human Review Points
|
||||
|
||||
Jobs wait for human input at these points:
|
||||
- **AWAITING_AUDIO_SELECTION**: User selects audio source from search results
|
||||
- **AWAITING_REVIEW**: User reviews/corrects lyrics AND selects instrumental
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Any state can transition to **FAILED** on unrecoverable errors
|
||||
- **CANCELLED** is set when user cancels the job
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate state machine diagram")
|
||||
parser.add_argument(
|
||||
"--update-docs",
|
||||
action="store_true",
|
||||
help="Update ARCHITECTURE.md with the diagram"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--markdown",
|
||||
action="store_true",
|
||||
help="Output markdown section instead of raw Mermaid"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.update_docs:
|
||||
# TODO: Implement auto-update of ARCHITECTURE.md
|
||||
print("Auto-update of ARCHITECTURE.md not yet implemented")
|
||||
print("Please copy the output below to the appropriate section:")
|
||||
print()
|
||||
print(generate_markdown_section())
|
||||
elif args.markdown:
|
||||
print(generate_markdown_section())
|
||||
else:
|
||||
print(generate_mermaid())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,221 @@
|
||||
"""
|
||||
Job health and consistency checking service.
|
||||
|
||||
Provides functions to detect state inconsistencies in jobs that might indicate
|
||||
bugs in the state machine or trigger logic.
|
||||
|
||||
Created as part of state machine robustness improvements (2026-02-02).
|
||||
"""
|
||||
import logging
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Jobs in these statuses should NOT have audio_complete flag set
|
||||
# (unless they're actively processing or beyond)
|
||||
STATUSES_BEFORE_AUDIO_START = {
|
||||
JobStatus.PENDING,
|
||||
JobStatus.SEARCHING_AUDIO,
|
||||
JobStatus.AWAITING_AUDIO_SELECTION,
|
||||
}
|
||||
|
||||
# Jobs in these statuses should NOT have lyrics_complete flag set
|
||||
STATUSES_BEFORE_LYRICS_START = {
|
||||
JobStatus.PENDING,
|
||||
JobStatus.SEARCHING_AUDIO,
|
||||
JobStatus.AWAITING_AUDIO_SELECTION,
|
||||
}
|
||||
|
||||
# Statuses where screens_progress.stage='complete' is valid
|
||||
STATUSES_AFTER_SCREENS_COMPLETE = {
|
||||
JobStatus.AWAITING_REVIEW,
|
||||
JobStatus.IN_REVIEW,
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
JobStatus.AWAITING_INSTRUMENTAL_SELECTION, # Legacy
|
||||
JobStatus.INSTRUMENTAL_SELECTED,
|
||||
JobStatus.GENERATING_VIDEO,
|
||||
JobStatus.ENCODING,
|
||||
JobStatus.PACKAGING,
|
||||
JobStatus.UPLOADING,
|
||||
JobStatus.NOTIFYING,
|
||||
JobStatus.COMPLETE,
|
||||
JobStatus.PREP_COMPLETE,
|
||||
}
|
||||
|
||||
|
||||
def check_job_consistency(job: Job) -> List[str]:
|
||||
"""
|
||||
Check for state inconsistencies in a job.
|
||||
|
||||
This function detects situations where state_data flags indicate
|
||||
progress that doesn't match the job status - which usually indicates
|
||||
a bug in the state machine or trigger logic.
|
||||
|
||||
Args:
|
||||
job: Job object to check
|
||||
|
||||
Returns:
|
||||
List of issue descriptions found. Empty list means no issues.
|
||||
"""
|
||||
issues = []
|
||||
state_data = job.state_data or {}
|
||||
|
||||
# Normalize status to JobStatus enum if it's a string
|
||||
status = job.status
|
||||
if isinstance(status, str):
|
||||
try:
|
||||
status = JobStatus(status)
|
||||
except ValueError:
|
||||
issues.append(f"invalid_status: '{job.status}' is not a valid JobStatus")
|
||||
return issues
|
||||
|
||||
# Check: audio_complete=True but status still in pre-audio states
|
||||
if state_data.get('audio_complete') and status in STATUSES_BEFORE_AUDIO_START:
|
||||
issues.append(
|
||||
f"audio_complete_status_mismatch: audio_complete=True but status={status.value}"
|
||||
)
|
||||
|
||||
# Check: lyrics_complete=True but status still in pre-lyrics states
|
||||
if state_data.get('lyrics_complete') and status in STATUSES_BEFORE_LYRICS_START:
|
||||
issues.append(
|
||||
f"lyrics_complete_status_mismatch: lyrics_complete=True but status={status.value}"
|
||||
)
|
||||
|
||||
# Check: screens_progress.stage='complete' but status not in post-screens states
|
||||
screens_progress = state_data.get('screens_progress', {})
|
||||
if isinstance(screens_progress, dict) and screens_progress.get('stage') == 'complete':
|
||||
if status not in STATUSES_AFTER_SCREENS_COMPLETE:
|
||||
issues.append(
|
||||
f"screens_complete_status_mismatch: screens_progress.stage=complete but status={status.value}"
|
||||
)
|
||||
|
||||
# Check: job has input_media_gcs_path but is stuck at pending
|
||||
# This was the exact bug that got jobs 06cfea29 and 984da08b stuck
|
||||
if job.input_media_gcs_path and status == JobStatus.PENDING:
|
||||
# This could be legitimate if job was just created and workers haven't started
|
||||
# But if it's been more than a few minutes, it's likely stuck
|
||||
# We flag it as a warning, not an error
|
||||
issues.append(
|
||||
f"pending_with_input_media: has input_media_gcs_path but status=pending (may be stuck)"
|
||||
)
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def check_job_consistency_detailed(job: Job) -> Dict[str, Any]:
|
||||
"""
|
||||
Check job consistency with detailed information for debugging.
|
||||
|
||||
Returns a dictionary with:
|
||||
- job_id: The job ID
|
||||
- status: Current job status
|
||||
- issues: List of issue descriptions
|
||||
- state_data_summary: Summary of relevant state_data fields
|
||||
- is_healthy: Boolean indicating if no issues were found
|
||||
|
||||
Args:
|
||||
job: Job object to check
|
||||
|
||||
Returns:
|
||||
Dictionary with detailed consistency information
|
||||
"""
|
||||
issues = check_job_consistency(job)
|
||||
state_data = job.state_data or {}
|
||||
|
||||
return {
|
||||
"job_id": job.job_id,
|
||||
"status": job.status.value if hasattr(job.status, 'value') else str(job.status),
|
||||
"issues": issues,
|
||||
"state_data_summary": {
|
||||
"audio_complete": state_data.get('audio_complete', False),
|
||||
"lyrics_complete": state_data.get('lyrics_complete', False),
|
||||
"screens_progress_stage": state_data.get('screens_progress', {}).get('stage'),
|
||||
"has_input_media_gcs_path": bool(job.input_media_gcs_path),
|
||||
},
|
||||
"is_healthy": len(issues) == 0,
|
||||
}
|
||||
|
||||
|
||||
def get_worker_valid_statuses() -> Dict[str, List[JobStatus]]:
|
||||
"""
|
||||
Get mapping of worker names to their valid job statuses.
|
||||
|
||||
This is used by workers to validate they should run for a given job.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping worker name to list of valid JobStatus values
|
||||
"""
|
||||
return {
|
||||
"audio_worker": [
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.DOWNLOADING_AUDIO,
|
||||
JobStatus.SEPARATING_STAGE1,
|
||||
JobStatus.SEPARATING_STAGE2,
|
||||
],
|
||||
"lyrics_worker": [
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.TRANSCRIBING,
|
||||
JobStatus.CORRECTING,
|
||||
],
|
||||
"screens_worker": [
|
||||
# Screens worker can be triggered when both audio+lyrics complete
|
||||
# which can happen in any of these states
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.AUDIO_COMPLETE,
|
||||
JobStatus.LYRICS_COMPLETE,
|
||||
# Also valid if re-running after a failure
|
||||
JobStatus.GENERATING_SCREENS,
|
||||
],
|
||||
"video_worker": [
|
||||
JobStatus.INSTRUMENTAL_SELECTED,
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
JobStatus.GENERATING_VIDEO,
|
||||
],
|
||||
"render_video_worker": [
|
||||
JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def validate_worker_can_run(worker_name: str, job: Job) -> Optional[str]:
|
||||
"""
|
||||
Validate that a worker should run for the given job.
|
||||
|
||||
Args:
|
||||
worker_name: Name of the worker (e.g., "screens_worker")
|
||||
job: Job object to validate
|
||||
|
||||
Returns:
|
||||
None if valid, or an error message string if invalid
|
||||
"""
|
||||
valid_statuses_map = get_worker_valid_statuses()
|
||||
|
||||
if worker_name not in valid_statuses_map:
|
||||
# Unknown worker, can't validate
|
||||
return None
|
||||
|
||||
valid_statuses = valid_statuses_map[worker_name]
|
||||
|
||||
# Normalize status
|
||||
status = job.status
|
||||
if isinstance(status, str):
|
||||
try:
|
||||
status = JobStatus(status)
|
||||
except ValueError:
|
||||
return f"Invalid job status: {job.status}"
|
||||
|
||||
if status not in valid_statuses:
|
||||
valid_names = [s.value for s in valid_statuses]
|
||||
return (
|
||||
f"{worker_name} called but job status is {status.value}, "
|
||||
f"expected one of {valid_names}. This may indicate a bug in the trigger logic."
|
||||
)
|
||||
|
||||
return None
|
||||
+113
-17
@@ -14,7 +14,7 @@ from datetime import datetime
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
from backend.config import settings
|
||||
from backend.exceptions import RateLimitExceededError
|
||||
from backend.exceptions import RateLimitExceededError, InvalidStateTransitionError
|
||||
from backend.models.job import Job, JobStatus, JobCreate, STATE_TRANSITIONS
|
||||
from backend.models.worker_log import WorkerLogEntry
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
@@ -316,53 +316,87 @@ class JobManager:
|
||||
self.firestore.delete_job(job_id)
|
||||
logger.info(f"Deleted job {job_id}")
|
||||
|
||||
def validate_state_transition(self, job_id: str, new_status: JobStatus) -> bool:
|
||||
def validate_state_transition(
|
||||
self,
|
||||
job_id: str,
|
||||
new_status: JobStatus,
|
||||
raise_on_invalid: bool = True
|
||||
) -> bool:
|
||||
"""
|
||||
Validate that a state transition is legal.
|
||||
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
new_status: Target state
|
||||
raise_on_invalid: If True, raise InvalidStateTransitionError on invalid
|
||||
transitions. If False, return False (legacy behavior).
|
||||
|
||||
Returns:
|
||||
True if transition is valid, False otherwise
|
||||
True if transition is valid
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If transition is invalid and raise_on_invalid=True
|
||||
ValueError: If job not found and raise_on_invalid=True
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
error_msg = f"Job {job_id} not found"
|
||||
logger.error(error_msg)
|
||||
if raise_on_invalid:
|
||||
raise ValueError(error_msg)
|
||||
return False
|
||||
|
||||
|
||||
current_status = job.status
|
||||
valid_transitions = STATE_TRANSITIONS.get(current_status, [])
|
||||
|
||||
|
||||
if new_status not in valid_transitions:
|
||||
logger.error(
|
||||
error_msg = (
|
||||
f"Invalid state transition for job {job_id}: "
|
||||
f"{current_status} -> {new_status}. "
|
||||
f"Valid transitions: {valid_transitions}"
|
||||
f"Valid transitions: {[s.value for s in valid_transitions]}"
|
||||
)
|
||||
logger.error(error_msg)
|
||||
|
||||
if raise_on_invalid:
|
||||
raise InvalidStateTransitionError(
|
||||
message=error_msg,
|
||||
job_id=job_id,
|
||||
from_status=current_status,
|
||||
to_status=new_status.value if hasattr(new_status, 'value') else str(new_status),
|
||||
valid_transitions=[s.value for s in valid_transitions]
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def transition_to_state(
|
||||
self,
|
||||
job_id: str,
|
||||
new_status: JobStatus,
|
||||
progress: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
state_data_updates: Optional[Dict[str, Any]] = None
|
||||
state_data_updates: Optional[Dict[str, Any]] = None,
|
||||
raise_on_invalid: bool = True
|
||||
) -> bool:
|
||||
"""
|
||||
Transition job to new state with validation.
|
||||
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
new_status: Target state
|
||||
progress: Progress percentage (0-100)
|
||||
message: Timeline message
|
||||
state_data_updates: Updates to state_data field
|
||||
|
||||
raise_on_invalid: If True (default), raise InvalidStateTransitionError
|
||||
on invalid transitions. If False, return False silently.
|
||||
|
||||
Returns:
|
||||
True if transition succeeded, False otherwise
|
||||
True if transition succeeded, False if failed (only when raise_on_invalid=False)
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If transition is invalid and raise_on_invalid=True
|
||||
"""
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
if not self.validate_state_transition(job_id, new_status, raise_on_invalid=raise_on_invalid):
|
||||
return False
|
||||
|
||||
updates = {
|
||||
@@ -768,7 +802,69 @@ class JobManager:
|
||||
lyrics_complete = job.state_data.get('lyrics_complete', False)
|
||||
|
||||
return audio_complete and lyrics_complete
|
||||
|
||||
|
||||
async def start_job_processing(
|
||||
self,
|
||||
job_id: str,
|
||||
progress: int = 15,
|
||||
message: str = "Starting audio and lyrics processing"
|
||||
) -> None:
|
||||
"""
|
||||
Transition job to DOWNLOADING and trigger audio + lyrics workers.
|
||||
|
||||
This is the single entry point for starting job processing after audio
|
||||
is available. All handlers (file_upload, webhook, audio_search) should
|
||||
use this method to ensure consistent state transitions.
|
||||
|
||||
The method:
|
||||
1. Validates job exists and has input_media_gcs_path
|
||||
2. Transitions to DOWNLOADING status (raises if invalid)
|
||||
3. Triggers audio and lyrics workers in parallel
|
||||
|
||||
Args:
|
||||
job_id: Job ID to start processing
|
||||
progress: Progress percentage to set (default 15)
|
||||
message: Timeline message (default "Starting audio and lyrics processing")
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If job can't transition to DOWNLOADING
|
||||
ValueError: If job not found or missing input_media_gcs_path
|
||||
|
||||
Example:
|
||||
# After downloading audio to GCS:
|
||||
job_manager.update_job(job_id, {'input_media_gcs_path': audio_path})
|
||||
await job_manager.start_job_processing(job_id)
|
||||
"""
|
||||
import asyncio
|
||||
from backend.services.worker_service import get_worker_service
|
||||
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
|
||||
if not job.input_media_gcs_path:
|
||||
raise ValueError(
|
||||
f"Job {job_id} missing input_media_gcs_path. "
|
||||
"Audio must be downloaded to GCS before starting processing."
|
||||
)
|
||||
|
||||
# Transition to DOWNLOADING - this will raise if invalid
|
||||
self.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=progress,
|
||||
message=message
|
||||
)
|
||||
|
||||
# Trigger both workers in parallel
|
||||
worker_service = get_worker_service()
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id}: Started processing (audio + lyrics workers triggered)")
|
||||
|
||||
def mark_audio_complete(self, job_id: str) -> None:
|
||||
"""
|
||||
Mark audio processing as complete and check if can proceed.
|
||||
|
||||
@@ -218,16 +218,16 @@ class TestCountdownStateTransitions:
|
||||
"""
|
||||
Countdown state should survive job state transitions.
|
||||
|
||||
When job moves from RENDERING_VIDEO to ENCODING,
|
||||
When job moves from GENERATING_VIDEO to ENCODING,
|
||||
the lyrics_metadata should remain intact.
|
||||
"""
|
||||
job_id = f"transition-test-{datetime.now(UTC).timestamp()}"
|
||||
|
||||
try:
|
||||
# Create job in RENDERING_VIDEO state
|
||||
# Create job in GENERATING_VIDEO state (valid predecessor to ENCODING)
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.RENDERING_VIDEO,
|
||||
status=JobStatus.GENERATING_VIDEO,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
@@ -241,7 +241,7 @@ class TestCountdownStateTransitions:
|
||||
)
|
||||
firestore_service.create_job(job)
|
||||
|
||||
# Transition to ENCODING
|
||||
# Transition to ENCODING (valid transition from GENERATING_VIDEO)
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.ENCODING,
|
||||
|
||||
@@ -229,6 +229,12 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
mock_job.job_id = "test-job-workers"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it to call workers directly
|
||||
async def mock_start_job_processing(job_id, progress=15, message=""):
|
||||
await mock_worker_service_local.trigger_audio_worker(job_id)
|
||||
await mock_worker_service_local.trigger_lyrics_worker(job_id)
|
||||
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
@@ -245,7 +251,7 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
email_service=mock_email_service,
|
||||
)
|
||||
|
||||
# Verify workers were triggered
|
||||
# Verify workers were triggered via start_job_processing
|
||||
mock_worker_service_local.trigger_audio_worker.assert_called_once_with(mock_job.job_id)
|
||||
mock_worker_service_local.trigger_lyrics_worker.assert_called_once_with(mock_job.job_id)
|
||||
|
||||
@@ -269,6 +275,9 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
mock_job.job_id = "test-job-notifications"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it as simple AsyncMock
|
||||
mock_job_manager.start_job_processing = AsyncMock()
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
@@ -389,6 +398,12 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
mock_job.job_id = "test-job-no-search"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it to call workers directly
|
||||
async def mock_start_job_processing(job_id, progress=15, message=""):
|
||||
await mock_worker_service_local.trigger_audio_worker(job_id)
|
||||
await mock_worker_service_local.trigger_lyrics_worker(job_id)
|
||||
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.audio_search_service.get_audio_search_service') as mock_get_audio, \
|
||||
@@ -412,7 +427,7 @@ class TestMadeForYouYouTubeOrderFlow:
|
||||
# Audio search should NOT be called for YouTube URL orders
|
||||
mock_audio_service.search.assert_not_called()
|
||||
|
||||
# Workers should be triggered directly
|
||||
# Workers should be triggered directly via start_job_processing
|
||||
mock_worker_service_local.trigger_audio_worker.assert_called_once()
|
||||
|
||||
|
||||
@@ -548,6 +563,12 @@ class TestMadeForYouWebhookIntegration:
|
||||
mock_job.job_id = "test-job-webhook-flow"
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
|
||||
# start_job_processing is async - mock it to call workers directly
|
||||
async def mock_start_job_processing(job_id, progress=15, message=""):
|
||||
await mock_worker_service_local.trigger_audio_worker(job_id)
|
||||
await mock_worker_service_local.trigger_lyrics_worker(job_id)
|
||||
mock_job_manager.start_job_processing = AsyncMock(side_effect=mock_start_job_processing)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service_local), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
@@ -573,7 +594,7 @@ class TestMadeForYouWebhookIntegration:
|
||||
# 2. Verify YouTube download was called
|
||||
mock_youtube_download_service.download.assert_called_once()
|
||||
|
||||
# 3. Verify workers were triggered
|
||||
# 3. Verify workers were triggered via start_job_processing
|
||||
mock_worker_service_local.trigger_audio_worker.assert_called_once()
|
||||
mock_worker_service_local.trigger_lyrics_worker.assert_called_once()
|
||||
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
"""
|
||||
Integration tests for the job state machine flow.
|
||||
|
||||
These tests verify complete state transitions without mocking the state machine,
|
||||
ensuring the flow works correctly end-to-end (with mocked external services).
|
||||
|
||||
Created as part of state machine robustness improvements (2026-02-02).
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
|
||||
# Mock Firestore before importing JobManager
|
||||
import sys
|
||||
sys.modules['google.cloud.firestore'] = MagicMock()
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_manager import JobManager, InvalidStateTransitionError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_firestore_service():
|
||||
"""Mock FirestoreService."""
|
||||
with patch('backend.services.job_manager.FirestoreService') as mock:
|
||||
service = Mock()
|
||||
mock.return_value = service
|
||||
yield service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def job_manager(mock_firestore_service):
|
||||
"""Create JobManager instance with mocked Firestore."""
|
||||
return JobManager()
|
||||
|
||||
|
||||
class TestMadeForYouYouTubeFlow:
|
||||
"""
|
||||
Integration tests for Made-For-You YouTube order flow.
|
||||
|
||||
This was the flow that caused jobs 06cfea29 and 984da08b to get stuck.
|
||||
The fix ensures proper state transitions from PENDING -> DOWNLOADING.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_flow_pending_to_downloading(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test the flow from PENDING to DOWNLOADING for YouTube orders.
|
||||
|
||||
This verifies the fix for the bug where jobs got stuck at PENDING
|
||||
because start_job_processing wasn't called properly.
|
||||
"""
|
||||
# Create initial job in PENDING state
|
||||
job = Job(
|
||||
job_id="test-mfy-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/test.mp3",
|
||||
state_data={},
|
||||
)
|
||||
# Access the firestore through the job_manager's instance
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job_status.return_value = True
|
||||
|
||||
# Mock worker service
|
||||
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
|
||||
mock_worker = Mock()
|
||||
mock_worker.trigger_audio_worker = AsyncMock()
|
||||
mock_worker.trigger_lyrics_worker = AsyncMock()
|
||||
mock_get_worker.return_value = mock_worker
|
||||
|
||||
# Call start_job_processing - this is what _handle_made_for_you_order now uses
|
||||
await job_manager.start_job_processing("test-mfy-001")
|
||||
|
||||
# Verify transition to DOWNLOADING was called
|
||||
job_manager.firestore.update_job_status.assert_called()
|
||||
call_kwargs = job_manager.firestore.update_job_status.call_args.kwargs
|
||||
assert call_kwargs['status'] == JobStatus.DOWNLOADING
|
||||
|
||||
# Verify both workers were triggered
|
||||
mock_worker.trigger_audio_worker.assert_called_once_with("test-mfy-001")
|
||||
mock_worker.trigger_lyrics_worker.assert_called_once_with("test-mfy-001")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fails_if_job_stuck_at_wrong_state(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that starting processing fails if job is in wrong state.
|
||||
|
||||
Jobs that are already past PENDING (e.g., DOWNLOADING) shouldn't
|
||||
be able to start processing again - this would indicate a bug.
|
||||
"""
|
||||
# Create job already in DOWNLOADING state
|
||||
job = Job(
|
||||
job_id="test-mfy-002",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING, # Already past PENDING
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/test.mp3",
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
|
||||
# Attempting to start processing should raise InvalidStateTransitionError
|
||||
# because DOWNLOADING -> DOWNLOADING is not a valid transition
|
||||
with pytest.raises(InvalidStateTransitionError):
|
||||
await job_manager.start_job_processing("test-mfy-002")
|
||||
|
||||
|
||||
class TestParallelProcessingCoordination:
|
||||
"""
|
||||
Integration tests for parallel audio + lyrics processing coordination.
|
||||
|
||||
Both workers run in parallel and set flags when complete.
|
||||
Screens worker is triggered when both are done.
|
||||
"""
|
||||
|
||||
def test_audio_complete_updates_state_data(self, job_manager, mock_firestore_service):
|
||||
"""Test that mark_audio_complete updates state_data correctly."""
|
||||
job = Job(
|
||||
job_id="test-parallel-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job.return_value = True
|
||||
|
||||
# Call mark_audio_complete
|
||||
job_manager.mark_audio_complete("test-parallel-001")
|
||||
|
||||
# Verify update_job was called to set the audio_complete flag
|
||||
job_manager.firestore.update_job.assert_called()
|
||||
|
||||
def test_lyrics_complete_updates_state_data(self, job_manager, mock_firestore_service):
|
||||
"""Test that mark_lyrics_complete updates state_data correctly."""
|
||||
job = Job(
|
||||
job_id="test-parallel-002",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job.return_value = True
|
||||
|
||||
# Call mark_lyrics_complete
|
||||
job_manager.mark_lyrics_complete("test-parallel-002")
|
||||
|
||||
# Verify update_job was called to set the lyrics_complete flag
|
||||
job_manager.firestore.update_job.assert_called()
|
||||
|
||||
def test_both_complete_triggers_screens(self, job_manager, mock_firestore_service):
|
||||
"""Test that screens worker is triggered when both complete."""
|
||||
# First call: Job with audio already complete (for update_state_data)
|
||||
# Second call: Job with both complete (for check_parallel_processing_complete)
|
||||
job_audio_only = Job(
|
||||
job_id="test-parallel-003",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True},
|
||||
)
|
||||
job_both_complete = Job(
|
||||
job_id="test-parallel-003",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True, 'lyrics_complete': True},
|
||||
)
|
||||
# Return different jobs on subsequent calls
|
||||
job_manager.firestore.get_job.side_effect = [job_audio_only, job_both_complete]
|
||||
job_manager.firestore.update_job.return_value = True
|
||||
|
||||
# Mock the screens worker trigger
|
||||
with patch.object(job_manager, '_trigger_screens_worker') as mock_trigger:
|
||||
# Now lyrics completes
|
||||
job_manager.mark_lyrics_complete("test-parallel-003")
|
||||
|
||||
# Verify screens worker was triggered (both complete)
|
||||
mock_trigger.assert_called_once_with("test-parallel-003")
|
||||
|
||||
|
||||
class TestStateTransitionValidation:
|
||||
"""
|
||||
Integration tests for state transition validation.
|
||||
|
||||
Ensures invalid transitions are rejected with clear errors.
|
||||
"""
|
||||
|
||||
def test_valid_transition_succeeds(self, job_manager, mock_firestore_service):
|
||||
"""Test that valid transitions succeed."""
|
||||
job = Job(
|
||||
job_id="test-transition-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
job_manager.firestore.update_job_status.return_value = True
|
||||
|
||||
# PENDING -> DOWNLOADING is valid
|
||||
result = job_manager.transition_to_state(
|
||||
job_id="test-transition-001",
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=10,
|
||||
message="Starting processing"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transitions raise exception by default."""
|
||||
job = Job(
|
||||
job_id="test-transition-002",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
|
||||
# PENDING -> COMPLETE is invalid (skips many steps)
|
||||
with pytest.raises(InvalidStateTransitionError) as exc_info:
|
||||
job_manager.transition_to_state(
|
||||
job_id="test-transition-002",
|
||||
new_status=JobStatus.COMPLETE,
|
||||
)
|
||||
|
||||
assert "pending" in str(exc_info.value).lower()
|
||||
assert "complete" in str(exc_info.value).lower()
|
||||
|
||||
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transitions return False when raise_on_invalid=False."""
|
||||
job = Job(
|
||||
job_id="test-transition-003",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={},
|
||||
)
|
||||
job_manager.firestore.get_job.return_value = job
|
||||
|
||||
# PENDING -> COMPLETE is invalid, but with raise_on_invalid=False
|
||||
result = job_manager.transition_to_state(
|
||||
job_id="test-transition-003",
|
||||
new_status=JobStatus.COMPLETE,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestAdminResetFlow:
|
||||
"""
|
||||
Integration tests for admin reset functionality.
|
||||
|
||||
Admin resets bypass normal state machine validation but must
|
||||
properly clear state_data flags to avoid inconsistent state.
|
||||
"""
|
||||
|
||||
def test_reset_to_pending_clears_parallel_flags(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that resetting to pending clears audio_complete and lyrics_complete.
|
||||
|
||||
This was a bug where the Start button would reset to pending but
|
||||
leave these flags set, causing inconsistent state.
|
||||
"""
|
||||
# This is tested via the admin endpoint, not directly in JobManager
|
||||
# The test in test_admin_job_reset.py covers this
|
||||
pass
|
||||
|
||||
|
||||
class TestConsistencyWithHealthService:
|
||||
"""
|
||||
Integration tests that verify the job health service correctly
|
||||
identifies inconsistencies in real job flows.
|
||||
"""
|
||||
|
||||
def test_detects_stuck_job_scenario(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that the health service would detect the exact bug that
|
||||
caused jobs 06cfea29 and 984da08b to get stuck.
|
||||
"""
|
||||
from backend.services.job_health_service import check_job_consistency
|
||||
|
||||
# Simulate the stuck job scenario:
|
||||
# - Job has input_media_gcs_path (YouTube download completed)
|
||||
# - But status is still PENDING (transition never happened)
|
||||
stuck_job = Job(
|
||||
job_id="test-stuck-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/downloaded.mp3",
|
||||
state_data={},
|
||||
)
|
||||
|
||||
issues = check_job_consistency(stuck_job)
|
||||
|
||||
# Should detect the pending_with_input_media issue
|
||||
assert len(issues) > 0
|
||||
assert any("pending_with_input_media" in issue for issue in issues)
|
||||
|
||||
def test_detects_flag_status_mismatch(self, job_manager, mock_firestore_service):
|
||||
"""
|
||||
Test that the health service detects when flags don't match status.
|
||||
"""
|
||||
from backend.services.job_health_service import check_job_consistency
|
||||
|
||||
# Simulate: audio_complete=True but status still PENDING
|
||||
# (would happen if mark_audio_complete ran but transition didn't)
|
||||
inconsistent_job = Job(
|
||||
job_id="test-inconsistent-001",
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True},
|
||||
)
|
||||
|
||||
issues = check_job_consistency(inconsistent_job)
|
||||
|
||||
# Should detect the audio_complete_status_mismatch issue
|
||||
assert len(issues) > 0
|
||||
assert any("audio_complete_status_mismatch" in issue for issue in issues)
|
||||
@@ -162,27 +162,6 @@ class TestResetJobToAwaitingReview:
|
||||
assert data["new_status"] == "awaiting_review"
|
||||
|
||||
|
||||
class TestResetJobToAwaitingInstrumentalSelection:
|
||||
"""Tests for resetting job to AWAITING_INSTRUMENTAL_SELECTION state."""
|
||||
|
||||
def test_reset_to_awaiting_instrumental_selection(self, client, mock_job):
|
||||
"""Test resetting a job to AWAITING_INSTRUMENTAL_SELECTION state."""
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
"/api/admin/jobs/test-job-123/reset",
|
||||
json={"target_state": "awaiting_instrumental_selection"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["new_status"] == "awaiting_instrumental_selection"
|
||||
|
||||
|
||||
class TestResetJobValidation:
|
||||
"""Tests for reset endpoint validation."""
|
||||
|
||||
@@ -353,6 +332,51 @@ class TestResetJobClearsStateData:
|
||||
data = response.json()
|
||||
assert "cleared_data" in data or "state_data" in str(data)
|
||||
|
||||
def test_reset_to_pending_clears_parallel_processing_flags(self, client):
|
||||
"""Test that resetting to PENDING clears audio_complete and lyrics_complete.
|
||||
|
||||
This is critical for proper re-processing - without clearing these flags,
|
||||
workers may think audio/lyrics processing is already done.
|
||||
"""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime, UTC
|
||||
|
||||
# Create a job with audio_complete and lyrics_complete set
|
||||
mock_job = Job(
|
||||
job_id="test-job-123",
|
||||
status=JobStatus.COMPLETE,
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={
|
||||
"audio_complete": True,
|
||||
"lyrics_complete": True,
|
||||
"audio_search_results": [{"source": "test"}],
|
||||
}
|
||||
)
|
||||
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
"/api/admin/jobs/test-job-123/reset",
|
||||
json={"target_state": "pending"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify the cleared_data includes the parallel processing flags
|
||||
cleared_keys = data.get("cleared_data", [])
|
||||
assert "audio_complete" in cleared_keys, \
|
||||
f"audio_complete should be cleared. Cleared: {cleared_keys}"
|
||||
assert "lyrics_complete" in cleared_keys, \
|
||||
f"lyrics_complete should be cleared. Cleared: {cleared_keys}"
|
||||
|
||||
def test_reset_to_awaiting_review_preserves_audio_data(self, client, mock_job):
|
||||
"""Test that resetting to AWAITING_REVIEW preserves audio stems."""
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
@@ -368,12 +392,15 @@ class TestResetJobClearsStateData:
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_reset_to_instrumental_preserves_review_data(self, client, mock_job):
|
||||
"""Test that resetting to AWAITING_INSTRUMENTAL_SELECTION preserves review data."""
|
||||
def test_rejects_deprecated_awaiting_instrumental_selection(self, client, mock_job):
|
||||
"""Test that awaiting_instrumental_selection is no longer a valid reset target.
|
||||
|
||||
This state was deprecated in Jan 2026 when lyrics review and instrumental
|
||||
selection were combined into a single review step.
|
||||
"""
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -381,7 +408,9 @@ class TestResetJobClearsStateData:
|
||||
json={"target_state": "awaiting_instrumental_selection"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
# Should be rejected as invalid
|
||||
assert response.status_code == 400
|
||||
assert "invalid" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
class TestResetJobToInstrumentalSelected:
|
||||
|
||||
@@ -7,7 +7,7 @@ route logic without hitting real cloud services.
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from unittest.mock import MagicMock, AsyncMock, patch, Mock
|
||||
from fastapi.testclient import TestClient
|
||||
from io import BytesIO
|
||||
|
||||
@@ -131,13 +131,91 @@ class TestHealthRoutes:
|
||||
assert data["queue_length"] == 5
|
||||
|
||||
|
||||
class TestJobConsistencyEndpoint:
|
||||
"""Tests for the /health/job-consistency endpoint."""
|
||||
|
||||
@pytest.fixture
|
||||
def client(self):
|
||||
"""Create test client."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
from backend.main import app
|
||||
return TestClient(app)
|
||||
|
||||
def test_job_consistency_returns_empty_when_no_jobs(self, client):
|
||||
"""Test endpoint returns empty results when no jobs exist."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.list_jobs.return_value = []
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_checked"] == 0
|
||||
assert data["inconsistent_count"] == 0
|
||||
assert data["inconsistent_jobs"] == []
|
||||
|
||||
def test_job_consistency_detects_issues(self, client):
|
||||
"""Test endpoint detects inconsistent jobs."""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime, UTC
|
||||
|
||||
stuck_job = Job(
|
||||
job_id="stuck-123",
|
||||
artist="Test",
|
||||
title="Song",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={'audio_complete': True},
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.list_jobs.return_value = [stuck_job]
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_checked"] == 1
|
||||
assert data["inconsistent_count"] == 1
|
||||
assert len(data["inconsistent_jobs"]) == 1
|
||||
assert data["inconsistent_jobs"][0]["job_id"] == "stuck-123"
|
||||
|
||||
def test_job_consistency_filters_by_status(self, client):
|
||||
"""Test endpoint filters by status parameter."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.list_jobs.return_value = []
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency?status=pending")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status_filter"] == "pending"
|
||||
|
||||
def test_job_consistency_rejects_invalid_status(self, client):
|
||||
"""Test endpoint rejects invalid status parameter."""
|
||||
with patch('backend.api.routes.health.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.get("/api/health/job-consistency?status=invalid_status")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
assert "Invalid status" in data["error"]
|
||||
|
||||
|
||||
class TestJobRoutes:
|
||||
"""Tests for jobs.py routes.
|
||||
|
||||
|
||||
Note: These tests verify the route module structure.
|
||||
Full integration tests are in test_api_integration.py.
|
||||
"""
|
||||
|
||||
|
||||
def test_jobs_router_exists(self):
|
||||
"""Test jobs router can be imported."""
|
||||
from backend.api.routes import jobs
|
||||
|
||||
@@ -269,10 +269,12 @@ class TestUrlJobWorkerSequencing:
|
||||
async def test_parallel_worker_triggers_both_workers(self):
|
||||
"""
|
||||
_trigger_workers_parallel should trigger both workers (for uploaded files).
|
||||
Note: This function now lives in jobs.py (direct job creation endpoint).
|
||||
Most code paths use job_manager.start_job_processing() instead.
|
||||
"""
|
||||
from backend.api.routes.file_upload import _trigger_workers_parallel
|
||||
from backend.api.routes.jobs import _trigger_workers_parallel
|
||||
|
||||
with patch('backend.api.routes.file_upload.worker_service') as mock_ws:
|
||||
with patch('backend.api.routes.jobs.worker_service') as mock_ws:
|
||||
mock_ws.trigger_audio_worker = AsyncMock()
|
||||
mock_ws.trigger_lyrics_worker = AsyncMock()
|
||||
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
"""
|
||||
Tests for the job health service.
|
||||
|
||||
Tests consistency checks and worker validation functions.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_health_service import (
|
||||
check_job_consistency,
|
||||
check_job_consistency_detailed,
|
||||
validate_worker_can_run,
|
||||
get_worker_valid_statuses,
|
||||
)
|
||||
|
||||
|
||||
def create_test_job(
|
||||
job_id: str = "test-job-123",
|
||||
status: JobStatus = JobStatus.PENDING,
|
||||
state_data: dict = None,
|
||||
input_media_gcs_path: str = None,
|
||||
) -> Job:
|
||||
"""Helper to create test Job objects."""
|
||||
return Job(
|
||||
job_id=job_id,
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
status=status,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data=state_data or {},
|
||||
input_media_gcs_path=input_media_gcs_path,
|
||||
)
|
||||
|
||||
|
||||
class TestCheckJobConsistency:
|
||||
"""Tests for check_job_consistency function."""
|
||||
|
||||
def test_healthy_job_has_no_issues(self):
|
||||
"""Test that a properly configured job has no issues."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data={},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
def test_detects_audio_complete_with_pending_status(self):
|
||||
"""Test detection of audio_complete flag with pending status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={'audio_complete': True},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "audio_complete_status_mismatch" in issues[0]
|
||||
assert "audio_complete=True but status=pending" in issues[0]
|
||||
|
||||
def test_detects_lyrics_complete_with_pending_status(self):
|
||||
"""Test detection of lyrics_complete flag with pending status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={'lyrics_complete': True},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "lyrics_complete_status_mismatch" in issues[0]
|
||||
|
||||
def test_detects_screens_complete_with_wrong_status(self):
|
||||
"""Test detection of screens complete but status not in post-screens states."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data={'screens_progress': {'stage': 'complete'}},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "screens_complete_status_mismatch" in issues[0]
|
||||
|
||||
def test_screens_complete_valid_with_awaiting_review(self):
|
||||
"""Test that screens complete is valid with AWAITING_REVIEW status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.AWAITING_REVIEW,
|
||||
state_data={'screens_progress': {'stage': 'complete'}},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
def test_screens_complete_valid_with_complete_status(self):
|
||||
"""Test that screens complete is valid with COMPLETE status."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.COMPLETE,
|
||||
state_data={'screens_progress': {'stage': 'complete'}},
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
def test_detects_pending_with_input_media_path(self):
|
||||
"""Test detection of job stuck at pending with input_media_gcs_path."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "pending_with_input_media" in issues[0]
|
||||
|
||||
def test_multiple_issues_detected(self):
|
||||
"""Test that multiple issues are detected simultaneously."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={
|
||||
'audio_complete': True,
|
||||
'lyrics_complete': True,
|
||||
},
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 3
|
||||
|
||||
def test_handles_invalid_status_string(self):
|
||||
"""Test handling of invalid status string."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
job.status = "invalid_status" # type: ignore
|
||||
issues = check_job_consistency(job)
|
||||
assert len(issues) == 1
|
||||
assert "invalid_status" in issues[0]
|
||||
|
||||
def test_handles_empty_state_data(self):
|
||||
"""Test handling of empty or None state_data."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data=None,
|
||||
)
|
||||
issues = check_job_consistency(job)
|
||||
assert issues == []
|
||||
|
||||
|
||||
class TestCheckJobConsistencyDetailed:
|
||||
"""Tests for check_job_consistency_detailed function."""
|
||||
|
||||
def test_returns_detailed_info(self):
|
||||
"""Test that detailed info is returned."""
|
||||
job = create_test_job(
|
||||
status=JobStatus.PENDING,
|
||||
state_data={'audio_complete': True},
|
||||
input_media_gcs_path="gs://bucket/audio.mp3",
|
||||
)
|
||||
result = check_job_consistency_detailed(job)
|
||||
|
||||
assert result['job_id'] == "test-job-123"
|
||||
assert result['status'] == "pending"
|
||||
assert len(result['issues']) == 2
|
||||
assert result['state_data_summary']['audio_complete'] is True
|
||||
assert result['state_data_summary']['lyrics_complete'] is False
|
||||
assert result['state_data_summary']['has_input_media_gcs_path'] is True
|
||||
assert result['is_healthy'] is False
|
||||
|
||||
def test_healthy_job_returns_is_healthy_true(self):
|
||||
"""Test that healthy job returns is_healthy=True."""
|
||||
job = create_test_job(status=JobStatus.DOWNLOADING)
|
||||
result = check_job_consistency_detailed(job)
|
||||
assert result['is_healthy'] is True
|
||||
assert result['issues'] == []
|
||||
|
||||
|
||||
class TestValidateWorkerCanRun:
|
||||
"""Tests for validate_worker_can_run function."""
|
||||
|
||||
def test_audio_worker_valid_with_downloading_status(self):
|
||||
"""Test audio worker is valid with DOWNLOADING status."""
|
||||
job = create_test_job(status=JobStatus.DOWNLOADING)
|
||||
error = validate_worker_can_run("audio_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_audio_worker_invalid_with_pending_status(self):
|
||||
"""Test audio worker is invalid with PENDING status."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
error = validate_worker_can_run("audio_worker", job)
|
||||
assert error is not None
|
||||
assert "audio_worker called but job status is pending" in error
|
||||
|
||||
def test_lyrics_worker_valid_with_downloading_status(self):
|
||||
"""Test lyrics worker is valid with DOWNLOADING status."""
|
||||
job = create_test_job(status=JobStatus.DOWNLOADING)
|
||||
error = validate_worker_can_run("lyrics_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_screens_worker_valid_with_audio_complete_status(self):
|
||||
"""Test screens worker is valid with AUDIO_COMPLETE status."""
|
||||
job = create_test_job(status=JobStatus.AUDIO_COMPLETE)
|
||||
error = validate_worker_can_run("screens_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_screens_worker_valid_with_lyrics_complete_status(self):
|
||||
"""Test screens worker is valid with LYRICS_COMPLETE status."""
|
||||
job = create_test_job(status=JobStatus.LYRICS_COMPLETE)
|
||||
error = validate_worker_can_run("screens_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_screens_worker_invalid_with_awaiting_review_status(self):
|
||||
"""Test screens worker is invalid with AWAITING_REVIEW status (too late)."""
|
||||
job = create_test_job(status=JobStatus.AWAITING_REVIEW)
|
||||
error = validate_worker_can_run("screens_worker", job)
|
||||
assert error is not None
|
||||
assert "screens_worker called but job status is awaiting_review" in error
|
||||
|
||||
def test_video_worker_valid_with_instrumental_selected_status(self):
|
||||
"""Test video worker is valid with INSTRUMENTAL_SELECTED status."""
|
||||
job = create_test_job(status=JobStatus.INSTRUMENTAL_SELECTED)
|
||||
error = validate_worker_can_run("video_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_video_worker_invalid_with_pending_status(self):
|
||||
"""Test video worker is invalid with PENDING status."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
error = validate_worker_can_run("video_worker", job)
|
||||
assert error is not None
|
||||
assert "video_worker called but job status is pending" in error
|
||||
|
||||
def test_render_video_worker_valid_with_review_complete_status(self):
|
||||
"""Test render video worker is valid with REVIEW_COMPLETE status."""
|
||||
job = create_test_job(status=JobStatus.REVIEW_COMPLETE)
|
||||
error = validate_worker_can_run("render_video_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_unknown_worker_returns_none(self):
|
||||
"""Test that unknown worker names return None (no validation)."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
error = validate_worker_can_run("unknown_worker", job)
|
||||
assert error is None
|
||||
|
||||
def test_handles_invalid_status_string(self):
|
||||
"""Test handling of invalid status string."""
|
||||
job = create_test_job(status=JobStatus.PENDING)
|
||||
job.status = "invalid_status" # type: ignore
|
||||
error = validate_worker_can_run("audio_worker", job)
|
||||
assert error is not None
|
||||
assert "Invalid job status" in error
|
||||
|
||||
|
||||
class TestGetWorkerValidStatuses:
|
||||
"""Tests for get_worker_valid_statuses function."""
|
||||
|
||||
def test_returns_dict_with_all_workers(self):
|
||||
"""Test that all known workers are included."""
|
||||
statuses = get_worker_valid_statuses()
|
||||
assert "audio_worker" in statuses
|
||||
assert "lyrics_worker" in statuses
|
||||
assert "screens_worker" in statuses
|
||||
assert "video_worker" in statuses
|
||||
assert "render_video_worker" in statuses
|
||||
|
||||
def test_audio_worker_includes_downloading(self):
|
||||
"""Test that audio worker includes DOWNLOADING status."""
|
||||
statuses = get_worker_valid_statuses()
|
||||
assert JobStatus.DOWNLOADING in statuses["audio_worker"]
|
||||
|
||||
def test_lyrics_worker_includes_downloading(self):
|
||||
"""Test that lyrics worker includes DOWNLOADING status."""
|
||||
statuses = get_worker_valid_statuses()
|
||||
assert JobStatus.DOWNLOADING in statuses["lyrics_worker"]
|
||||
@@ -5,7 +5,7 @@ Tests the job lifecycle management without requiring actual Firestore connection
|
||||
Uses mocking to isolate the business logic.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
from unittest.mock import Mock, MagicMock, patch, AsyncMock
|
||||
from datetime import datetime, UTC
|
||||
|
||||
# Mock Firestore before importing JobManager
|
||||
@@ -14,6 +14,7 @@ sys.modules['google.cloud.firestore'] = MagicMock()
|
||||
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.models.job import Job, JobCreate, JobStatus
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -568,6 +569,288 @@ class TestDeleteStateDataKeys:
|
||||
assert result == []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# InvalidStateTransitionError Tests
|
||||
# =============================================================================
|
||||
|
||||
class TestInvalidStateTransitionError:
|
||||
"""Tests for InvalidStateTransitionError exception."""
|
||||
|
||||
def test_exception_stores_all_attributes(self):
|
||||
"""Test exception stores job_id, from_status, to_status, and valid_transitions."""
|
||||
error = InvalidStateTransitionError(
|
||||
message="Test error",
|
||||
job_id="test-123",
|
||||
from_status="pending",
|
||||
to_status="generating_screens",
|
||||
valid_transitions=["downloading", "searching_audio", "failed"]
|
||||
)
|
||||
|
||||
assert error.message == "Test error"
|
||||
assert error.job_id == "test-123"
|
||||
assert error.from_status == "pending"
|
||||
assert error.to_status == "generating_screens"
|
||||
assert error.valid_transitions == ["downloading", "searching_audio", "failed"]
|
||||
assert str(error) == "Test error"
|
||||
|
||||
def test_exception_defaults(self):
|
||||
"""Test exception has sensible defaults."""
|
||||
error = InvalidStateTransitionError(message="Minimal error")
|
||||
|
||||
assert error.message == "Minimal error"
|
||||
assert error.job_id == ""
|
||||
assert error.from_status == ""
|
||||
assert error.to_status == ""
|
||||
assert error.valid_transitions == []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# State Transition Validation Tests (raise_on_invalid behavior)
|
||||
# =============================================================================
|
||||
|
||||
class TestValidateStateTransitionRaisesBehavior:
|
||||
"""Tests for validate_state_transition with raise_on_invalid parameter."""
|
||||
|
||||
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition raises InvalidStateTransitionError by default."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING cannot transition directly to GENERATING_SCREENS
|
||||
with pytest.raises(InvalidStateTransitionError) as exc_info:
|
||||
job_manager.validate_state_transition("test123", JobStatus.GENERATING_SCREENS)
|
||||
|
||||
error = exc_info.value
|
||||
assert error.job_id == "test123"
|
||||
assert error.from_status == JobStatus.PENDING
|
||||
assert error.to_status == "generating_screens"
|
||||
assert "downloading" in error.valid_transitions
|
||||
|
||||
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition returns False when raise_on_invalid=False."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING cannot transition directly to GENERATING_SCREENS
|
||||
result = job_manager.validate_state_transition(
|
||||
"test123",
|
||||
JobStatus.GENERATING_SCREENS,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_valid_transition_returns_true(self, job_manager, mock_firestore_service):
|
||||
"""Test that valid transition returns True."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING can transition to DOWNLOADING
|
||||
result = job_manager.validate_state_transition("test123", JobStatus.DOWNLOADING)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_job_not_found_raises_value_error(self, job_manager, mock_firestore_service):
|
||||
"""Test that missing job raises ValueError when raise_on_invalid=True."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
with pytest.raises(ValueError, match="Job nonexistent not found"):
|
||||
job_manager.validate_state_transition("nonexistent", JobStatus.DOWNLOADING)
|
||||
|
||||
def test_job_not_found_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that missing job returns False when raise_on_invalid=False."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
result = job_manager.validate_state_transition(
|
||||
"nonexistent",
|
||||
JobStatus.DOWNLOADING,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestTransitionToStateRaisesBehavior:
|
||||
"""Tests for transition_to_state with raise_on_invalid parameter."""
|
||||
|
||||
def test_invalid_transition_raises_by_default(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition raises InvalidStateTransitionError by default."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
# PENDING cannot transition directly to AWAITING_REVIEW
|
||||
with pytest.raises(InvalidStateTransitionError) as exc_info:
|
||||
job_manager.transition_to_state("test123", JobStatus.AWAITING_REVIEW)
|
||||
|
||||
error = exc_info.value
|
||||
assert error.job_id == "test123"
|
||||
|
||||
def test_invalid_transition_returns_false_when_not_raising(self, job_manager, mock_firestore_service):
|
||||
"""Test that invalid transition returns False when raise_on_invalid=False."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
result = job_manager.transition_to_state(
|
||||
"test123",
|
||||
JobStatus.AWAITING_REVIEW,
|
||||
raise_on_invalid=False
|
||||
)
|
||||
|
||||
assert result is False
|
||||
# Verify update was NOT called since transition was invalid
|
||||
mock_firestore_service.update_job.assert_not_called()
|
||||
|
||||
def test_valid_transition_updates_job(self, job_manager, mock_firestore_service):
|
||||
"""Test that valid transition updates the job."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
result = job_manager.transition_to_state(
|
||||
"test123",
|
||||
JobStatus.DOWNLOADING,
|
||||
progress=15,
|
||||
message="Starting processing"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
# transition_to_state uses update_job_status internally
|
||||
mock_firestore_service.update_job_status.assert_called()
|
||||
call_kwargs = mock_firestore_service.update_job_status.call_args.kwargs
|
||||
assert call_kwargs['job_id'] == "test123"
|
||||
assert call_kwargs['status'] == JobStatus.DOWNLOADING
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# start_job_processing Tests
|
||||
# =============================================================================
|
||||
|
||||
class TestStartJobProcessing:
|
||||
"""Tests for start_job_processing method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_if_job_not_found(self, job_manager, mock_firestore_service):
|
||||
"""Test raises ValueError if job doesn't exist."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
with pytest.raises(ValueError, match="Job missing-job not found"):
|
||||
await job_manager.start_job_processing("missing-job")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_if_missing_input_media_path(self, job_manager, mock_firestore_service):
|
||||
"""Test raises ValueError if job has no input_media_gcs_path."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path=None # Missing
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="missing input_media_gcs_path"):
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_if_invalid_state_transition(self, job_manager, mock_firestore_service):
|
||||
"""Test raises InvalidStateTransitionError if job can't transition to DOWNLOADING."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.AWAITING_REVIEW, # Can't go back to DOWNLOADING
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/audio.mp3"
|
||||
)
|
||||
|
||||
with pytest.raises(InvalidStateTransitionError):
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transitions_and_triggers_workers(self, job_manager, mock_firestore_service):
|
||||
"""Test successful flow: transitions to DOWNLOADING and triggers workers."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/audio.mp3"
|
||||
)
|
||||
|
||||
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
|
||||
mock_worker = Mock()
|
||||
mock_worker.trigger_audio_worker = AsyncMock()
|
||||
mock_worker.trigger_lyrics_worker = AsyncMock()
|
||||
mock_get_worker.return_value = mock_worker
|
||||
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
# Verify transition was called (uses update_job_status internally)
|
||||
mock_firestore_service.update_job_status.assert_called()
|
||||
call_kwargs = mock_firestore_service.update_job_status.call_args.kwargs
|
||||
assert call_kwargs['status'] == JobStatus.DOWNLOADING
|
||||
|
||||
# Verify both workers were triggered
|
||||
mock_worker.trigger_audio_worker.assert_called_once_with("test123")
|
||||
mock_worker.trigger_lyrics_worker.assert_called_once_with("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workers_triggered_in_parallel(self, job_manager, mock_firestore_service):
|
||||
"""Test that audio and lyrics workers are triggered concurrently."""
|
||||
mock_firestore_service.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="gs://bucket/audio.mp3"
|
||||
)
|
||||
|
||||
call_order = []
|
||||
|
||||
async def track_audio(*args):
|
||||
call_order.append(('audio_start', datetime.now()))
|
||||
# Simulate some work
|
||||
call_order.append(('audio_end', datetime.now()))
|
||||
|
||||
async def track_lyrics(*args):
|
||||
call_order.append(('lyrics_start', datetime.now()))
|
||||
# Simulate some work
|
||||
call_order.append(('lyrics_end', datetime.now()))
|
||||
|
||||
with patch('backend.services.worker_service.get_worker_service') as mock_get_worker:
|
||||
mock_worker = Mock()
|
||||
mock_worker.trigger_audio_worker = AsyncMock(side_effect=track_audio)
|
||||
mock_worker.trigger_lyrics_worker = AsyncMock(side_effect=track_lyrics)
|
||||
mock_get_worker.return_value = mock_worker
|
||||
|
||||
await job_manager.start_job_processing("test123")
|
||||
|
||||
# Both workers should have been called
|
||||
assert any('audio_start' in str(c) for c in call_order)
|
||||
assert any('lyrics_start' in str(c) for c in call_order)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -1332,10 +1332,13 @@ class TestHandleMadeForYouOrder:
|
||||
mock_theme_service
|
||||
):
|
||||
"""
|
||||
Orders with YouTube URL should download audio then trigger workers (no search).
|
||||
Orders with YouTube URL should download audio then start processing (no search).
|
||||
|
||||
Note: The download step was added in Jan 2026 to fix job 811ec4e5 failure.
|
||||
See docs/LESSONS-LEARNED.md "YouTube URL Downloads Must Happen Before Workers".
|
||||
|
||||
The handler now uses job_manager.start_job_processing() which handles
|
||||
transitioning to DOWNLOADING and triggering workers.
|
||||
"""
|
||||
from backend.api.routes.users import _handle_made_for_you_order
|
||||
|
||||
@@ -1348,18 +1351,16 @@ class TestHandleMadeForYouOrder:
|
||||
"youtube_url": "https://youtube.com/watch?v=abc123",
|
||||
}
|
||||
|
||||
mock_worker_service = MagicMock()
|
||||
mock_worker_service.trigger_audio_worker = AsyncMock()
|
||||
mock_worker_service.trigger_lyrics_worker = AsyncMock()
|
||||
|
||||
# Mock YouTube download service to return a GCS path
|
||||
mock_youtube_service = MagicMock()
|
||||
mock_youtube_service.download = AsyncMock(
|
||||
return_value="uploads/test-job-123/audio/test.webm"
|
||||
)
|
||||
|
||||
# Mock start_job_processing as an async method
|
||||
mock_job_manager.start_job_processing = AsyncMock()
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.worker_service.get_worker_service', return_value=mock_worker_service), \
|
||||
patch('backend.services.audio_search_service.get_audio_search_service') as mock_get_audio, \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
patch('backend.api.routes.users.get_theme_service', return_value=mock_theme_service), \
|
||||
@@ -1380,9 +1381,8 @@ class TestHandleMadeForYouOrder:
|
||||
# Audio should be downloaded first
|
||||
mock_youtube_service.download.assert_called_once()
|
||||
|
||||
# Workers should be triggered after download
|
||||
mock_worker_service.trigger_audio_worker.assert_called_once()
|
||||
mock_worker_service.trigger_lyrics_worker.assert_called_once()
|
||||
# start_job_processing should be called (handles transition + workers)
|
||||
mock_job_manager.start_job_processing.assert_called_once_with("test-job-123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stripe_session_marked_processed(
|
||||
@@ -1588,6 +1588,69 @@ class TestMadeForYouYouTubeDownloadContract:
|
||||
f"input_media_gcs_path must be set BEFORE workers are triggered. " \
|
||||
f"Call order was: {call_order}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_youtube_url_calls_start_job_processing_after_download(
|
||||
self, mock_job_manager, mock_email_service, mock_user_service,
|
||||
mock_theme_service, youtube_order_metadata
|
||||
):
|
||||
"""
|
||||
CRITICAL: start_job_processing must be called AFTER download completes.
|
||||
|
||||
The handler uses job_manager.start_job_processing() which:
|
||||
1. Transitions job to DOWNLOADING
|
||||
2. Triggers audio and lyrics workers
|
||||
|
||||
Without start_job_processing, workers would run but all status updates
|
||||
would fail silently because transitions from PENDING to GENERATING_SCREENS
|
||||
or AWAITING_REVIEW are invalid.
|
||||
|
||||
This bug caused made-for-you orders to get stuck at PENDING status even though
|
||||
all processing completed successfully. See job IDs 06cfea29 and 984da08b.
|
||||
"""
|
||||
from backend.api.routes.users import _handle_made_for_you_order
|
||||
|
||||
# Track call order to verify sequence
|
||||
call_order = []
|
||||
|
||||
# Mock YouTubeDownloadService to return a GCS path
|
||||
mock_youtube_service = MagicMock()
|
||||
mock_youtube_service.download = AsyncMock(
|
||||
return_value="uploads/test-job-123/audio/test.webm",
|
||||
side_effect=lambda **kwargs: (
|
||||
call_order.append('youtube_download'),
|
||||
"uploads/test-job-123/audio/test.webm"
|
||||
)[1]
|
||||
)
|
||||
|
||||
# Track when start_job_processing is called
|
||||
mock_job_manager.start_job_processing = AsyncMock(
|
||||
side_effect=lambda job_id: call_order.append('start_job_processing')
|
||||
)
|
||||
|
||||
with patch('backend.services.job_manager.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.storage_service.StorageService'), \
|
||||
patch('backend.api.routes.users.get_theme_service', return_value=mock_theme_service), \
|
||||
patch('backend.api.routes.users.get_youtube_download_service', return_value=mock_youtube_service):
|
||||
|
||||
await _handle_made_for_you_order(
|
||||
session_id="sess_123",
|
||||
metadata=youtube_order_metadata,
|
||||
user_service=mock_user_service,
|
||||
email_service=mock_email_service,
|
||||
)
|
||||
|
||||
# CRITICAL: start_job_processing must be called
|
||||
assert 'start_job_processing' in call_order, \
|
||||
f"start_job_processing must be called. Call order was: {call_order}"
|
||||
|
||||
# CRITICAL: start_job_processing must happen AFTER download
|
||||
download_index = call_order.index('youtube_download')
|
||||
processing_index = call_order.index('start_job_processing')
|
||||
|
||||
assert download_index < processing_index, \
|
||||
f"YouTube download must complete BEFORE start_job_processing. " \
|
||||
f"Call order was: {call_order}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_youtube_download_called_with_correct_params(
|
||||
self, mock_job_manager, mock_email_service, mock_user_service,
|
||||
|
||||
@@ -24,6 +24,7 @@ from pathlib import Path
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.config import get_settings
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.workers.registry import worker_registry
|
||||
@@ -288,7 +289,15 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
|
||||
# Validate job status is appropriate for audio worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("audio_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"[job:{job_id}] {status_error}")
|
||||
job_log.warning(status_error)
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
# Create temporary working directory
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_{job_id}_")
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
|
||||
@@ -32,6 +32,7 @@ from backend.models.job import JobStatus
|
||||
from karaoke_gen.utils import sanitize_filename
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.services.lyrics_cache_service import LyricsCacheService
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.workers.style_helper import load_style_config
|
||||
@@ -190,7 +191,15 @@ async def process_lyrics_transcription(job_id: str) -> bool:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
|
||||
# Validate job status is appropriate for lyrics worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("lyrics_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"[job:{job_id}] {status_error}")
|
||||
job_log.warning(status_error)
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
add_span_attribute("artist", job.artist)
|
||||
add_span_attribute("title", job.title)
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from dataclasses import dataclass
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.config import get_settings
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
@@ -98,7 +99,15 @@ async def process_render_video(job_id: str) -> bool:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
|
||||
# Validate job status is appropriate for render video worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("render_video_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"[job:{job_id}] {status_error}")
|
||||
job_log.warning(status_error)
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
job_log.info(f"Starting video render for {job.artist} - {job.title}")
|
||||
logger.info(f"[job:{job_id}] Starting video render (post-review)")
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ from pathlib import Path
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.config import get_settings
|
||||
from backend.workers.style_helper import load_style_config, StyleConfig
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
@@ -238,6 +239,14 @@ def _validate_prerequisites(job) -> bool:
|
||||
Returns:
|
||||
True if prerequisites met, False otherwise
|
||||
"""
|
||||
# Validate job status is appropriate for screens worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("screens_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"Job {job.job_id}: {status_error}")
|
||||
# Log warning but don't fail - status validation is a safety net,
|
||||
# the state_data flags below are the authoritative check
|
||||
|
||||
# SAFETY NET: Enforce theme requirement at processing time
|
||||
# This catches any jobs that somehow bypassed JobManager.create_job() validation
|
||||
if not job.theme_id:
|
||||
@@ -262,7 +271,7 @@ def _validate_prerequisites(job) -> bool:
|
||||
if not job.artist or not job.title:
|
||||
logger.error(f"Job {job.job_id}: Missing artist or title")
|
||||
return False
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from pathlib import Path
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_health_service import validate_worker_can_run
|
||||
from backend.services.rclone_service import get_rclone_service
|
||||
from backend.services.youtube_service import get_youtube_service
|
||||
from backend.services.encoding_service import get_encoding_service
|
||||
@@ -894,6 +895,13 @@ async def _handle_native_distribution(
|
||||
|
||||
def _validate_prerequisites(job) -> bool:
|
||||
"""Validate that all prerequisites are met for video generation."""
|
||||
# Validate job status is appropriate for video worker
|
||||
# This helps catch bugs where the worker is triggered incorrectly
|
||||
status_error = validate_worker_can_run("video_worker", job)
|
||||
if status_error:
|
||||
logger.warning(f"Job {job.job_id}: {status_error}")
|
||||
# Continue anyway - this is a safety net warning, not a hard failure
|
||||
|
||||
# Check instrumental selection
|
||||
instrumental_selection = job.state_data.get('instrumental_selection')
|
||||
if not instrumental_selection:
|
||||
|
||||
@@ -102,6 +102,8 @@
|
||||
|
||||
**Note**: `awaiting_instrumental_selection` exists for backwards compatibility with historical jobs but is no longer used. Instrumental selection is now part of the combined review (`awaiting_review` → `in_review` → `review_complete`).
|
||||
|
||||
**State Machine Robustness (Feb 2026)**: State transitions are enforced by `STATE_TRANSITIONS` in `models/job.py`. Invalid transitions raise `InvalidStateTransitionError` by default. Use `scripts/generate_state_diagram.py` to generate a Mermaid diagram of valid transitions. The `/health/job-consistency` endpoint detects jobs stuck in invalid states. See `docs/archive/2026-02-02-state-machine-robustness-plan.md` for implementation details.
|
||||
|
||||
## GCS File Structure
|
||||
|
||||
```text
|
||||
|
||||
@@ -45,6 +45,19 @@ When UI removes options, backend must apply sensible defaults. Don't rely on eac
|
||||
### Centralize Job Creation Logic
|
||||
When multiple code paths create jobs (file upload, audio search, webhooks), use a shared service for default resolution. The made-for-you webhook handler diverged from regular job creation, missing CDG/TXT defaults because it didn't call the shared `resolve_cdg_txt_defaults()` function. Fix: Create `job_defaults_service.py` with centralized helpers used by ALL job creation paths.
|
||||
|
||||
### Make State Transitions Loud, Not Silent (Feb 2026)
|
||||
**What happened**: Jobs 06cfea29 and 984da08b got stuck at `pending` forever. The Made-For-You webhook handler triggered workers without first transitioning from PENDING to DOWNLOADING. The state machine returned `False` silently, workers ran, but job status never advanced.
|
||||
|
||||
**Root cause**: `transition_to_state()` returned `False` on invalid transitions, but callers didn't check the return value. Silent failures let bugs hide in production.
|
||||
|
||||
**Fix implemented**:
|
||||
1. **Make failures loud**: `transition_to_state()` now raises `InvalidStateTransitionError` by default (use `raise_on_invalid=False` for soft failures)
|
||||
2. **Centralize the pattern**: Created `start_job_processing()` helper that atomically transitions + triggers workers - all handlers use this instead of implementing independently
|
||||
3. **Runtime consistency checks**: Added `job_health_service.py` with `check_job_consistency()` to detect stuck jobs (audio_complete but status=pending)
|
||||
4. **Worker validation**: Workers now log warnings if triggered when job is in unexpected status
|
||||
|
||||
**Pattern**: When a function can fail, prefer exceptions over boolean returns. Silent failures accumulate into production bugs. Use centralized helpers for multi-step operations (state change + side effects) to prevent divergent implementations.
|
||||
|
||||
### Fix Both Sides of Dual Code Paths
|
||||
When fixing a bug in a system with multiple code paths (e.g., legacy vs orchestrator, local vs cloud), verify ALL paths are fixed. PR #271 fixed the GCE worker to READ `instrumental_selection` but only checked the legacy path which was already SENDING it. The orchestrator path (production default) wasn't sending it. **Pattern**: If a component receives config from multiple callers, check ALL callers when fixing the receiving side. Write integration tests that cover each path.
|
||||
|
||||
@@ -214,6 +227,37 @@ worker_index = call_order.index('audio_worker')
|
||||
assert download_index < worker_index, "Download must happen BEFORE workers"
|
||||
```
|
||||
|
||||
### Made-For-You Jobs Must Transition to DOWNLOADING Before Workers (Feb 2026)
|
||||
|
||||
**Problem**: Made-for-you orders with YouTube URLs (jobs 06cfea29, 984da08b) got stuck at `pending` status even though all processing completed successfully. Workers ran, `state_data` showed `audio_complete: true`, `lyrics_complete: true`, `screens_progress.stage: complete`, but the `status` field remained `pending`.
|
||||
|
||||
**Root cause**: The `_handle_made_for_you_order` function in `users.py` triggered workers for YouTube URL orders **without first transitioning the job from PENDING to DOWNLOADING**. The job state machine defines strict valid transitions:
|
||||
- `PENDING` → `DOWNLOADING`, `SEARCHING_AUDIO`, `FAILED`, `CANCELLED` (only these are valid)
|
||||
- Workers try: `PENDING` → `GENERATING_SCREENS` → `AWAITING_REVIEW` (INVALID!)
|
||||
|
||||
All status transitions after workers complete fail silently because they're invalid from `PENDING`.
|
||||
|
||||
**Why it was subtle**:
|
||||
1. Workers completed successfully (no errors in logs)
|
||||
2. `state_data` updates worked (they bypass state machine validation)
|
||||
3. Only the `status` field failed to update (uses `transition_to_state` which validates)
|
||||
|
||||
**Fix**:
|
||||
1. Immediate: Add `transition_to_state(job_id, JobStatus.DOWNLOADING)` before triggering workers
|
||||
2. Robustness: Make `transition_to_state()` raise `InvalidStateTransitionError` by default instead of returning `False` silently
|
||||
3. Pattern: Create centralized `job_manager.start_job_processing(job_id)` helper that handles transition + worker triggers atomically
|
||||
|
||||
**Robustness changes** (see `docs/archive/2026-02-02-state-machine-robustness-plan.md`):
|
||||
- `InvalidStateTransitionError` exception with full context (job_id, from_status, to_status, valid_transitions)
|
||||
- `validate_state_transition()` and `transition_to_state()` now have `raise_on_invalid=True` default
|
||||
- `start_job_processing()` centralizes the pattern: validate prerequisites → transition → trigger workers
|
||||
|
||||
**Pattern**: Always verify state machine flow in webhook handlers. Use centralized helpers like `start_job_processing()` instead of duplicating transition + worker logic.
|
||||
|
||||
**Tests added**:
|
||||
- `test_youtube_url_calls_start_job_processing_after_download` - verifies proper sequence
|
||||
- `TestInvalidStateTransitionError`, `TestValidateStateTransitionRaisesBehavior`, `TestStartJobProcessing` in `test_job_manager.py`
|
||||
|
||||
### Use data-testid for E2E
|
||||
Prefer `data-testid` over label/text selectors. They're immune to label changes and won't break when similar fields are added.
|
||||
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# State Machine Robustness Plan
|
||||
|
||||
**Date**: 2026-02-02
|
||||
**Status**: ✓ Complete (Weeks 1-3 implemented)
|
||||
**Context**: Jobs 06cfea29 and 984da08b got stuck at `pending` because state transitions failed silently.
|
||||
|
||||
## Problem Analysis
|
||||
|
||||
The root issues that allowed this bug:
|
||||
|
||||
1. **Silent failures**: `transition_to_state()` returns `False` on invalid transitions, but callers don't check it
|
||||
2. **No centralized worker trigger pattern**: Each handler (file_upload.py, users.py, audio_search.py) implements "transition + trigger workers" independently
|
||||
3. **No runtime consistency checks**: Nothing detects when `state_data` says complete but `status` is wrong
|
||||
4. **Workers don't validate prerequisites**: Workers run regardless of job status
|
||||
|
||||
## Proposed Improvements
|
||||
|
||||
### 1. Make State Transition Failures Loud (High Priority)
|
||||
|
||||
**Option A**: Raise exception by default (breaking change, but safest)
|
||||
```python
|
||||
class InvalidStateTransitionError(Exception):
|
||||
"""Raised when an invalid state transition is attempted."""
|
||||
pass
|
||||
|
||||
def transition_to_state(self, job_id: str, new_status: JobStatus, ...,
|
||||
raise_on_invalid: bool = True) -> bool:
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
if raise_on_invalid:
|
||||
raise InvalidStateTransitionError(
|
||||
f"Invalid transition for job {job_id}: {current} -> {new_status}"
|
||||
)
|
||||
return False
|
||||
...
|
||||
```
|
||||
|
||||
**Option B**: Always log + metric, return bool (non-breaking)
|
||||
```python
|
||||
def transition_to_state(...) -> bool:
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
# Log with structured data for alerting
|
||||
logger.error(
|
||||
f"STATE_TRANSITION_FAILED job={job_id} from={current} to={new_status}",
|
||||
extra={"job_id": job_id, "from_status": current, "to_status": new_status}
|
||||
)
|
||||
# Emit metric for monitoring
|
||||
emit_metric("state_transition_failed", 1, {"from": current, "to": new_status})
|
||||
return False
|
||||
...
|
||||
```
|
||||
|
||||
**Recommendation**: Option A with `raise_on_invalid=True` default. Forces callers to handle failures explicitly.
|
||||
|
||||
### 2. Centralize "Start Processing" Pattern (High Priority)
|
||||
|
||||
Create a single function that handles the transition + worker trigger pattern:
|
||||
|
||||
```python
|
||||
# backend/services/job_manager.py
|
||||
|
||||
async def start_job_processing(self, job_id: str, worker_service) -> None:
|
||||
"""
|
||||
Transition job to DOWNLOADING and trigger workers.
|
||||
|
||||
This is the single entry point for starting job processing.
|
||||
All handlers (file_upload, webhook, audio_search) should use this.
|
||||
|
||||
Raises:
|
||||
InvalidStateTransitionError: If job can't transition to DOWNLOADING
|
||||
ValueError: If job not found or missing required fields
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
|
||||
if not job.input_media_gcs_path:
|
||||
raise ValueError(f"Job {job_id} missing input_media_gcs_path")
|
||||
|
||||
# This will raise if transition is invalid
|
||||
self.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.DOWNLOADING,
|
||||
progress=15,
|
||||
message="Starting audio and lyrics processing"
|
||||
)
|
||||
|
||||
# Only trigger workers if transition succeeded
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
```
|
||||
|
||||
Then update all handlers to use this:
|
||||
```python
|
||||
# In _handle_made_for_you_order, file_upload, audio_search:
|
||||
await job_manager.start_job_processing(job_id, worker_service)
|
||||
```
|
||||
|
||||
### 3. Add State Consistency Checks (Medium Priority)
|
||||
|
||||
Add a health check that detects inconsistent state:
|
||||
|
||||
```python
|
||||
# backend/services/job_health_service.py
|
||||
|
||||
def check_job_consistency(job: Job) -> List[str]:
|
||||
"""
|
||||
Check for state inconsistencies in a job.
|
||||
Returns list of issues found.
|
||||
"""
|
||||
issues = []
|
||||
state_data = job.state_data or {}
|
||||
|
||||
# Check: audio_complete but status still pending
|
||||
if state_data.get('audio_complete') and job.status == JobStatus.PENDING:
|
||||
issues.append(f"audio_complete=True but status=pending")
|
||||
|
||||
# Check: lyrics_complete but status still pending
|
||||
if state_data.get('lyrics_complete') and job.status == JobStatus.PENDING:
|
||||
issues.append(f"lyrics_complete=True but status=pending")
|
||||
|
||||
# Check: screens_progress.stage=complete but status not awaiting_review
|
||||
screens = state_data.get('screens_progress', {})
|
||||
if screens.get('stage') == 'complete' and job.status not in [
|
||||
JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE,
|
||||
JobStatus.RENDERING_VIDEO, JobStatus.INSTRUMENTAL_SELECTED, ...
|
||||
]:
|
||||
issues.append(f"screens complete but status={job.status}")
|
||||
|
||||
return issues
|
||||
```
|
||||
|
||||
Run this:
|
||||
- In a daily Cloud Scheduler job
|
||||
- After each worker completes
|
||||
- Expose via `/health/job-consistency` endpoint
|
||||
|
||||
### 4. Workers Validate Prerequisites (Medium Priority)
|
||||
|
||||
Workers should verify job is in expected state before processing:
|
||||
|
||||
```python
|
||||
# backend/workers/screens_worker.py
|
||||
|
||||
async def generate_screens(job_id: str) -> bool:
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
# Validate job is in expected state for this worker
|
||||
valid_statuses = [JobStatus.DOWNLOADING, JobStatus.AUDIO_COMPLETE, JobStatus.LYRICS_COMPLETE]
|
||||
if job.status not in [s.value for s in valid_statuses]:
|
||||
logger.error(
|
||||
f"[job:{job_id}] screens_worker called but job status is {job.status}, "
|
||||
f"expected one of {valid_statuses}. This indicates a bug in the trigger logic."
|
||||
)
|
||||
# Option: return False, or try to recover, or alert
|
||||
return False
|
||||
|
||||
# ... rest of worker
|
||||
```
|
||||
|
||||
### 5. Add Integration Tests for State Flow (Medium Priority)
|
||||
|
||||
Add tests that verify the complete state machine flow without mocking transitions:
|
||||
|
||||
```python
|
||||
# backend/tests/integration/test_state_machine_flow.py
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_made_for_you_youtube_flow_state_transitions():
|
||||
"""
|
||||
Verify complete state flow for made-for-you YouTube orders.
|
||||
Uses real JobManager (mocked Firestore) to validate state machine.
|
||||
"""
|
||||
# Create job
|
||||
job = job_manager.create_job(...)
|
||||
assert job.status == JobStatus.PENDING
|
||||
|
||||
# Simulate YouTube download complete
|
||||
job_manager.update_job(job_id, {'input_media_gcs_path': 'test/path'})
|
||||
|
||||
# Start processing (this should transition to DOWNLOADING)
|
||||
await job_manager.start_job_processing(job_id, mock_worker_service)
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
assert job.status == JobStatus.DOWNLOADING, \
|
||||
f"Expected DOWNLOADING after start_job_processing, got {job.status}"
|
||||
|
||||
# ... continue through full flow
|
||||
```
|
||||
|
||||
### 6. State Machine Diagram Generation (Low Priority)
|
||||
|
||||
Auto-generate a Mermaid diagram from `STATE_TRANSITIONS`:
|
||||
|
||||
```python
|
||||
# scripts/generate_state_diagram.py
|
||||
|
||||
def generate_mermaid():
|
||||
lines = ["stateDiagram-v2"]
|
||||
for from_state, to_states in STATE_TRANSITIONS.items():
|
||||
for to_state in to_states:
|
||||
lines.append(f" {from_state} --> {to_state}")
|
||||
return "\n".join(lines)
|
||||
```
|
||||
|
||||
Add to `docs/ARCHITECTURE.md` and regenerate on changes.
|
||||
|
||||
### 7. Fix Admin Reset Buttons (High Priority)
|
||||
|
||||
The admin dashboard has reset buttons that bypass state machine validation and have several issues:
|
||||
|
||||
**Current buttons:**
|
||||
- "Start" → reset to `pending`
|
||||
- "Audio" → reset to `awaiting_audio_selection`
|
||||
- "Lyrics" → reset to `awaiting_review`
|
||||
- "Inst." → reset to `awaiting_instrumental_selection`
|
||||
- "Reprocess" → reset to `instrumental_selected`
|
||||
|
||||
**Issues identified:**
|
||||
|
||||
1. **"Start" button leaves job stuck**: Resets to `pending` but doesn't trigger workers or clear `audio_complete`/`lyrics_complete` flags. Job sits indefinitely.
|
||||
|
||||
2. **"Lyrics" and "Inst." are obsolete**: Combined review flow (Jan 2026) merged these into a single step. Should be replaced with single "Review" button that resets to `awaiting_review`.
|
||||
|
||||
3. **Missing state_data keys in clear list for `pending`**:
|
||||
```python
|
||||
# Should also clear:
|
||||
"audio_complete", "lyrics_complete"
|
||||
```
|
||||
|
||||
4. **State machine bypass**: Admin resets use `update_job()` directly, not `transition_to_state()`. This is intentional for admin overrides but should be documented.
|
||||
|
||||
**Fixes needed:**
|
||||
|
||||
```python
|
||||
# backend/api/routes/admin.py
|
||||
|
||||
# 1. Update STATE_DATA_CLEAR_KEYS for "pending"
|
||||
STATE_DATA_CLEAR_KEYS = {
|
||||
"pending": [
|
||||
# ... existing keys ...
|
||||
"audio_complete", # NEW - clear parallel processing flag
|
||||
"lyrics_complete", # NEW - clear parallel processing flag
|
||||
],
|
||||
# ...
|
||||
}
|
||||
|
||||
# 2. Remove awaiting_instrumental_selection from ALLOWED_RESET_STATES
|
||||
ALLOWED_RESET_STATES = {
|
||||
"pending",
|
||||
"awaiting_audio_selection",
|
||||
"awaiting_review", # Combined review (lyrics + instrumental)
|
||||
# "awaiting_instrumental_selection", # REMOVED - deprecated
|
||||
"instrumental_selected",
|
||||
}
|
||||
|
||||
# 3. After reset to pending, trigger audio search if job has artist/title
|
||||
# or provide clear guidance to admin about next steps
|
||||
```
|
||||
|
||||
```tsx
|
||||
// frontend/app/admin/jobs/page.tsx
|
||||
|
||||
// Replace "Lyrics" and "Inst." buttons with single "Review" button:
|
||||
{[
|
||||
{ state: "pending", icon: RotateCcw, label: "Start" },
|
||||
{ state: "awaiting_audio_selection", icon: Music, label: "Audio" },
|
||||
{ state: "awaiting_review", icon: Mic, label: "Review" }, // Combined
|
||||
// REMOVED: { state: "awaiting_instrumental_selection", ... }
|
||||
{ state: "instrumental_selected", icon: RefreshCw, label: "Reprocess" },
|
||||
]}
|
||||
```
|
||||
|
||||
## Implementation Priority
|
||||
|
||||
1. **Week 1** ✓ DONE:
|
||||
- ✓ Make `transition_to_state` raise by default (`InvalidStateTransitionError`, `raise_on_invalid=True`)
|
||||
- ✓ Create `start_job_processing()` helper in `job_manager.py`
|
||||
- ✓ Update `_handle_made_for_you_order` to use it
|
||||
- ✓ Added 15 unit tests covering exception behavior and new helper
|
||||
|
||||
2. **Week 1.5** ✓ DONE (Admin Button Fixes):
|
||||
- ✓ Add `audio_complete`, `lyrics_complete` to STATE_DATA_CLEAR_KEYS for "pending"
|
||||
- ✓ Remove `awaiting_instrumental_selection` from ALLOWED_RESET_STATES
|
||||
- ✓ Update frontend to replace "Lyrics"/"Inst." with single "Review" button
|
||||
- ✓ Add tests for admin reset endpoint (`test_reset_to_pending_clears_parallel_processing_flags`, `test_rejects_deprecated_awaiting_instrumental_selection`)
|
||||
|
||||
3. **Week 2** ✓ DONE:
|
||||
- ✓ Update `file_upload.py` and `audio_search.py` to use `start_job_processing()`
|
||||
- ✓ Fixed double-transition bug in YouTube URL upload flow
|
||||
- ✓ Removed unused `_trigger_workers_parallel` helper from file_upload.py and audio_search.py
|
||||
- ✓ Add worker prerequisite validation (`validate_worker_can_run()` in all workers)
|
||||
- ✓ Add state consistency health check (`job_health_service.py` with `check_job_consistency()`)
|
||||
- ✓ Added 26 tests for job_health_service
|
||||
|
||||
4. **Week 3** ✓ DONE:
|
||||
- ✓ Add integration tests for state flow (`tests/integration/test_state_machine_flow.py` - 11 tests)
|
||||
- ✓ Generate state machine diagram script (`scripts/generate_state_diagram.py`)
|
||||
- ✓ Expose `/health/job-consistency` endpoint for admin dashboard
|
||||
- TODO: Add monitoring/alerting on `STATE_TRANSITION_FAILED` logs (requires Cloud Logging alert setup)
|
||||
|
||||
## Quick Wins (Can Do Now)
|
||||
|
||||
1. Add explicit error logging when transition fails (already exists but add job_id to structured logging)
|
||||
2. Create Cloud Logging alert for "Invalid state transition" errors
|
||||
3. Add consistency check to admin dashboard "stuck jobs" view
|
||||
@@ -496,7 +496,7 @@ function AdminJobsPageContent() {
|
||||
const info: Record<string, { label: string; description: string; icon: any }> = {
|
||||
pending: {
|
||||
label: "Pending",
|
||||
description: "Restart from the beginning. Clears all processing data.",
|
||||
description: "Restart from the beginning. Clears all processing data including audio/lyrics progress.",
|
||||
icon: RotateCcw,
|
||||
},
|
||||
awaiting_audio_selection: {
|
||||
@@ -505,15 +505,10 @@ function AdminJobsPageContent() {
|
||||
icon: Music,
|
||||
},
|
||||
awaiting_review: {
|
||||
label: "Lyrics Review",
|
||||
description: "Re-review lyrics. Preserves audio stems.",
|
||||
label: "Combined Review",
|
||||
description: "Re-review lyrics AND re-select instrumental. Preserves audio stems.",
|
||||
icon: Mic,
|
||||
},
|
||||
awaiting_instrumental_selection: {
|
||||
label: "Instrumental",
|
||||
description: "Re-select instrumental. Preserves lyrics review.",
|
||||
icon: Sliders,
|
||||
},
|
||||
instrumental_selected: {
|
||||
label: "Reprocess Video",
|
||||
description: "Re-encode and re-distribute with current settings. Triggers video worker automatically.",
|
||||
@@ -816,8 +811,7 @@ function AdminJobsPageContent() {
|
||||
{[
|
||||
{ state: "pending", icon: RotateCcw, label: "Start" },
|
||||
{ state: "awaiting_audio_selection", icon: Music, label: "Audio" },
|
||||
{ state: "awaiting_review", icon: Mic, label: "Lyrics" },
|
||||
{ state: "awaiting_instrumental_selection", icon: Sliders, label: "Inst." },
|
||||
{ state: "awaiting_review", icon: Mic, label: "Review" }, // Combined lyrics + instrumental review
|
||||
{ state: "instrumental_selected", icon: RefreshCw, label: "Reprocess" },
|
||||
].map(({ state, icon: Icon, label }) => (
|
||||
<Tooltip key={state}>
|
||||
@@ -1638,8 +1632,7 @@ function AdminJobsPageContent() {
|
||||
<span className="font-medium">What happens next:</span>{" "}
|
||||
{resetResult.new_status === "pending" && "Job will need to go through audio search, download, and all processing steps."}
|
||||
{resetResult.new_status === "awaiting_audio_selection" && "User can select from the cached audio search results."}
|
||||
{resetResult.new_status === "awaiting_review" && "User can review and correct the lyrics."}
|
||||
{resetResult.new_status === "awaiting_instrumental_selection" && "User can select the instrumental track."}
|
||||
{resetResult.new_status === "awaiting_review" && "User can review lyrics and select instrumental track."}
|
||||
{resetResult.new_status === "instrumental_selected" && (
|
||||
resetResult.worker_triggered
|
||||
? "Video generation is now in progress."
|
||||
|
||||
@@ -16,8 +16,8 @@ import { createEmailHelper, isEmailTestingAvailable } from '../helpers/email-tes
|
||||
* 4. Create Job - Search for test song via UI
|
||||
* 5. Audio Selection - Select audio source via UI dialog
|
||||
* 6. Wait for Processing - Monitor via UI status updates
|
||||
* 7. Lyrics Review - Open review UI, interact, submit via UI
|
||||
* 8. Instrumental Selection - Open selection UI, interact, submit via UI
|
||||
* 7. Combined Review - Open review UI, preview video, proceed to instrumental
|
||||
* 8. Instrumental Selection - Select instrumental and submit (same page via hash nav)
|
||||
* 9. Wait for Completion - Monitor via UI
|
||||
* 10. Verify Downloads - Check download links work
|
||||
* 11. Verify Distribution - Check YouTube/Dropbox/GDrive indicators
|
||||
@@ -540,11 +540,15 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
console.log('STEP 6 COMPLETE: Lyrics processing finished');
|
||||
|
||||
// =========================================================================
|
||||
// STEP 7: Lyrics Review (via UI - REAL USER FLOW)
|
||||
// The minimal path: scroll to bottom -> click "Preview Video" -> wait for video -> click "Complete Review"
|
||||
// STEP 7: Combined Lyrics Review & Instrumental Selection (via UI - REAL USER FLOW)
|
||||
// After the UI redesign, lyrics review and instrumental selection are combined:
|
||||
// 1. User opens lyrics review page
|
||||
// 2. User clicks "Preview Video" to open modal
|
||||
// 3. User clicks "Proceed to Instrumental Review" which navigates to instrumental selection
|
||||
// 4. User selects instrumental and clicks "Confirm & Continue"
|
||||
// =========================================================================
|
||||
console.log('\n========================================');
|
||||
console.log('STEP 7: Lyrics Review (UI Interaction)');
|
||||
console.log('STEP 7: Combined Lyrics Review & Instrumental Selection');
|
||||
console.log('========================================');
|
||||
|
||||
// Click the Review Lyrics button/link
|
||||
@@ -621,7 +625,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
const errorText = await errorAlert.textContent();
|
||||
console.log(` WARNING: Preview error: ${errorText}`);
|
||||
await reviewPage.screenshot({ path: 'test-results/07e-preview-error.png', fullPage: true });
|
||||
// Continue anyway - we can still click Complete Review even if preview failed
|
||||
// Continue anyway - we can still proceed to instrumental even if preview failed
|
||||
} else if (await videoElement.isVisible({ timeout: 10000 }).catch(() => false)) {
|
||||
console.log(' Video element visible in modal');
|
||||
// Give the video a moment to buffer/load
|
||||
@@ -632,31 +636,93 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/07f-video-state.png', fullPage: true });
|
||||
|
||||
// Click "Complete Review" button in the modal
|
||||
// This button has variant="contained", color="success", and endIcon={<CloudUpload />}
|
||||
const completeReviewBtn = reviewPage.getByRole('button', { name: /complete review/i });
|
||||
await expect(completeReviewBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found "Complete Review" button');
|
||||
// Click "Proceed to Instrumental Review" button in the modal
|
||||
// This button saves corrections and navigates to the instrumental selection UI
|
||||
const proceedBtn = reviewPage.getByRole('button', { name: /proceed to instrumental/i });
|
||||
await expect(proceedBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found "Proceed to Instrumental Review" button');
|
||||
|
||||
await completeReviewBtn.click();
|
||||
console.log(' Clicked "Complete Review" button');
|
||||
await proceedBtn.click();
|
||||
console.log(' Clicked "Proceed to Instrumental Review" button');
|
||||
|
||||
// Wait for the submission to process
|
||||
// The handleSubmitToServer function calls apiClient.submitCorrections() then window.close()
|
||||
// Wait for the navigation to instrumental selection (happens via hash change)
|
||||
// The page navigates to #/{jobId}/instrumental
|
||||
await reviewPage.waitForTimeout(3000);
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/07g-after-proceed.png', fullPage: true });
|
||||
|
||||
// =========================================================================
|
||||
// STEP 8: Instrumental Selection (continuation of combined flow)
|
||||
// The page has navigated to the instrumental selection UI via hash routing
|
||||
// =========================================================================
|
||||
console.log('\n========================================');
|
||||
console.log('STEP 8: Instrumental Selection');
|
||||
console.log('========================================');
|
||||
|
||||
// Wait for the instrumental selection UI to load
|
||||
// The InstrumentalSelector component renders selection options
|
||||
console.log(' Waiting for instrumental selection UI to load...');
|
||||
|
||||
// Wait for either the selection options or the loading indicator to appear
|
||||
try {
|
||||
await reviewPage.waitForSelector('.selection-option, .selection-panel, [class*="selection"]', { timeout: 30000 });
|
||||
console.log(' Instrumental selection UI loaded');
|
||||
} catch {
|
||||
// If selector not found, check for loading state
|
||||
const loadingState = reviewPage.getByText(/loading instrumental/i);
|
||||
if (await loadingState.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
console.log(' Waiting for instrumental analysis to complete...');
|
||||
await expect(loadingState).not.toBeVisible({ timeout: 60000 });
|
||||
}
|
||||
}
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/08a-instrumental-opened.png', fullPage: true });
|
||||
console.log(' Instrumental selection UI opened');
|
||||
|
||||
// Select "Clean" instrumental option
|
||||
// The options have class "selection-option" and contain labels with "Clean" or "With Backing"
|
||||
const cleanOption = reviewPage.locator('.selection-option:has-text("Clean")').first();
|
||||
if (await cleanOption.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await cleanOption.click();
|
||||
console.log(' Selected "Clean" instrumental option');
|
||||
} else {
|
||||
// If we can't find "Clean" specifically, the first option might already be selected
|
||||
console.log(' Clean option not found - using default selection');
|
||||
}
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/08b-clean-selected.png', fullPage: true });
|
||||
|
||||
// Click the submit button: "✓ Confirm & Continue" (id="submit-btn")
|
||||
const submitBtn = reviewPage.locator('#submit-btn');
|
||||
await expect(submitBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found submit button');
|
||||
|
||||
await submitBtn.click();
|
||||
console.log(' Clicked "Confirm & Continue" button');
|
||||
|
||||
// Wait for submission to complete - the page shows a success screen then redirects
|
||||
// In cloud mode, after success it redirects to /app after a countdown
|
||||
await reviewPage.waitForTimeout(5000);
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/07g-after-complete-review.png', fullPage: true });
|
||||
await reviewPage.screenshot({ path: 'test-results/08c-instrumental-submitted.png', fullPage: true });
|
||||
|
||||
// The review page may close automatically after submission (window.close())
|
||||
// If it's still open, close it manually
|
||||
// The page redirects to /app after completion, or may close
|
||||
// Wait a bit for the redirect/close to happen
|
||||
if (!reviewPage.isClosed()) {
|
||||
await reviewPage.close();
|
||||
console.log(' Closed review UI');
|
||||
// Wait for potential redirect or success screen
|
||||
try {
|
||||
await reviewPage.waitForURL(/\/app/, { timeout: 10000 });
|
||||
console.log(' Redirected to /app after instrumental selection');
|
||||
} catch {
|
||||
// If no redirect, close manually
|
||||
await reviewPage.close();
|
||||
console.log(' Closed review UI');
|
||||
}
|
||||
} else {
|
||||
console.log(' Review UI closed automatically after submission');
|
||||
}
|
||||
|
||||
// Refresh main app to see updated status
|
||||
// Refresh main app
|
||||
await page.bringToFront();
|
||||
const refreshBtn = page.getByRole('button', { name: /refresh/i });
|
||||
if (await refreshBtn.isVisible().catch(() => false)) {
|
||||
@@ -664,121 +730,10 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
await page.waitForTimeout(3000);
|
||||
}
|
||||
|
||||
await page.screenshot({ path: 'test-results/07h-after-review-refresh.png' });
|
||||
console.log('STEP 7 COMPLETE: Lyrics review submitted via UI');
|
||||
|
||||
// =========================================================================
|
||||
// STEP 8: Wait for Video Rendering & Instrumental Selection (via UI - REAL USER FLOW)
|
||||
// The instrumental UI has: selection-option elements, submit button with id="submit-btn"
|
||||
// =========================================================================
|
||||
console.log('\n========================================');
|
||||
console.log('STEP 8: Wait for Video Rendering & Instrumental Selection');
|
||||
console.log('========================================');
|
||||
|
||||
// Wait for instrumental selection to become available
|
||||
console.log(' Waiting for video rendering and instrumental selection...');
|
||||
|
||||
let foundInstrumentalStatus = false;
|
||||
const renderStartTime = Date.now();
|
||||
|
||||
while (Date.now() - renderStartTime < TIMEOUTS.videoRendering) {
|
||||
// Refresh
|
||||
if (await refreshBtn.isVisible().catch(() => false)) {
|
||||
await refreshBtn.click();
|
||||
await page.waitForTimeout(2000);
|
||||
}
|
||||
|
||||
// Check job card status
|
||||
const statusText = await jobCard.textContent() || '';
|
||||
console.log(` Status: ${statusText.substring(0, 80)}...`);
|
||||
|
||||
// Look for instrumental selection UI availability
|
||||
// Users ALWAYS have to go through the instrumental review UI - there's no skip
|
||||
if (statusText.toLowerCase().includes('select instrumental') ||
|
||||
statusText.toLowerCase().includes('awaiting_instrumental') ||
|
||||
(statusText.toLowerCase().includes('action needed') && statusText.toLowerCase().includes('instrumental'))) {
|
||||
foundInstrumentalStatus = true;
|
||||
console.log(' Job ready for instrumental selection');
|
||||
break;
|
||||
}
|
||||
|
||||
if (statusText.toLowerCase().includes('failed')) {
|
||||
throw new Error('Job failed during video rendering');
|
||||
}
|
||||
|
||||
await page.waitForTimeout(10000);
|
||||
}
|
||||
|
||||
if (!foundInstrumentalStatus) {
|
||||
throw new Error('Timeout waiting for instrumental selection - users must always complete this step');
|
||||
}
|
||||
|
||||
{
|
||||
// Click the Select Instrumental button/link
|
||||
const instrumentalLink = jobCard.getByRole('link', { name: /select instrumental/i });
|
||||
await expect(instrumentalLink).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
|
||||
const instrumentalUrl = await instrumentalLink.getAttribute('href');
|
||||
console.log(` Instrumental URL: ${instrumentalUrl}`);
|
||||
|
||||
// Open instrumental UI in new page
|
||||
const instrumentalPage = await context.newPage();
|
||||
// Handle both absolute and relative URLs
|
||||
const fullUrl = instrumentalUrl?.startsWith('http') ? instrumentalUrl : `${PROD_URL}${instrumentalUrl}`;
|
||||
// Use 'load' instead of 'networkidle' - instrumental page has ongoing audio requests
|
||||
// that prevent networkidle from ever being reached
|
||||
await instrumentalPage.goto(fullUrl, { waitUntil: 'load' });
|
||||
await instrumentalPage.waitForTimeout(3000);
|
||||
|
||||
await instrumentalPage.screenshot({ path: 'test-results/08a-instrumental-opened.png', fullPage: true });
|
||||
console.log(' Instrumental selection UI opened');
|
||||
|
||||
// Wait for the page to fully load - the instrumental UI renders selection options
|
||||
await instrumentalPage.waitForSelector('.selection-option, .selection-panel', { timeout: 30000 });
|
||||
|
||||
// Select "Clean" instrumental option
|
||||
// The options have class "selection-option" and contain labels with "Clean" or "With Backing"
|
||||
const cleanOption = instrumentalPage.locator('.selection-option:has-text("Clean")').first();
|
||||
if (await cleanOption.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await cleanOption.click();
|
||||
console.log(' Selected "Clean" instrumental option');
|
||||
} else {
|
||||
// If we can't find "Clean" specifically, the first option might already be selected
|
||||
console.log(' Clean option not found - using default selection');
|
||||
}
|
||||
|
||||
await instrumentalPage.screenshot({ path: 'test-results/08b-clean-selected.png', fullPage: true });
|
||||
|
||||
// Click the submit button: "✓ Confirm & Continue" (id="submit-btn")
|
||||
const submitBtn = instrumentalPage.locator('#submit-btn');
|
||||
await expect(submitBtn).toBeVisible({ timeout: TIMEOUTS.action });
|
||||
console.log(' Found submit button');
|
||||
|
||||
await submitBtn.click();
|
||||
console.log(' Clicked "Confirm & Continue" button');
|
||||
|
||||
// Wait for submission to complete - the page shows a success screen then auto-closes
|
||||
await instrumentalPage.waitForTimeout(5000);
|
||||
|
||||
await instrumentalPage.screenshot({ path: 'test-results/08c-instrumental-submitted.png', fullPage: true });
|
||||
|
||||
// The page may auto-close after submission
|
||||
if (!instrumentalPage.isClosed()) {
|
||||
await instrumentalPage.close();
|
||||
console.log(' Closed instrumental UI');
|
||||
} else {
|
||||
console.log(' Instrumental UI closed automatically after submission');
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh main app
|
||||
await page.bringToFront();
|
||||
if (await refreshBtn.isVisible().catch(() => false)) {
|
||||
await refreshBtn.click();
|
||||
await page.waitForTimeout(3000);
|
||||
}
|
||||
|
||||
await page.screenshot({ path: 'test-results/08d-after-instrumental.png' });
|
||||
await reviewPage.screenshot({ path: 'test-results/08d-after-instrumental.png', fullPage: true }).catch(() => {
|
||||
// reviewPage may be closed already, that's OK
|
||||
});
|
||||
await page.screenshot({ path: 'test-results/08d-main-after-instrumental.png' });
|
||||
console.log('STEP 8 COMPLETE: Instrumental selection handled via UI');
|
||||
|
||||
// =========================================================================
|
||||
|
||||
@@ -190,43 +190,51 @@ test.describe('Lyrics Review Only - Focused Test', () => {
|
||||
|
||||
await page.screenshot({ path: 'test-results/focused-06-video-state.png', fullPage: true });
|
||||
|
||||
// Click "Complete Review" button in the modal
|
||||
console.log('6. Looking for "Complete Review" button...');
|
||||
// Click "Proceed to Instrumental Review" button in the modal
|
||||
// (Previously "Complete Review" - changed with combined review UI)
|
||||
console.log('6. Looking for "Proceed to Instrumental Review" button...');
|
||||
|
||||
// List all buttons in the modal
|
||||
const modalButtons = await previewModal.getByRole('button').allTextContents();
|
||||
console.log(` Modal buttons: ${modalButtons.join(', ')}`);
|
||||
|
||||
const completeReviewBtn = page.getByRole('button', { name: /complete review/i });
|
||||
const proceedBtn = page.getByRole('button', { name: /proceed to instrumental/i });
|
||||
|
||||
if (await completeReviewBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
console.log(' Found "Complete Review" button - clicking...');
|
||||
await completeReviewBtn.click();
|
||||
console.log(' Clicked "Complete Review" button');
|
||||
if (await proceedBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
console.log(' Found "Proceed to Instrumental Review" button - clicking...');
|
||||
await proceedBtn.click();
|
||||
console.log(' Clicked "Proceed to Instrumental Review" button');
|
||||
} else {
|
||||
// Try to find any submit-like button
|
||||
const submitBtn = previewModal.getByRole('button', { name: /submit|confirm|done|save/i });
|
||||
// Try to find any submit-like button (for backwards compatibility)
|
||||
const submitBtn = previewModal.getByRole('button', { name: /submit|confirm|done|save|complete/i });
|
||||
if (await submitBtn.isVisible().catch(() => false)) {
|
||||
const btnText = await submitBtn.textContent();
|
||||
console.log(` Found alternative submit button: "${btnText}" - clicking...`);
|
||||
await submitBtn.click();
|
||||
} else {
|
||||
console.log(' Could not find Complete Review or similar button');
|
||||
await page.screenshot({ path: 'test-results/focused-07-no-complete-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Complete Review button');
|
||||
console.log(' Could not find Proceed to Instrumental or similar button');
|
||||
await page.screenshot({ path: 'test-results/focused-07-no-proceed-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Proceed to Instrumental Review button');
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for the submission to process
|
||||
console.log('7. Waiting for submission to complete...');
|
||||
// Wait for navigation to instrumental selection
|
||||
console.log('7. Waiting for navigation to instrumental selection...');
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
await page.screenshot({ path: 'test-results/focused-08-after-submit.png', fullPage: true });
|
||||
await page.screenshot({ path: 'test-results/focused-08-after-proceed.png', fullPage: true });
|
||||
|
||||
// Note: This focused test only covers the lyrics review portion.
|
||||
// The instrumental selection happens next on the same page via hash navigation.
|
||||
// For full flow testing, use happy-path-real-user.spec.ts
|
||||
|
||||
console.log('');
|
||||
console.log('========================================');
|
||||
console.log('FOCUSED TEST COMPLETE');
|
||||
console.log('========================================');
|
||||
console.log('NOTE: This test completes the lyrics review step only.');
|
||||
console.log(' The instrumental selection UI should now be visible on this page.');
|
||||
console.log(' For the full flow, use happy-path-real-user.spec.ts');
|
||||
});
|
||||
|
||||
test('Add Reference Lyrics via API', async ({ page, context, request }) => {
|
||||
|
||||
@@ -256,27 +256,28 @@ async function testLyricsReview(page: any, context: any, jobId: string) {
|
||||
console.log(' WARNING: No video element found');
|
||||
}
|
||||
|
||||
// Click "Complete Review" button
|
||||
console.log('10. Clicking "Complete Review" button...');
|
||||
// Click "Proceed to Instrumental Review" button (previously "Complete Review")
|
||||
console.log('10. Clicking "Proceed to Instrumental Review" button...');
|
||||
const modalButtons = await previewModal.getByRole('button').allTextContents();
|
||||
console.log(` Modal buttons: ${modalButtons.join(', ')}`);
|
||||
|
||||
const completeReviewBtn = reviewPage.getByRole('button', { name: /complete review/i });
|
||||
const proceedBtn = reviewPage.getByRole('button', { name: /proceed to instrumental/i });
|
||||
|
||||
if (await completeReviewBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await completeReviewBtn.click();
|
||||
console.log(' Clicked Complete Review button');
|
||||
if (await proceedBtn.isVisible({ timeout: 5000 }).catch(() => false)) {
|
||||
await proceedBtn.click();
|
||||
console.log(' Clicked Proceed to Instrumental Review button');
|
||||
} else {
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-09-no-complete-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Complete Review button');
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-09-no-proceed-btn.png', fullPage: true });
|
||||
throw new Error('Could not find Proceed to Instrumental Review button');
|
||||
}
|
||||
|
||||
// Wait for submission
|
||||
console.log('11. Waiting for submission...');
|
||||
// Wait for navigation to instrumental selection (happens via hash change on same page)
|
||||
console.log('11. Waiting for navigation to instrumental selection...');
|
||||
await reviewPage.waitForTimeout(5000);
|
||||
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-10-after-submit.png', fullPage: true });
|
||||
console.log(' Lyrics review completed successfully!');
|
||||
await reviewPage.screenshot({ path: 'test-results/existing-10-after-proceed.png', fullPage: true });
|
||||
console.log(' Lyrics review step completed - now on instrumental selection!');
|
||||
console.log(' NOTE: With combined UI, instrumental selection is on the same page.');
|
||||
|
||||
await reviewPage.close();
|
||||
}
|
||||
|
||||
@@ -73,7 +73,11 @@ def create_database() -> dict:
|
||||
firestore.IndexFieldArgs(field_path="made_for_you", order="ASCENDING"),
|
||||
firestore.IndexFieldArgs(field_path="created_at", order="DESCENDING"),
|
||||
],
|
||||
opts=pulumi.ResourceOptions(depends_on=[firestore_db]),
|
||||
opts=pulumi.ResourceOptions(
|
||||
depends_on=[firestore_db],
|
||||
# Import existing index created before Pulumi managed it
|
||||
import_="projects/nomadkaraoke/databases/(default)/collectionGroups/jobs/indexes/CICAgOi36pgK",
|
||||
),
|
||||
)
|
||||
|
||||
# Sessions: query by user_email + is_active, order by created_at
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.114.15"
|
||||
version = "0.114.16"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user