mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - **Local stack:** `scripts/local-stack.sh up --frontend` runs the **real** backend on the Firestore + GCS emulators with `FAKE_WORKERS=1`. In-process fake workers move jobs through their real states in seconds, using tiny real media, so web Playwright and mobile Maestro full-journey tests can run offline against the same endpoints. - **Email-code sign-in** (nomad-mobile spec §4.1): `POST /api/users/auth/send-code` + `verify-code`. The code logic is shared with kjbox via `LoginCodeStore`, and sends are throttled per email and per IP. - **Bug fix found by the new suite:** `confirm-duration` moved jobs to `separating_stage1`, from which they can never reach review. It now resumes at `downloading`. No prod job had taken this path yet. ## Changes - `backend/services/fake_workers/`: dispatch seam in `WorkerService.trigger_*`, plus fake search, download, checkout and outbox, scenarios, and per-user config. The real stem upload, transcode, backing-vocals analysis and preview render are reused. Every check is strict (`is True`), so a MagicMock settings object never enables the fakes, and startup **refuses FAKE_WORKERS in production**. - `/api/test/*` router (mounted only in that mode): `status`, `scenario`, `fake-config`, `emails`, `reset`, plus the fake Stripe checkout page. Paying it runs the real webhook handler. - `frontend/e2e/local-stack/` (9 specs, ~50s): - sign-up → make → review → instrumental → download - buy and cancel credits - duration confirm, failed, complete and review states - code sign-in and lockout - New required CI job `local-stack-e2e`. - Docs: TESTING.md § Local stack, DEVELOPMENT.md, API.md, README, LESSONS-LEARNED. - `MAX_SIGNUPS_PER_IP` can now be set by env. Only the local stack raises it; production keeps the default of 2. ## Testing - [x] Backend unit: 6175 passed - [x] Emulator: 152 passed - [x] Frontend Jest: 1706 passed - [x] Local-stack Playwright: 9/9 passed (CI mode) ## Review - [x] CodeRabbit CLI review completed locally. 1 minor finding declined: it suggested swallowing infrastructure errors in send-code as fake "sent" responses, which would hide real outages from users. @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01AGuxtgoscF6RJGtpjHPg3W --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
747 lines
37 KiB
Python
747 lines
37 KiB
Python
"""
|
|
Job data models for karaoke generation.
|
|
|
|
This module defines the complete state machine for karaoke generation jobs,
|
|
mirroring the 8-stage CLI workflow with human-in-the-loop interaction points.
|
|
"""
|
|
from typing import Optional, Dict, Any, List
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from pydantic import BaseModel, Field, validator
|
|
|
|
from karaoke_gen.utils import normalize_text
|
|
|
|
|
|
class JobStatus(str, Enum):
|
|
"""
|
|
Job status enumeration - Complete state machine.
|
|
|
|
The workflow has 8 main stages with up to 3 human interaction points:
|
|
1. Input & Setup (may include audio source selection)
|
|
1b. Optional Audio Editing (BLOCKING) - trim/cut/mute/join input audio
|
|
2. Parallel Processing (audio + lyrics)
|
|
3. Title/End Screen Generation + Backing Vocals Analysis
|
|
4. Countdown Padding Synchronization
|
|
5. Combined Human Review (BLOCKING) - lyrics review + instrumental selection
|
|
6. Video Finalization
|
|
7. Distribution
|
|
|
|
Note: As of 2026-01, lyrics review and instrumental selection are combined
|
|
into a single human review step. The user selects their instrumental
|
|
during the lyrics review phase, before the video is rendered.
|
|
"""
|
|
# Initial states
|
|
PENDING = "pending" # Job created, queued for processing
|
|
|
|
# Audio search states (for artist+title search mode)
|
|
SEARCHING_AUDIO = "searching_audio" # Searching for audio sources via flacfetch
|
|
AWAITING_AUDIO_SELECTION = "awaiting_audio_selection" # ⚠️ WAITING FOR USER - select audio source
|
|
DOWNLOADING_AUDIO = "downloading_audio" # Downloading selected audio from source
|
|
DOWNLOAD_PENDING_RETRY = "download_pending_retry" # Download hit a transient issue (few/no seeders, tracker blip); auto-retrying for up to 24h
|
|
|
|
DOWNLOADING = "downloading" # Downloading from URL or processing upload
|
|
|
|
# Optional: Audio editing (before processing begins)
|
|
AWAITING_AUDIO_EDIT = "awaiting_audio_edit" # ⚠️ WAITING FOR USER - edit input audio
|
|
IN_AUDIO_EDIT = "in_audio_edit" # User is actively editing audio
|
|
AUDIO_EDIT_COMPLETE = "audio_edit_complete" # User submitted edited audio
|
|
|
|
# Optional: duration cost confirmation (BLOCKING) - confirm credits before heavy processing
|
|
AWAITING_DURATION_CONFIRM = "awaiting_duration_confirm" # ⚠️ WAITING FOR USER - confirm duration-based cost
|
|
|
|
# Stage 2a: Audio separation (parallel track 1)
|
|
SEPARATING_STAGE1 = "separating_stage1" # Clean instrumental separation (Modal API)
|
|
SEPARATING_STAGE2 = "separating_stage2" # Backing vocals separation (Modal API)
|
|
AUDIO_COMPLETE = "audio_complete" # All audio stems ready
|
|
|
|
# Stage 2b: Lyrics processing (parallel track 2)
|
|
TRANSCRIBING = "transcribing" # AudioShake API transcription
|
|
CORRECTING = "correcting" # Automatic lyrics correction
|
|
LYRICS_COMPLETE = "lyrics_complete" # Corrections JSON ready
|
|
|
|
# Stage 3: Title/End screens + Backing Vocals Analysis
|
|
GENERATING_SCREENS = "generating_screens" # Creating title and end screen videos
|
|
|
|
# Stage 4: Countdown padding (automatic)
|
|
APPLYING_PADDING = "applying_padding" # Synchronizing countdown padding
|
|
|
|
# Stage 5: Combined Human Review (BLOCKING) - lyrics + instrumental selection
|
|
AWAITING_REVIEW = "awaiting_review" # ⚠️ WAITING FOR USER - combined review needed
|
|
IN_REVIEW = "in_review" # User is actively reviewing
|
|
REVIEW_COMPLETE = "review_complete" # User submitted lyrics + instrumental selection
|
|
|
|
# Stage 5.5: Render video with corrected lyrics (post-review)
|
|
RENDERING_VIDEO = "rendering_video" # Using OutputGenerator to create with_vocals.mkv
|
|
RENDER_PENDING_CAPACITY = "render_pending_capacity" # Waiting for GCE encoding capacity (auto-retry)
|
|
|
|
# Stage 6: Instrumental already selected (during combined review)
|
|
# Note: AWAITING_INSTRUMENTAL_SELECTION kept for DB compatibility with historical jobs only
|
|
AWAITING_INSTRUMENTAL_SELECTION = "awaiting_instrumental_selection" # LEGACY - no longer used
|
|
INSTRUMENTAL_SELECTED = "instrumental_selected" # Instrumental was selected during combined review
|
|
|
|
# Stage 7: Video generation and finalization
|
|
GENERATING_VIDEO = "generating_video" # Creating initial karaoke video
|
|
ENCODING = "encoding" # Multi-format video encoding (Cloud Build)
|
|
PACKAGING = "packaging" # CDG/TXT ZIP generation
|
|
|
|
# Stage 8: Distribution (optional)
|
|
UPLOADING = "uploading" # YouTube/Dropbox upload
|
|
NOTIFYING = "notifying" # Discord/Email notifications
|
|
|
|
# Terminal states
|
|
COMPLETE = "complete" # All processing finished successfully
|
|
PREP_COMPLETE = "prep_complete" # Prep-only job completed (stops after review)
|
|
FAILED = "failed" # Unrecoverable error occurred
|
|
CANCELLED = "cancelled" # User cancelled the job
|
|
|
|
# Legacy compatibility (will be removed)
|
|
QUEUED = "queued" # Deprecated: use PENDING
|
|
PROCESSING = "processing" # Deprecated: use specific states
|
|
READY_FOR_FINALIZATION = "ready_for_finalization" # Deprecated
|
|
FINALIZING = "finalizing" # Deprecated: use ENCODING/PACKAGING
|
|
ERROR = "error" # Deprecated: use FAILED
|
|
|
|
|
|
# Valid state transitions
|
|
STATE_TRANSITIONS = {
|
|
# PENDING can go to DOWNLOADING (file upload), SEARCHING_AUDIO (artist+title search),
|
|
# or DOWNLOADING_AUDIO (create-from-search where audio is pre-selected)
|
|
JobStatus.PENDING: [JobStatus.DOWNLOADING, JobStatus.SEARCHING_AUDIO, JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
|
|
# Audio search flow (for artist+title search mode)
|
|
JobStatus.SEARCHING_AUDIO: [JobStatus.AWAITING_AUDIO_SELECTION, JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED],
|
|
JobStatus.AWAITING_AUDIO_SELECTION: [JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
JobStatus.DOWNLOADING_AUDIO: [JobStatus.DOWNLOADING, JobStatus.AWAITING_DURATION_CONFIRM, JobStatus.DOWNLOAD_PENDING_RETRY, JobStatus.FAILED],
|
|
# DOWNLOAD_PENDING_RETRY = parked when a torrent download hits a transient
|
|
# issue (few/no seeders, tracker outage); auto-retried by the recover-stuck
|
|
# cron for up to 24h before failing permanently.
|
|
JobStatus.DOWNLOAD_PENDING_RETRY: [JobStatus.DOWNLOADING_AUDIO, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
|
|
# DOWNLOADING allows parallel processing (audio + lyrics) and then screens when both complete
|
|
# Can also enter optional audio edit phase if requires_audio_edit flag is set
|
|
JobStatus.DOWNLOADING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING, JobStatus.GENERATING_SCREENS, JobStatus.AWAITING_AUDIO_EDIT, JobStatus.AWAITING_DURATION_CONFIRM, JobStatus.FAILED],
|
|
|
|
# Optional audio edit phase (trim, cut, mute, join input audio before processing)
|
|
JobStatus.AWAITING_AUDIO_EDIT: [JobStatus.IN_AUDIO_EDIT, JobStatus.AUDIO_EDIT_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
JobStatus.IN_AUDIO_EDIT: [JobStatus.AWAITING_AUDIO_EDIT, JobStatus.AUDIO_EDIT_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
JobStatus.AUDIO_EDIT_COMPLETE: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING, JobStatus.GENERATING_SCREENS, JobStatus.AWAITING_DURATION_CONFIRM, JobStatus.FAILED],
|
|
|
|
# Duration cost confirmation (BLOCKING) - user confirms before heavy processing begins
|
|
JobStatus.AWAITING_DURATION_CONFIRM: [
|
|
# DOWNLOADING = confirmed, back to the prep phase (see jobs.confirm_duration).
|
|
JobStatus.DOWNLOADING,
|
|
JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING,
|
|
JobStatus.GENERATING_SCREENS, JobStatus.AWAITING_AUDIO_EDIT,
|
|
JobStatus.FAILED, JobStatus.CANCELLED,
|
|
],
|
|
|
|
# Audio separation flow
|
|
JobStatus.SEPARATING_STAGE1: [JobStatus.SEPARATING_STAGE2, JobStatus.FAILED],
|
|
JobStatus.SEPARATING_STAGE2: [JobStatus.AUDIO_COMPLETE, JobStatus.FAILED],
|
|
JobStatus.AUDIO_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
|
|
|
# Lyrics flow
|
|
JobStatus.TRANSCRIBING: [JobStatus.CORRECTING, JobStatus.FAILED],
|
|
JobStatus.CORRECTING: [JobStatus.LYRICS_COMPLETE, JobStatus.FAILED],
|
|
JobStatus.LYRICS_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
|
|
|
# Post-parallel processing (screens + backing vocals analysis)
|
|
# GENERATING_SCREENS -> REVIEW_COMPLETE is the auto-approval path: when the
|
|
# scorer is fully confident the review screens are skipped entirely (no
|
|
# review token / "ready for review" notification is ever issued).
|
|
JobStatus.GENERATING_SCREENS: [JobStatus.APPLYING_PADDING, JobStatus.AWAITING_REVIEW, JobStatus.REVIEW_COMPLETE, JobStatus.FAILED],
|
|
JobStatus.APPLYING_PADDING: [JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
|
|
|
# Combined human review flow (lyrics + instrumental selection)
|
|
# AWAITING_REVIEW can go directly to REVIEW_COMPLETE (quick review) or to IN_REVIEW (editing)
|
|
JobStatus.AWAITING_REVIEW: [JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
JobStatus.IN_REVIEW: [JobStatus.REVIEW_COMPLETE, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
|
JobStatus.REVIEW_COMPLETE: [
|
|
JobStatus.RENDERING_VIDEO,
|
|
JobStatus.PREP_COMPLETE, # for prep-only jobs
|
|
JobStatus.RENDER_PENDING_CAPACITY, # GCE capacity exhausted before render started
|
|
JobStatus.FAILED,
|
|
],
|
|
|
|
# Video rendering (post-review) - instrumental was already selected during combined review
|
|
# RENDER_PENDING_CAPACITY = parked when GCE encoding capacity is exhausted; auto-retried by scheduler.
|
|
JobStatus.RENDERING_VIDEO: [
|
|
JobStatus.INSTRUMENTAL_SELECTED,
|
|
JobStatus.PREP_COMPLETE,
|
|
JobStatus.RENDER_PENDING_CAPACITY,
|
|
JobStatus.FAILED,
|
|
],
|
|
JobStatus.RENDER_PENDING_CAPACITY: [
|
|
JobStatus.RENDERING_VIDEO, # auto-retry succeeded, picked up again
|
|
JobStatus.REVIEW_COMPLETE, # admin / scheduler resets to re-trigger render
|
|
JobStatus.FAILED, # exceeded max wait or operator gave up
|
|
JobStatus.CANCELLED, # user cancelled while waiting
|
|
],
|
|
|
|
# AWAITING_INSTRUMENTAL_SELECTION is LEGACY - kept for historical jobs in DB
|
|
# New jobs never enter this state; they go directly to INSTRUMENTAL_SELECTED after render
|
|
JobStatus.AWAITING_INSTRUMENTAL_SELECTION: [JobStatus.INSTRUMENTAL_SELECTED, JobStatus.FAILED, JobStatus.CANCELLED],
|
|
JobStatus.INSTRUMENTAL_SELECTED: [JobStatus.GENERATING_VIDEO, JobStatus.FAILED],
|
|
|
|
# Video generation flow
|
|
JobStatus.GENERATING_VIDEO: [JobStatus.ENCODING, JobStatus.FAILED],
|
|
JobStatus.ENCODING: [JobStatus.PACKAGING, JobStatus.COMPLETE, JobStatus.FAILED],
|
|
JobStatus.PACKAGING: [JobStatus.UPLOADING, JobStatus.COMPLETE, JobStatus.FAILED],
|
|
|
|
# Distribution flow
|
|
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
|
|
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
|
|
|
|
# Terminal states - COMPLETE allows edit (reopen review) and visibility change
|
|
# FAILED and CANCELLED allow retry transitions to resume from checkpoints
|
|
# PREP_COMPLETE allows continuation from combined review
|
|
JobStatus.COMPLETE: [JobStatus.AWAITING_REVIEW, JobStatus.LYRICS_COMPLETE], # Edit track or visibility change
|
|
JobStatus.PREP_COMPLETE: [JobStatus.AWAITING_REVIEW, JobStatus.FAILED], # Continue from combined review
|
|
JobStatus.FAILED: [
|
|
JobStatus.DOWNLOADING, # Retry from beginning (if input audio exists)
|
|
JobStatus.DOWNLOADING_AUDIO, # Retry audio download (audio search flow)
|
|
JobStatus.INSTRUMENTAL_SELECTED, # Retry from video generation
|
|
JobStatus.REVIEW_COMPLETE, # Retry from render stage
|
|
JobStatus.LYRICS_COMPLETE, # Retry from screens generation
|
|
JobStatus.AWAITING_REVIEW, # Retry from combined review
|
|
JobStatus.RENDER_PENDING_CAPACITY, # Operator parks a stale-failed job for auto-retry
|
|
JobStatus.AWAITING_AUDIO_SELECTION, # Choose different audio after a failed download
|
|
],
|
|
JobStatus.CANCELLED: [
|
|
JobStatus.DOWNLOADING, # Retry from beginning (if input audio exists)
|
|
JobStatus.DOWNLOADING_AUDIO, # Retry audio download (audio search flow)
|
|
JobStatus.INSTRUMENTAL_SELECTED, # Retry from video generation
|
|
JobStatus.REVIEW_COMPLETE, # Retry from render stage
|
|
JobStatus.LYRICS_COMPLETE, # Retry from screens generation
|
|
JobStatus.AWAITING_REVIEW, # Retry from combined review
|
|
],
|
|
|
|
# Legacy states (for backward compatibility)
|
|
JobStatus.QUEUED: [JobStatus.PENDING],
|
|
JobStatus.PROCESSING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING],
|
|
JobStatus.READY_FOR_FINALIZATION: [JobStatus.GENERATING_VIDEO],
|
|
JobStatus.FINALIZING: [JobStatus.ENCODING],
|
|
JobStatus.ERROR: [JobStatus.FAILED],
|
|
}
|
|
|
|
|
|
class TimelineEvent(BaseModel):
|
|
"""Timeline event for job progress tracking."""
|
|
status: str
|
|
timestamp: str
|
|
progress: Optional[int] = None
|
|
message: Optional[str] = None
|
|
metadata: Optional[Dict[str, Any]] = None
|
|
|
|
|
|
class LogEntry(BaseModel):
|
|
"""Worker log entry for debugging and monitoring."""
|
|
timestamp: str
|
|
level: str # DEBUG, INFO, WARNING, ERROR
|
|
worker: str # audio, lyrics, screens, video, render
|
|
message: str
|
|
|
|
|
|
class Job(BaseModel):
|
|
"""
|
|
Complete job data model.
|
|
|
|
Tracks the full lifecycle of a karaoke generation job from submission
|
|
through all 8 stages to completion or failure.
|
|
"""
|
|
job_id: str
|
|
status: JobStatus
|
|
progress: int = 0 # 0-100 percentage for UI display
|
|
created_at: datetime
|
|
updated_at: datetime
|
|
|
|
# Input data
|
|
url: Optional[str] = None # YouTube URL (if provided)
|
|
artist: Optional[str] = None
|
|
title: Optional[str] = None
|
|
filename: Optional[str] = None # Original uploaded filename
|
|
input_media_gcs_path: Optional[str] = None # GCS path to uploaded file
|
|
|
|
# Audio source tracking (for retry capability)
|
|
audio_source_type: Optional[str] = None # Source type: 'audio_search', 'youtube_url', 'file_upload'
|
|
source_name: Optional[str] = None # Provider name (RED, OPS, YouTube, Spotify) - for audio_search
|
|
source_id: Optional[str] = None # Source-specific ID (torrent ID, video ID, track ID) - for audio_search
|
|
target_file: Optional[str] = None # For torrents, specific file to extract - for audio_search
|
|
download_url: Optional[str] = None # Direct download URL if available - for audio_search
|
|
|
|
# User preferences
|
|
enable_cdg: bool = False # Generate CDG+MP3 package (requires style config)
|
|
enable_txt: bool = False # Generate TXT+MP3 package (requires style config)
|
|
enable_youtube_upload: bool = False # Upload to YouTube
|
|
youtube_description: Optional[str] = None # YouTube video description
|
|
webhook_url: Optional[str] = None # Webhook for notifications
|
|
user_email: Optional[str] = None # Email for notifications
|
|
non_interactive: bool = False # Skip interactive steps (lyrics review, instrumental selection)
|
|
|
|
# Review autonomy level. "auto" (default): skip the combined review screens
|
|
# whenever the auto-approval scorer is fully confident, falling back to human
|
|
# review otherwise. "always_review": every job stops at the review screens.
|
|
# (Distinct from legacy non_interactive, which is CLI packaging/upload
|
|
# semantics and never gated the cloud review flow.)
|
|
review_mode: str = "auto"
|
|
|
|
# Backing-vocals preference chosen up-front at submission. "auto" (default):
|
|
# retain backing vocals wherever the 3-stem decider is confident it's safe,
|
|
# else fall through to human review. "clean": always use the clean
|
|
# instrumental (never retain backing). "review": always let a human pick the
|
|
# instrumental. Consumed by the auto-approval instrumental decision.
|
|
backing_preference: str = "auto"
|
|
|
|
# Translated lyrics: language code (one of the UI locales, e.g. "en", "es") whose
|
|
# LLM translation of each reviewed line is drawn in smaller text beneath it in the
|
|
# karaoke videos. None = no translation row. See backend/services/lyrics_translation.py.
|
|
translation_language: Optional[str] = None
|
|
|
|
# Multi-tenant support ("" = consumer portal, "vocalstar"/etc = tenant portal)
|
|
tenant_id: str = "" # Tenant ID for white-label portal scoping
|
|
|
|
# UI language the job was submitted in (primary Accept-Language subtag, e.g.
|
|
# "pt", "ja", "zh"). Any of the 33 supported UI locales — NOT narrowed to the
|
|
# 3 email locales. Admin-only signal for support/intervention. None for jobs
|
|
# created before this field existed or where the header was absent.
|
|
locale: Optional[str] = None
|
|
|
|
# Theme configuration (pre-made themes from GCS)
|
|
theme_id: Optional[str] = None # Theme identifier (e.g., "nomad", "default")
|
|
# When the theme was last re-snapshotted onto the job after creation (theme
|
|
# edit refresh, re-render, Edit). None = the creation-time snapshot (created_at).
|
|
theme_applied_at: Optional[datetime] = None
|
|
color_overrides: Dict[str, str] = Field(default_factory=dict)
|
|
"""
|
|
User color overrides applied on top of theme. Keys:
|
|
- artist_color: Hex color for artist name (#RRGGBB)
|
|
- title_color: Hex color for song title
|
|
- sung_lyrics_color: Hex color for highlighted lyrics
|
|
- unsung_lyrics_color: Hex color for unhighlighted lyrics
|
|
"""
|
|
|
|
# Style configuration (uploaded files - used when theme_id is not set)
|
|
style_params_gcs_path: Optional[str] = None # GCS path to style_params.json
|
|
style_assets: Dict[str, str] = Field(default_factory=dict)
|
|
"""
|
|
GCS paths to style asset files:
|
|
{
|
|
"intro_background": "gs://bucket/jobs/{job_id}/style/intro_bg.png",
|
|
"karaoke_background": "gs://bucket/jobs/{job_id}/style/karaoke_bg.png",
|
|
"end_background": "gs://bucket/jobs/{job_id}/style/end_bg.png",
|
|
"font": "gs://bucket/jobs/{job_id}/style/font.ttf",
|
|
"cdg_instrumental_background": "gs://bucket/jobs/{job_id}/style/cdg_instr.png",
|
|
"cdg_title_background": "gs://bucket/jobs/{job_id}/style/cdg_title.png",
|
|
"cdg_outro_background": "gs://bucket/jobs/{job_id}/style/cdg_outro.png"
|
|
}
|
|
"""
|
|
|
|
# Finalisation configuration
|
|
brand_prefix: Optional[str] = None # Brand code prefix (e.g., "NOMAD")
|
|
discord_webhook_url: Optional[str] = None # Discord notification webhook
|
|
youtube_description_template: Optional[str] = None # YouTube description template text
|
|
|
|
# Distribution configuration (native API - for remote CLI)
|
|
dropbox_path: Optional[str] = None # Dropbox folder path for organized output (e.g., "/Karaoke/Tracks-Organized")
|
|
gdrive_folder_id: Optional[str] = None # Google Drive folder ID for public share uploads
|
|
|
|
# Legacy distribution configuration (rclone - for local CLI backward compat)
|
|
organised_dir_rclone_root: Optional[str] = None # Deprecated: use dropbox_path instead
|
|
|
|
# Lyrics configuration (overrides for search/transcription)
|
|
lyrics_artist: Optional[str] = None # Override artist name for lyrics search
|
|
lyrics_title: Optional[str] = None # Override title for lyrics search
|
|
lyrics_file_gcs_path: Optional[str] = None # GCS path to user-provided lyrics file
|
|
subtitle_offset_ms: int = 0 # Offset for subtitle timing (positive = delay)
|
|
|
|
# Audio separation model configuration
|
|
clean_instrumental_model: Optional[str] = None # Model for clean instrumental separation
|
|
backing_vocals_models: Optional[List[str]] = None # Models for backing vocals separation
|
|
other_stems_models: Optional[List[str]] = None # Models for other stems (bass, drums, etc.)
|
|
|
|
# Existing instrumental configuration (Batch 3)
|
|
existing_instrumental_gcs_path: Optional[str] = None # GCS path to user-provided instrumental file
|
|
|
|
# Audio search configuration (Batch 5 - artist+title search mode)
|
|
audio_search_artist: Optional[str] = None # Artist name used for audio search
|
|
audio_search_title: Optional[str] = None # Title used for audio search
|
|
auto_download: bool = False # Auto-select best audio source (skip selection)
|
|
|
|
# Two-phase workflow configuration (Batch 6)
|
|
prep_only: bool = False # Stop after review, don't run finalisation
|
|
finalise_only: bool = False # Skip prep, run only finalisation (requires uploaded prep outputs)
|
|
keep_brand_code: Optional[str] = None # Preserve existing brand code instead of generating new one
|
|
|
|
# Review authentication (Batch 7)
|
|
review_token: Optional[str] = None # Job-scoped token for lyrics review UI access (generated when entering AWAITING_REVIEW)
|
|
review_token_expires_at: Optional[datetime] = None # Token expiry time (optional, for extra security)
|
|
instrumental_token: Optional[str] = None # Job-scoped token for instrumental review UI access (generated when entering AWAITING_INSTRUMENTAL_SELECTION)
|
|
instrumental_token_expires_at: Optional[datetime] = None # Token expiry time
|
|
|
|
# Made-for-you order tracking
|
|
made_for_you: bool = False # Flag indicating this is a made-for-you customer order
|
|
customer_email: Optional[str] = None # Customer email for final delivery (job owned by admin during processing)
|
|
customer_notes: Optional[str] = None # Notes provided by customer with their order
|
|
|
|
# Private (non-published) track mode
|
|
is_private: bool = False # Private tracks: Dropbox only (Tracks-NonPublished/NOMADNP), no YouTube/GDrive
|
|
|
|
# Creation-time decision params (for observability — what did the user actually choose?)
|
|
creation_params: Dict[str, Any] = Field(default_factory=dict)
|
|
|
|
# Anti-abuse: IP address at job creation time (for cross-account correlation)
|
|
creation_ip: Optional[str] = None
|
|
|
|
# Output deletion tracking (for admin cleanup without deleting job)
|
|
outputs_deleted_at: Optional[datetime] = None # Timestamp when outputs were deleted by admin
|
|
outputs_deleted_by: Optional[str] = None # Admin email who deleted outputs
|
|
|
|
# Storage retention (backend/services/storage_retention.py): set when the
|
|
# daily purge deleted this job's regenerable renders / separated stems.
|
|
# Cleared when they're rebuilt (regenerate / re-render / stems restore).
|
|
renders_purged_at: Optional[datetime] = None
|
|
stems_purged_at: Optional[datetime] = None
|
|
renders_regenerated_at: Optional[datetime] = None # last GCS-only regenerate
|
|
storage_purge: Optional[Dict[str, Any]] = None # manifest of the last purge
|
|
|
|
# Credit refund tracking (prevents double-refund on cancel→delete or fail→delete)
|
|
credit_refunded: bool = False
|
|
|
|
# Edit tracking (how many times a completed track has been re-edited)
|
|
edit_count: int = 0
|
|
|
|
# Whole-track tempo change applied in the audio editor (1.0 / None = original
|
|
# speed). When set, job.title carries a "(90% Tempo)" label so every published
|
|
# output says so; see backend/services/tempo_label.py.
|
|
tempo_factor: Optional[float] = None
|
|
|
|
# Processing state
|
|
track_output_dir: Optional[str] = None # Local output directory (temp)
|
|
audio_hash: Optional[str] = None # Hash for deduplication
|
|
|
|
# State-specific data (JSON field for stage-specific metadata)
|
|
state_data: Dict[str, Any] = Field(default_factory=dict)
|
|
"""
|
|
Stage-specific metadata. Examples:
|
|
- audio_complete: {"stems": {"clean": "gs://...", "backing": "gs://..."}}
|
|
- lyrics_complete: {"corrections_url": "gs://...", "audio_url": "gs://..."}
|
|
- review_complete: {"corrected_lyrics": {...}}
|
|
- instrumental_selected: {"selection": "clean" | "with_backing"}
|
|
- encoding: {"build_id": "...", "progress": 45}
|
|
"""
|
|
|
|
# Timeline tracking
|
|
timeline: List[TimelineEvent] = Field(default_factory=list)
|
|
|
|
# Worker logs for debugging (limited to last N entries to avoid document size issues)
|
|
worker_logs: List[LogEntry] = Field(default_factory=list)
|
|
|
|
# File URLs (GCS storage)
|
|
file_urls: Dict[str, Any] = Field(default_factory=dict)
|
|
"""
|
|
File storage URLs. Structure:
|
|
{
|
|
"input": "gs://bucket/jobs/{job_id}/input.flac",
|
|
"stems": {
|
|
"instrumental_clean": "gs://...",
|
|
"instrumental_with_backing": "gs://...",
|
|
"vocals": "gs://...",
|
|
"backing_vocals": "gs://...",
|
|
"lead_vocals": "gs://...",
|
|
"bass": "gs://...",
|
|
"drums": "gs://...",
|
|
"guitar": "gs://...",
|
|
"piano": "gs://...",
|
|
"other": "gs://..."
|
|
},
|
|
"lyrics": {
|
|
"corrections": "gs://bucket/jobs/{job_id}/lyrics/corrections.json",
|
|
"audio": "gs://bucket/jobs/{job_id}/lyrics/audio.flac",
|
|
"lrc": "gs://...",
|
|
"ass": "gs://..."
|
|
},
|
|
"screens": {
|
|
"title": "gs://bucket/jobs/{job_id}/screens/title.mov",
|
|
"end": "gs://bucket/jobs/{job_id}/screens/end.mov"
|
|
},
|
|
"videos": {
|
|
"with_vocals": "gs://bucket/jobs/{job_id}/videos/with_vocals.mkv"
|
|
},
|
|
"finals": {
|
|
"lossless_4k_mp4": "gs://...",
|
|
"lossless_4k_mkv": "gs://...",
|
|
"lossy_4k_mp4": "gs://...",
|
|
"lossy_720p_mp4": "gs://..."
|
|
},
|
|
"packages": {
|
|
"cdg_zip": "gs://...",
|
|
"txt_zip": "gs://..."
|
|
},
|
|
"youtube": {
|
|
"url": "https://youtube.com/watch?v=...",
|
|
"video_id": "..."
|
|
}
|
|
}
|
|
"""
|
|
|
|
# Results (for backward compatibility, will be deprecated)
|
|
output_files: Dict[str, str] = Field(default_factory=dict)
|
|
download_urls: Dict[str, str] = Field(default_factory=dict)
|
|
|
|
# Error handling
|
|
error_message: Optional[str] = None
|
|
error_details: Optional[Dict[str, Any]] = None # Structured error information
|
|
retry_count: int = 0 # Number of retry attempts
|
|
|
|
# Worker tracking
|
|
worker_ids: Dict[str, str] = Field(default_factory=dict)
|
|
"""
|
|
IDs of background workers/jobs:
|
|
{
|
|
"audio_worker": "cloud-run-request-id",
|
|
"lyrics_worker": "cloud-run-request-id",
|
|
"video_encoder": "cloud-build-id",
|
|
"distribution": "cloud-run-request-id"
|
|
}
|
|
"""
|
|
|
|
# Processing metadata (immutable provenance data written by workers during processing)
|
|
processing_metadata: Dict[str, Any] = Field(default_factory=dict)
|
|
"""
|
|
Immutable processing provenance data. Written once by each worker during processing.
|
|
Never modified after initial write. Used for analytics, debugging, and data export.
|
|
|
|
Structure:
|
|
{
|
|
"transcription": {
|
|
"provider": "audioshake",
|
|
"audioshake_task_id": "...",
|
|
"audioshake_asset_id": "...",
|
|
"language_detected": "en",
|
|
"word_count": 358,
|
|
"segment_count": 42,
|
|
},
|
|
"correction": {
|
|
"handlers_applied": [...],
|
|
"corrections_made": 12,
|
|
"correction_ratio": 0.034,
|
|
"reference_sources_found": ["genius", "spotify"],
|
|
"agentic_routing": "rule-based",
|
|
"duration_seconds": 33.2,
|
|
},
|
|
"separation": {
|
|
"provider": "modal",
|
|
"clean_model": "...",
|
|
"duration_seconds": 420.5,
|
|
},
|
|
"timing": {
|
|
"lyrics_worker_seconds": 125.3,
|
|
"audio_worker_seconds": 420.5,
|
|
...
|
|
},
|
|
}
|
|
"""
|
|
|
|
# Request metadata (captured at job creation for tracking and filtering)
|
|
request_metadata: Dict[str, Any] = Field(default_factory=dict)
|
|
"""
|
|
Metadata captured from the original API request.
|
|
Used for tracking, filtering, and operational management.
|
|
|
|
Standard fields:
|
|
{
|
|
"client_ip": "192.168.1.1", # IP address of the client
|
|
"user_agent": "karaoke-gen-remote/0.71.0", # User-Agent header
|
|
"environment": "test", # From X-Environment header (test/production/development)
|
|
"client_id": "cli-user-123", # From X-Client-ID header (customer/user identifier)
|
|
"server_version": "0.71.0", # Server version at job creation
|
|
"created_from": "upload", # "upload" (file) or "url" (YouTube URL)
|
|
"custom_headers": { # All X-* headers for extensibility
|
|
"X-Environment": "test",
|
|
"X-Client-ID": "cli-user-123",
|
|
"X-Request-ID": "abc-123"
|
|
}
|
|
}
|
|
|
|
Use cases:
|
|
- Filter test vs production jobs
|
|
- Track jobs by customer/client
|
|
- Debug issues with specific clients
|
|
- Bulk cleanup of test jobs
|
|
"""
|
|
|
|
# Note: Status transition validation is handled by JobManager.validate_state_transition()
|
|
# which is called before status updates. The Job model does not validate transitions
|
|
# because Firestore updates happen directly without reconstructing the model.
|
|
|
|
class Config:
|
|
use_enum_values = True
|
|
|
|
|
|
class JobCreate(BaseModel):
|
|
"""
|
|
Job creation request.
|
|
|
|
Either `url` OR file upload is required (file upload handled separately).
|
|
Artist and title are optional - will be auto-detected from YouTube if not provided.
|
|
"""
|
|
url: Optional[str] = None
|
|
artist: Optional[str] = None
|
|
title: Optional[str] = None
|
|
filename: Optional[str] = None # Original uploaded filename
|
|
|
|
# Optional preferences
|
|
enable_cdg: bool = False # Requires style config
|
|
enable_txt: bool = False # Requires style config
|
|
enable_youtube_upload: bool = False
|
|
youtube_description: Optional[str] = None
|
|
webhook_url: Optional[str] = None
|
|
user_email: Optional[str] = None
|
|
non_interactive: bool = False # Skip interactive steps (lyrics review, instrumental selection)
|
|
# "auto" (default) = skip review screens when the auto-approval scorer is
|
|
# fully confident; "always_review" = always stop at the review screens.
|
|
review_mode: str = "auto"
|
|
# "auto" (default) = retain backing vocals where confidently safe; "clean" =
|
|
# always strip backing vocals; "review" = always let a human pick the instrumental.
|
|
backing_preference: str = "auto"
|
|
# Translated lyrics language (see Job.translation_language); None = off
|
|
translation_language: Optional[str] = None
|
|
|
|
# Theme configuration (pre-made themes from GCS)
|
|
theme_id: Optional[str] = None # Theme identifier (e.g., "nomad", "default")
|
|
color_overrides: Dict[str, str] = Field(default_factory=dict)
|
|
"""
|
|
User color overrides applied on top of theme. Keys:
|
|
- artist_color: Hex color for artist name (#RRGGBB)
|
|
- title_color: Hex color for song title
|
|
- sung_lyrics_color: Hex color for highlighted lyrics
|
|
- unsung_lyrics_color: Hex color for unhighlighted lyrics
|
|
"""
|
|
|
|
# Style configuration (will be populated after file upload, or from theme)
|
|
style_params_gcs_path: Optional[str] = None
|
|
style_assets: Dict[str, str] = Field(default_factory=dict)
|
|
|
|
# Finalisation configuration
|
|
brand_prefix: Optional[str] = None
|
|
discord_webhook_url: Optional[str] = None
|
|
youtube_description_template: Optional[str] = None
|
|
|
|
# Distribution configuration (native API - for remote CLI)
|
|
dropbox_path: Optional[str] = None # Dropbox folder path for organized output
|
|
gdrive_folder_id: Optional[str] = None # Google Drive folder ID for public share uploads
|
|
|
|
# Legacy (rclone - deprecated, use dropbox_path instead)
|
|
organised_dir_rclone_root: Optional[str] = None
|
|
|
|
# Lyrics configuration (overrides for search/transcription)
|
|
lyrics_artist: Optional[str] = None # Override artist name for lyrics search
|
|
lyrics_title: Optional[str] = None # Override title for lyrics search
|
|
lyrics_file_gcs_path: Optional[str] = None # GCS path to user-provided lyrics file
|
|
subtitle_offset_ms: int = 0 # Offset for subtitle timing (positive = delay)
|
|
|
|
# Audio separation model configuration
|
|
clean_instrumental_model: Optional[str] = None # Model for clean instrumental separation
|
|
backing_vocals_models: Optional[List[str]] = None # Models for backing vocals separation
|
|
other_stems_models: Optional[List[str]] = None # Models for other stems (bass, drums, etc.)
|
|
|
|
# Existing instrumental configuration (Batch 3)
|
|
existing_instrumental_gcs_path: Optional[str] = None # GCS path to user-provided instrumental file
|
|
|
|
# Audio search configuration (Batch 5 - artist+title search mode)
|
|
audio_search_artist: Optional[str] = None # Artist name used for audio search
|
|
audio_search_title: Optional[str] = None # Title used for audio search
|
|
auto_download: bool = False # Auto-select best audio source (skip selection)
|
|
|
|
# Two-phase workflow configuration (Batch 6)
|
|
prep_only: bool = False # Stop after review, don't run finalisation
|
|
finalise_only: bool = False # Skip prep, run only finalisation
|
|
keep_brand_code: Optional[str] = None # Preserve existing brand code instead of generating new one
|
|
|
|
# Made-for-you order tracking
|
|
made_for_you: bool = False # Flag indicating this is a made-for-you customer order
|
|
customer_email: Optional[str] = None # Customer email for final delivery (job owned by admin during processing)
|
|
customer_notes: Optional[str] = None # Notes provided by customer with their order
|
|
|
|
# Private (non-published) track mode
|
|
is_private: bool = False # Private tracks: Dropbox only (Tracks-NonPublished/NOMADNP), no YouTube/GDrive
|
|
|
|
# Credit pricing: how many credits to charge for this job at creation time.
|
|
# Search and upload jobs pass the default (1) and may top up later.
|
|
# URL jobs with a known duration pass ceil(duration/600) credits.
|
|
credits: int = 1
|
|
|
|
# Request metadata (set by API endpoint from request headers)
|
|
request_metadata: Dict[str, Any] = Field(default_factory=dict)
|
|
"""
|
|
Populated by the API endpoint with request context:
|
|
- client_ip: Client IP address
|
|
- user_agent: User-Agent header
|
|
- environment: From X-Environment header (test/production/development)
|
|
- client_id: From X-Client-ID header
|
|
- server_version: Current server version
|
|
- custom_headers: All X-* headers
|
|
"""
|
|
|
|
# Tenant scoping for white-label portals ("" = consumer, "vocalstar"/etc = tenant)
|
|
tenant_id: str = "" # Tenant ID for job scoping
|
|
|
|
# UI language the job was submitted in (primary Accept-Language subtag, e.g.
|
|
# "pt", "ja"). Set by the API endpoint from the request header; admin-only.
|
|
locale: Optional[str] = None
|
|
|
|
@validator('url')
|
|
def validate_url(cls, v):
|
|
"""Validate URL is not empty."""
|
|
if v is not None and isinstance(v, str) and not v.strip():
|
|
raise ValueError("Field cannot be empty string")
|
|
return v.strip() if isinstance(v, str) else v
|
|
|
|
@validator('review_mode')
|
|
def _normalize_review_mode(cls, v):
|
|
"""Coerce an out-of-set value to the fail-safe default (always_review =
|
|
more human oversight), so a typo can never silently skip review. The
|
|
executor already treats any non-"auto" value as an enforcement blocker."""
|
|
return v if v in ("auto", "always_review") else "always_review"
|
|
|
|
@validator('translation_language')
|
|
def _normalize_translation_language(cls, v):
|
|
"""Supported language code; empty or unsupported values turn the feature off
|
|
(every creation route passes the client value straight through)."""
|
|
if v is None or (isinstance(v, str) and not v.strip()):
|
|
return None
|
|
from backend.services.lyrics_translation import normalize_language
|
|
|
|
return normalize_language(v)
|
|
|
|
@validator('backing_preference')
|
|
def _normalize_backing_preference(cls, v):
|
|
"""Coerce an out-of-set value to the default. Every creation route passes
|
|
the client value straight through, so this is the single choke point that
|
|
keeps a typo from reaching the instrumental decision as an unknown value."""
|
|
return v if v in ("auto", "clean", "review") else "auto"
|
|
|
|
@validator('artist', 'title')
|
|
def normalize_artist_title(cls, v):
|
|
"""Normalize artist/title text to standardize Unicode characters.
|
|
|
|
This ensures consistent data storage by converting:
|
|
- Curly quotes -> straight quotes
|
|
- Various dashes -> hyphen
|
|
- Unusual whitespace -> regular space
|
|
"""
|
|
if v is not None and isinstance(v, str):
|
|
if not v.strip():
|
|
raise ValueError("Field cannot be empty string")
|
|
# normalize_text handles stripping and Unicode normalization
|
|
return normalize_text(v)
|
|
return v
|
|
|
|
|
|
class JobResponse(BaseModel):
|
|
"""Job response model."""
|
|
status: str
|
|
job_id: str
|
|
message: str
|
|
|