Compare commits

...
Author SHA1 Message Date
Andrew Beveridge c94c553069 fix: allow job retry by adding FAILED state transitions
The retry endpoint was silently failing because FAILED status had no valid
transitions in STATE_TRANSITIONS. When retrying a failed job:

1. Error state was cleared correctly
2. transition_to_state() returned False (FAILED -> INSTRUMENTAL_SELECTED invalid)
3. Worker was triggered anyway, completed successfully
4. Final transition to COMPLETE also failed (still in FAILED state)

Fix:
- Add retry checkpoint states (INSTRUMENTAL_SELECTED, REVIEW_COMPLETE,
  LYRICS_COMPLETE) as valid transitions from FAILED status
- Add explicit error checking for transition failures in retry endpoint
- Add test coverage for FAILED state retry transitions

This fixes the bug where successfully retried jobs remained stuck in 'failed'
status with error_message=None.
2025-12-10 22:29:30 -05:00
Andrew Beveridge d757b0c860 feat: improve progress logging during video generation for remote CLI
This change addresses the long silence users experienced when monitoring
remote jobs, particularly during video generation (encoding) stage.

Changes:
- video_worker.py: Add detailed progress logging during file downloads
  with estimated times for large files
- video_worker.py: Fix VIDEO_WORKER_LOGGERS to capture KaraokeFinalise
  logs (logger name was mismatched)
- karaoke_finalise.py: Add step-by-step logging for each FFmpeg encoding
  operation with time estimates ([Step 1/6], [Step 2/6], etc.)
- remote_cli.py: Add heartbeat messages when polling has no updates for
  ~30 seconds to show the CLI is still running
- remote_cli.py: Add human-readable status descriptions (e.g. "encoding"
  becomes "Encoding final videos (15-20 min, 4 formats)")

Bump version to 0.71.18
2025-12-10 22:16:57 -05:00
Andrew Beveridge c632e68b5b Updated workflow to push on all branches 2025-12-10 21:54:06 -05:00
Andrew Beveridge ☄️ 9376d53fed Merge pull request #6 from nomadkaraoke/fix-discord-webhook-error
fix: handle Discord webhook errors gracefully and add job retry capability
2025-12-10 20:21:01 -05:00
Andrew Beveridge 18ee2df5a2 test: add comprehensive tests for --retry CLI mode to improve coverage 2025-12-10 20:15:11 -05:00
Andrew Beveridge 68b8cd25a4 fix: handle Discord webhook errors gracefully and add job retry capability
- Fix Discord webhook URL validation to strip trailing whitespace/newlines
  that can be introduced from environment variables or secrets (fixes %0A error)
- Make Discord notification failures non-fatal - log error and continue
- Add --retry CLI command to retry failed jobs from last checkpoint
- Add POST /api/jobs/{job_id}/retry endpoint for retrying failed jobs
- Bump version to 0.71.17

The root cause was environment variables or secrets containing trailing
newline characters (\n) which get URL-encoded as %0A, breaking Discord
API authentication.
2025-12-10 19:31:59 -05:00
Andrew Beveridge ☄️ cd91b06eb9 Merge pull request #5 from nomadkaraoke/replace-modal-with-google-cloud
Replace modal with google cloud for web hosted version
2025-12-10 16:47:53 -05:00
15 changed files with 723 additions and 51 deletions

No files matched your search

+21
View File
@@ -0,0 +1,21 @@
---
description: Git branching and PR workflow
alwaysApply: true
---
# Branching Workflow
**Never commit directly to `main`.** Always work on a feature branch.
Create descriptive branch names like `feature/add-discord-notifications`, `fix/audio-sync-issue`, or `refactor/cleanup-api-routes`.
# Creating a Pull Request
Once all changes are complete, tests pass, and commits are pushed, create a PR with:
1. **Summary** - Brief overview of the changes
2. **Changes Made** - List of specific modifications
3. **Testing** - How changes were tested (unit tests, manual testing)
4. **Related Issues** - Reference any GitHub issues (e.g., "Fixes #123")
Only merge after review and CI checks pass.
+4 -4
View File
@@ -1,8 +1,7 @@
name: Backend Tests
on:
push:
branches: [ main, replace-modal-with-google-cloud ]
push: # Run on all branch pushes
pull_request:
branches: [ main ]
@@ -143,8 +142,9 @@ jobs:
name: Deploy via Cloud Build
runs-on: ubuntu-latest
needs: [unit-tests, emulator-tests, lint]
# Only deploy on push to the branch (not on PRs)
if: github.event_name == 'push' && github.ref == 'refs/heads/replace-modal-with-google-cloud'
# Deploy on any push where tests pass (pre-alpha: rapid iteration mode)
# TODO: Change back to main-only when ready for production: github.ref == 'refs/heads/main'
if: github.event_name == 'push'
permissions:
contents: read
+153
View File
@@ -722,6 +722,159 @@ async def cancel_job(job_id: str, request: CancelJobRequest) -> dict:
}
@router.post("/{job_id}/retry")
async def retry_job(
job_id: str,
background_tasks: BackgroundTasks
) -> dict:
"""
Retry a failed job from the last successful checkpoint.
This endpoint allows resuming jobs that failed during:
- Video generation (re-runs video worker)
- Encoding (re-runs video worker)
- Packaging (re-runs video worker)
The retry logic determines the appropriate stage to resume from
based on what files/state already exist.
"""
job = job_manager.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if job.status != JobStatus.FAILED:
raise HTTPException(
status_code=400,
detail=f"Only failed jobs can be retried (current status: {job.status})"
)
try:
# Determine retry point based on what's already complete
error_details = job.error_details or {}
error_stage = error_details.get('stage', 'unknown')
logger.info(f"Job {job_id}: Retrying from failed stage '{error_stage}'")
# Check what state exists to determine retry point
file_urls = job.file_urls or {}
state_data = job.state_data or {}
# If we have a video with vocals and instrumental selection, retry video generation
if (file_urls.get('videos', {}).get('with_vocals') and
state_data.get('instrumental_selection')):
logger.info(f"Job {job_id}: Has rendered video and instrumental selection, retrying video generation")
# Clear error state
job_manager.update_job(job_id, {
'error_message': None,
'error_details': None,
})
# Reset to INSTRUMENTAL_SELECTED and trigger video worker
if not job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.INSTRUMENTAL_SELECTED,
progress=65,
message=f"Retrying video generation from failed state"
):
raise HTTPException(
status_code=500,
detail="Failed to transition job status for retry"
)
# Trigger video generation worker
background_tasks.add_task(worker_service.trigger_video_worker, job_id)
return {
"status": "success",
"job_status": "instrumental_selected",
"message": "Job retry started from video generation stage",
"retry_stage": "video_generation"
}
# If we have corrections and screens but no video, retry render
elif (file_urls.get('lyrics', {}).get('corrections') and
file_urls.get('screens', {}).get('title')):
logger.info(f"Job {job_id}: Has corrections and screens, retrying from render stage")
# Clear error state
job_manager.update_job(job_id, {
'error_message': None,
'error_details': None,
})
# Reset to REVIEW_COMPLETE and trigger render worker
if not job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.REVIEW_COMPLETE,
progress=70,
message=f"Retrying video render from failed state"
):
raise HTTPException(
status_code=500,
detail="Failed to transition job status for retry"
)
# Trigger render video worker
background_tasks.add_task(worker_service.trigger_render_video_worker, job_id)
return {
"status": "success",
"job_status": "review_complete",
"message": "Job retry started from render stage",
"retry_stage": "render_video"
}
# If we have stems and corrections, retry from screens generation
elif (file_urls.get('stems', {}).get('instrumental_clean') and
file_urls.get('lyrics', {}).get('corrections')):
logger.info(f"Job {job_id}: Has stems and corrections, retrying from screens stage")
# Clear error state
job_manager.update_job(job_id, {
'error_message': None,
'error_details': None,
})
# Reset to a state before screens and trigger screens worker
if not job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.LYRICS_COMPLETE,
progress=45,
message=f"Retrying from screens generation"
):
raise HTTPException(
status_code=500,
detail="Failed to transition job status for retry"
)
# Trigger screens worker
background_tasks.add_task(worker_service.trigger_screens_worker, job_id)
return {
"status": "success",
"job_status": "lyrics_complete",
"message": "Job retry started from screens generation",
"retry_stage": "screens_generation"
}
else:
# Can't determine a safe retry point, need to restart from beginning
raise HTTPException(
status_code=400,
detail="Cannot determine safe retry point. Job may need to be resubmitted."
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error retrying job {job_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))
@router.get("/{job_id}/logs")
async def get_worker_logs(
job_id: str,
+6 -2
View File
@@ -47,7 +47,10 @@ class Settings(BaseSettings):
# Default distribution settings (can be overridden per-request)
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
default_discord_webhook_url: Optional[str] = os.getenv("DEFAULT_DISCORD_WEBHOOK_URL")
# Strip whitespace/newlines from webhook URL - common issue when env vars are set with trailing newlines
default_discord_webhook_url: Optional[str] = (
os.getenv("DEFAULT_DISCORD_WEBHOOK_URL", "").strip() or None
)
# Secret Manager cache
_secret_cache: Dict[str, str] = {}
@@ -90,7 +93,8 @@ class Settings(BaseSettings):
client = secretmanager.SecretManagerServiceClient()
name = f"projects/{self.google_cloud_project}/secrets/{secret_id}/versions/latest"
response = client.access_secret_version(request={"name": name})
secret_value = response.payload.data.decode('UTF-8')
# Strip whitespace/newlines - common issue when secrets are created with trailing newlines
secret_value = response.payload.data.decode('UTF-8').strip()
# Cache it
self._secret_cache[secret_id] = secret_value
+7 -2
View File
@@ -120,9 +120,14 @@ STATE_TRANSITIONS = {
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
# Terminal states have no transitions
# Terminal states - COMPLETE and CANCELLED have no transitions
# FAILED allows retry transitions to resume from checkpoints
JobStatus.COMPLETE: [],
JobStatus.FAILED: [],
JobStatus.FAILED: [
JobStatus.INSTRUMENTAL_SELECTED, # Retry from video generation
JobStatus.REVIEW_COMPLETE, # Retry from render stage
JobStatus.LYRICS_COMPLETE, # Retry from screens generation
],
JobStatus.CANCELLED: [],
# Legacy states (for backward compatibility)
+10
View File
@@ -62,6 +62,16 @@ class TestJobStatusTransitions:
if status not in excluded:
valid_transitions = STATE_TRANSITIONS.get(status, [])
assert JobStatus.FAILED in valid_transitions, f"{status} should be able to fail"
def test_failed_can_transition_for_retry(self):
"""Test FAILED status can transition to retry checkpoint states."""
from backend.models.job import STATE_TRANSITIONS
valid_transitions = STATE_TRANSITIONS.get(JobStatus.FAILED, [])
# FAILED should allow retry transitions
assert JobStatus.INSTRUMENTAL_SELECTED in valid_transitions, "FAILED should allow retry to INSTRUMENTAL_SELECTED"
assert JobStatus.REVIEW_COMPLETE in valid_transitions, "FAILED should allow retry to REVIEW_COMPLETE"
assert JobStatus.LYRICS_COMPLETE in valid_transitions, "FAILED should allow retry to LYRICS_COMPLETE"
class TestJobModelSerialization:
+26 -13
View File
@@ -41,8 +41,10 @@ logger = logging.getLogger(__name__)
# Loggers to capture for video worker
# Include the full module path to properly capture KaraokeFinalise logs
VIDEO_WORKER_LOGGERS = [
"karaoke_gen.karaoke_finalise",
"karaoke_gen.karaoke_finalise.karaoke_finalise", # The actual logger name from __name__
]
@@ -116,9 +118,9 @@ async def generate_video(job_id: str) -> bool:
# Download and set up files in the format KaraokeFinalise expects
base_name = f"{job.artist} - {job.title}"
job_log.info("Downloading files from GCS...")
await _setup_working_directory(job_id, job, storage, temp_dir, base_name)
job_log.info("Files downloaded successfully")
job_log.info("Downloading files from GCS (this may take a few minutes for large files)...")
await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log)
job_log.info("All files downloaded successfully")
# Load style config for CDG styles
job_log.info("Loading CDG style configuration...")
@@ -443,7 +445,8 @@ async def _setup_working_directory(
job,
storage: StorageService,
temp_dir: str,
base_name: str
base_name: str,
job_log=None
) -> None:
"""
Download files from GCS and set up the directory structure KaraokeFinalise expects.
@@ -455,25 +458,34 @@ async def _setup_working_directory(
- {base_name} (Karaoke).lrc
- Instrumental audio file
"""
logger.info(f"Job {job_id}: Setting up working directory")
def log_progress(message: str):
"""Log to both module logger and job logger if available."""
logger.info(f"Job {job_id}: {message}")
if job_log:
job_log.info(message)
log_progress("Setting up working directory")
# Download title screen
log_progress("Downloading title screen...")
title_url = job.file_urls['screens']['title']
title_path = os.path.join(temp_dir, f"{base_name} (Title).mov")
storage.download_file(title_url, title_path)
logger.info(f"Job {job_id}: Downloaded title screen")
log_progress("Downloaded title screen")
# Download end screen
log_progress("Downloading end screen...")
end_url = job.file_urls['screens']['end']
end_path = os.path.join(temp_dir, f"{base_name} (End).mov")
storage.download_file(end_url, end_path)
logger.info(f"Job {job_id}: Downloaded end screen")
log_progress("Downloaded end screen")
# Download lyrics video (with vocals)
# Download lyrics video (with vocals) - this is the largest file, ~1-2GB
log_progress("Downloading karaoke video (largest file, may take 1-2 minutes)...")
lyrics_video_url = job.file_urls['videos']['with_vocals']
lyrics_video_path = os.path.join(temp_dir, f"{base_name} (With Vocals).mov")
storage.download_file(lyrics_video_url, lyrics_video_path)
logger.info(f"Job {job_id}: Downloaded lyrics video")
log_progress("Downloaded karaoke video")
# Download selected instrumental
instrumental_selection = job.state_data['instrumental_selection']
@@ -481,8 +493,9 @@ async def _setup_working_directory(
instrumental_url = job.file_urls['stems'][instrumental_key]
instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing"
instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac")
log_progress(f"Downloading {instrumental_selection} instrumental audio...")
storage.download_file(instrumental_url, instrumental_path)
logger.info(f"Job {job_id}: Downloaded instrumental ({instrumental_selection})")
log_progress(f"Downloaded instrumental ({instrumental_selection})")
# Download LRC file if available (needed for CDG/TXT)
lyrics_urls = job.file_urls.get('lyrics', {})
@@ -490,19 +503,19 @@ async def _setup_working_directory(
lrc_url = lyrics_urls['lrc']
lrc_path = os.path.join(temp_dir, f"{base_name} (Karaoke).lrc")
storage.download_file(lrc_url, lrc_path)
logger.info(f"Job {job_id}: Downloaded LRC file")
log_progress("Downloaded LRC file")
# Download title/end JPG files (used for YouTube thumbnail)
screens = job.file_urls.get('screens', {})
if screens.get('title_jpg'):
title_jpg_path = os.path.join(temp_dir, f"{base_name} (Title).jpg")
storage.download_file(screens['title_jpg'], title_jpg_path)
logger.info(f"Job {job_id}: Downloaded title JPG for thumbnail")
log_progress("Downloaded title JPG for thumbnail")
if screens.get('end_jpg'):
end_jpg_path = os.path.join(temp_dir, f"{base_name} (End).jpg")
storage.download_file(screens['end_jpg'], end_jpg_path)
logger.info(f"Job {job_id}: Downloaded end JPG")
log_progress("Downloaded end JPG")
async def _upload_results(
@@ -268,6 +268,8 @@ class KaraokeFinalise:
# Enable discord notifications if webhook URL is provided and is valid URL
if self.discord_webhook_url is not None:
# Strip whitespace/newlines that may have been introduced from environment variables or secrets
self.discord_webhook_url = self.discord_webhook_url.strip()
if not self.discord_webhook_url.startswith("https://discord.com/api/webhooks/"):
raise Exception(f"Discord webhook URL is not valid: {self.discord_webhook_url}")
@@ -858,7 +860,7 @@ class KaraokeFinalise:
return env_mov_input, ffmpeg_filter
def remux_and_encode_output_video_files(self, with_vocals_file, input_files, output_files):
self.logger.info(f"Remuxing and encoding output video files...")
self.logger.info(f"Remuxing and encoding output video files (4 formats, ~15-20 minutes total)...")
# Check if output files already exist
if os.path.isfile(output_files["final_karaoke_lossless_mp4"]) and os.path.isfile(output_files["final_karaoke_lossless_mkv"]):
@@ -869,16 +871,20 @@ class KaraokeFinalise:
return
# Create karaoke version with instrumental audio
self.logger.info(f"[Step 1/6] Remuxing video with instrumental audio...")
self.remux_with_instrumental(with_vocals_file, input_files["instrumental_audio"], output_files["karaoke_mp4"])
# Convert the with vocals video to MP4 if needed
if not with_vocals_file.endswith(".mp4"):
self.logger.info(f"[Step 2/6] Converting karaoke video to MP4...")
self.convert_mov_to_mp4(with_vocals_file, output_files["with_vocals_mp4"])
# Delete the with vocals mov after successfully converting it to mp4
if not self.dry_run and os.path.isfile(with_vocals_file):
self.logger.info(f"Deleting with vocals MOV file: {with_vocals_file}")
os.remove(with_vocals_file)
else:
self.logger.info(f"[Step 2/6] Skipped - video already in MP4 format")
# Quote file paths to handle special characters
title_mov_file = shlex.quote(os.path.abspath(input_files["title_mov"]))
@@ -887,10 +893,17 @@ class KaraokeFinalise:
# Prepare concat filter for combining videos
env_mov_input, ffmpeg_filter = self.prepare_concat_filter(input_files)
# Create all output versions
# Create all output versions with progress logging
self.logger.info(f"[Step 3/6] Encoding lossless 4K MP4 (title + karaoke + end, ~5 minutes)...")
self.encode_lossless_mp4(title_mov_file, karaoke_mp4_file, env_mov_input, ffmpeg_filter, output_files["final_karaoke_lossless_mp4"])
self.logger.info(f"[Step 4/6] Encoding lossy 4K MP4 with AAC audio (~1 minute)...")
self.encode_lossy_mp4(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossy_mp4"])
self.logger.info(f"[Step 5/6] Creating MKV with FLAC audio for YouTube (~1 minute)...")
self.encode_lossless_mkv(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossless_mkv"])
self.logger.info(f"[Step 6/6] Encoding 720p version (~3 minutes)...")
self.encode_720p_version(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossy_720p_mp4"])
# Skip user confirmation in non-interactive mode for Modal deployment
@@ -1298,8 +1311,13 @@ class KaraokeFinalise:
self.logger.error("YouTube upload failed in non-interactive mode, skipping")
# Discord notification - runs independently of YouTube upload
# Wrapped in try/except so failures don't crash the entire job
if self.discord_notication_enabled:
self.post_discord_notification()
try:
self.post_discord_notification()
except Exception as e:
self.logger.error(f"Failed to send Discord notification: {e}")
self.logger.warning("Continuing without Discord notification - this is non-fatal")
# Handle folder organization - different logic for server-side vs local mode
if self.server_side_mode and self.brand_prefix and self.organised_dir_rclone_root:
@@ -1661,11 +1679,17 @@ class KaraokeFinalise:
if self.dry_run:
self.logger.warning("Dry run enabled. No actions will be performed.")
self.logger.info("=" * 60)
self.logger.info("Starting KaraokeFinalise processing pipeline")
self.logger.info("=" * 60)
# Check required input files and parameters exist, get user to confirm features before proceeding
self.validate_input_parameters_for_features()
with_vocals_file = self.find_with_vocals_file()
base_name, artist, title = self.get_names_from_withvocals(with_vocals_file)
self.logger.info(f"Processing: {artist} - {title}")
# Use the selected instrumental file if provided, otherwise search for one
if self.selected_instrumental_file:
@@ -1681,13 +1705,20 @@ class KaraokeFinalise:
output_files = self.prepare_output_filenames(base_name)
if self.enable_cdg:
self.logger.info("Creating CDG package...")
self.create_cdg_zip_file(input_files, output_files, artist, title)
self.logger.info("CDG package created successfully")
if self.enable_txt:
self.logger.info("Creating TXT package...")
self.create_txt_zip_file(input_files, output_files)
self.logger.info("TXT package created successfully")
self.logger.info("Starting video encoding (this is the longest step, ~15-20 minutes)...")
self.remux_and_encode_output_video_files(with_vocals_file, input_files, output_files)
self.logger.info("Video encoding completed successfully")
self.logger.info("Executing distribution features (YouTube, Dropbox, Discord)...")
self.execute_optional_features(artist, title, base_name, input_files, output_files, replace_existing)
result = {
+5
View File
@@ -90,6 +90,11 @@ def create_parser(prog: str = "karaoke-gen") -> argparse.ArgumentParser:
metavar="JOB_ID",
help="Cancel a running job (remote mode only). Stops processing but keeps the job record. Example: --cancel abc12345",
)
workflow_group.add_argument(
"--retry",
metavar="JOB_ID",
help="Retry a failed job from the last successful checkpoint (remote mode only). Example: --retry abc12345",
)
workflow_group.add_argument(
"--delete",
metavar="JOB_ID",
+134 -8
View File
@@ -8,6 +8,7 @@ Set KARAOKE_GEN_URL environment variable to your cloud backend URL.
Usage:
karaoke-gen-remote <filepath> <artist> <title>
karaoke-gen-remote --resume <job_id>
karaoke-gen-remote --retry <job_id>
karaoke-gen-remote --list
karaoke-gen-remote --cancel <job_id>
karaoke-gen-remote --delete <job_id>
@@ -383,7 +384,25 @@ class RemoteKaraokeClient:
if response.status_code != 200:
raise RuntimeError(f"Error deleting job: {response.text}")
return response.json()
def retry_job(self, job_id: str) -> Dict[str, Any]:
"""Retry a failed job from the last successful checkpoint."""
response = self._request(
'POST',
f'/api/jobs/{job_id}/retry'
)
if response.status_code == 404:
raise ValueError(f"Job not found: {job_id}")
if response.status_code == 400:
try:
error_detail = response.json().get('detail', response.text)
except Exception:
error_detail = response.text
raise RuntimeError(f"Cannot retry job: {error_detail}")
if response.status_code != 200:
raise RuntimeError(f"Error retrying job: {response.text}")
return response.json()
def list_jobs(
self,
status: Optional[str] = None,
@@ -591,6 +610,40 @@ class JobMonitor:
self._last_timeline_index = 0
self._last_log_index = 0
self._show_worker_logs = True # Enable worker log display
self._polls_without_updates = 0 # Track polling activity for heartbeat
self._heartbeat_interval = 6 # Show heartbeat every N polls without updates (~30s with 5s poll)
# Status descriptions for user-friendly logging
STATUS_DESCRIPTIONS = {
'pending': 'Job queued, waiting to start',
'downloading': 'Downloading and preparing input files',
'separating_stage1': 'AI audio separation (stage 1 of 2)',
'separating_stage2': 'AI audio separation (stage 2 of 2)',
'audio_complete': 'Audio separation complete',
'transcribing': 'Transcribing lyrics from audio',
'correcting': 'Auto-correcting lyrics against reference sources',
'lyrics_complete': 'Lyrics processing complete',
'generating_screens': 'Creating title and end screens',
'applying_padding': 'Adding intro/outro padding',
'awaiting_review': 'Waiting for lyrics review',
'in_review': 'Lyrics review in progress',
'review_complete': 'Review complete, preparing video render',
'rendering_video': 'Rendering karaoke video with lyrics',
'awaiting_instrumental_selection': 'Waiting for instrumental selection',
'instrumental_selected': 'Instrumental selected, preparing final encoding',
'generating_video': 'Downloading files for final video encoding',
'encoding': 'Encoding final videos (15-20 min, 4 formats)',
'packaging': 'Creating CDG/TXT packages',
'uploading': 'Uploading to distribution services',
'notifying': 'Sending notifications',
'complete': 'All processing complete',
'failed': 'Job failed',
'cancelled': 'Job cancelled',
}
def _get_status_description(self, status: str) -> str:
"""Get user-friendly description for a status."""
return self.STATUS_DESCRIPTIONS.get(status, status)
def open_browser(self, url: str) -> None:
"""Open URL in the default browser."""
@@ -1082,6 +1135,7 @@ class JobMonitor:
self.logger.info(f"Monitoring job: {job_id}")
self.logger.info(f"Service URL: {self.config.service_url}")
self.logger.info(f"Polling every {self.config.poll_interval} seconds...")
self.logger.info("")
while True:
@@ -1092,16 +1146,40 @@ class JobMonitor:
artist = job_data.get('artist', '')
title = job_data.get('title', '')
# Track whether we got any new updates this poll
had_updates = False
prev_timeline_index = self._last_timeline_index
prev_log_index = self._last_log_index
# Log timeline updates (shows status changes and progress)
self.log_timeline_updates(job_data)
if self._last_timeline_index > prev_timeline_index:
had_updates = True
# Log worker logs (shows detailed worker output for debugging)
self.log_worker_logs(job_id)
if self._last_log_index > prev_log_index:
had_updates = True
# Log status changes
# Log status changes with user-friendly descriptions
if status != last_status:
self.logger.info(f"Status changed: {last_status} -> {status}")
description = self._get_status_description(status)
if last_status:
self.logger.info(f"Status: {status} - {description}")
else:
self.logger.info(f"Current status: {status} - {description}")
last_status = status
had_updates = True
# Heartbeat: if no updates for a while, show we're still alive
if had_updates:
self._polls_without_updates = 0
else:
self._polls_without_updates += 1
if self._polls_without_updates >= self._heartbeat_interval:
description = self._get_status_description(status)
self.logger.info(f" [Still processing: {description}]")
self._polls_without_updates = 0
# Handle human interaction points
if status in ['awaiting_review', 'in_review']:
@@ -1388,6 +1466,7 @@ def main():
logger.info("")
logger.info(f"Total: {len(jobs)} jobs")
logger.info("")
logger.info("To retry a failed job: karaoke-gen-remote --retry <JOB_ID>")
logger.info("To delete a job: karaoke-gen-remote --delete <JOB_ID>")
logger.info("To bulk delete: karaoke-gen-remote --bulk-delete --filter-environment=test")
logger.info("To cancel a job: karaoke-gen-remote --cancel <JOB_ID>")
@@ -1403,24 +1482,24 @@ def main():
logger.info("Karaoke Generator (Remote) - Cancel Job")
logger.info("=" * 60)
logger.info(f"Job ID: {args.cancel}")
try:
# Get job info first
job_data = client.get_job(args.cancel)
artist = job_data.get('artist', 'Unknown')
title = job_data.get('title', 'Unknown')
status = job_data.get('status', 'unknown')
logger.info(f"Artist: {artist}")
logger.info(f"Title: {title}")
logger.info(f"Current status: {status}")
logger.info("")
# Cancel the job
result = client.cancel_job(args.cancel)
logger.info(f"✓ Job cancelled successfully")
return 0
except ValueError as e:
logger.error(str(e))
return 1
@@ -1430,7 +1509,54 @@ def main():
except Exception as e:
logger.error(f"Error cancelling job: {e}")
return 1
# Handle retry job mode
if args.retry:
logger.info("=" * 60)
logger.info("Karaoke Generator (Remote) - Retry Failed Job")
logger.info("=" * 60)
logger.info(f"Job ID: {args.retry}")
try:
# Get job info first
job_data = client.get_job(args.retry)
artist = job_data.get('artist', 'Unknown')
title = job_data.get('title', 'Unknown')
status = job_data.get('status', 'unknown')
error_message = job_data.get('error_message', 'No error message')
logger.info(f"Artist: {artist}")
logger.info(f"Title: {title}")
logger.info(f"Current status: {status}")
if status == 'failed':
logger.info(f"Error: {error_message}")
logger.info("")
if status != 'failed':
logger.error(f"Only failed jobs can be retried (current status: {status})")
return 1
# Retry the job
result = client.retry_job(args.retry)
retry_stage = result.get('retry_stage', 'unknown')
logger.info(f"✓ Job retry started from stage: {retry_stage}")
logger.info("")
logger.info(f"Monitoring job progress...")
logger.info("")
# Monitor the retried job
return monitor.monitor(args.retry)
except ValueError as e:
logger.error(str(e))
return 1
except RuntimeError as e:
logger.error(str(e))
return 1
except Exception as e:
logger.error(f"Error retrying job: {e}")
return 1
# Handle delete job mode
if args.delete:
logger.info("=" * 60)
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.71.16"
version = "0.71.19"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+1 -1
View File
@@ -112,7 +112,7 @@ echo -e "${YELLOW}---BEGIN SERVICE ACCOUNT KEY---${NC}"
cat "$KEY_FILE"
echo -e "${YELLOW}---END SERVICE ACCOUNT KEY---${NC}"
echo ""
echo "3. After adding secrets, push to 'replace-modal-with-google-cloud' branch"
echo "3. After adding secrets, push to 'main' branch"
echo " to trigger an automated deployment!"
echo ""
echo -e "${RED}⚠️ IMPORTANT SECURITY NOTES:${NC}"
@@ -47,6 +47,42 @@ def mock_gmail_service():
# --- Discord Notification Tests ---
def test_discord_webhook_url_strips_whitespace(mock_logger):
"""Test that Discord webhook URL validation strips trailing whitespace/newlines."""
# This tests the fix for the %0A issue where webhook URLs have trailing newlines
config = MINIMAL_CONFIG.copy()
config.update({
# URL with trailing newline (simulating env var issue)
"discord_webhook_url": "https://discord.com/api/webhooks/123/abc\n",
})
with patch.object(KaraokeFinalise, 'detect_best_aac_codec', return_value='aac'):
finaliser = KaraokeFinalise(logger=mock_logger, **config)
# Call validate_input_parameters_for_features to trigger the strip
with patch('os.getcwd', return_value='/tmp'):
finaliser.validate_input_parameters_for_features()
# The URL should have been stripped
assert finaliser.discord_webhook_url == "https://discord.com/api/webhooks/123/abc"
assert finaliser.discord_notication_enabled is True
def test_discord_webhook_url_strips_whitespace_and_validates(mock_logger):
"""Test that invalid Discord URLs are rejected even after stripping."""
config = MINIMAL_CONFIG.copy()
config.update({
# Invalid URL that starts correctly but is malformed
"discord_webhook_url": "https://discord.com/invalid\n",
})
with patch.object(KaraokeFinalise, 'detect_best_aac_codec', return_value='aac'):
finaliser = KaraokeFinalise(logger=mock_logger, **config)
# Should raise because URL doesn't start with correct prefix after strip
with patch('os.getcwd', return_value='/tmp'):
with pytest.raises(Exception, match="Discord webhook URL is not valid"):
finaliser.validate_input_parameters_for_features()
@patch('requests.post')
def test_post_discord_message_success(mock_post, finaliser_for_notify):
"""Test successful posting to Discord."""
@@ -171,6 +171,40 @@ def test_execute_optional_features_youtube_fails(mock_input, mock_discord, mock_
assert finaliser_for_process.youtube_url == f"https://www.youtube.com/watch?v=manual_video_id"
mock_discord.assert_called_once() # Discord should still be called after manual ID entry
@patch.object(KaraokeFinalise, 'upload_final_mp4_to_youtube_with_title_thumbnail')
@patch.object(KaraokeFinalise, 'post_discord_notification')
@patch.object(KaraokeFinalise, 'get_next_brand_code', return_value=f"{BRAND_PREFIX}-0001")
@patch.object(KaraokeFinalise, 'move_files_to_brand_code_folder')
@patch.object(KaraokeFinalise, 'copy_final_files_to_public_share_dirs')
@patch.object(KaraokeFinalise, 'sync_public_share_dir_to_rclone_destination')
@patch.object(KaraokeFinalise, 'generate_organised_folder_sharing_link')
def test_execute_optional_features_discord_failure_continues(
mock_gen_link, mock_sync, mock_copy, mock_move, mock_get_next,
mock_discord, mock_youtube, finaliser_for_process):
"""Test that Discord notification failure doesn't crash the job."""
# Simulate Discord notification failure
mock_discord.side_effect = Exception("401 Unauthorized: Invalid webhook URL")
finaliser_for_process.keep_brand_code = False
finaliser_for_process.new_brand_code_dir_path = os.path.join(ORGANISED_DIR, f"{BRAND_PREFIX}-0001 - {ARTIST} - {TITLE}")
# Should not raise an exception - should continue with other features
finaliser_for_process.execute_optional_features(ARTIST, TITLE, BASE_NAME, ALL_INPUT_FILES, ALL_OUTPUT_FILES, False)
# Discord was called (and failed)
mock_discord.assert_called_once()
# But other features should still run
mock_youtube.assert_called_once()
mock_get_next.assert_called_once()
mock_move.assert_called_once()
mock_copy.assert_called_once()
mock_sync.assert_called_once()
mock_gen_link.assert_called_once()
# Logger should have logged the error (warning)
finaliser_for_process.logger.error.assert_called()
finaliser_for_process.logger.warning.assert_called()
# --- process Method Tests ---
@patch.object(KaraokeFinalise, 'validate_input_parameters_for_features')
+251 -17
View File
@@ -426,10 +426,51 @@ class TestRemoteKaraokeClient:
mock_response = MagicMock()
mock_response.status_code = 404
mock_request.return_value = mock_response
with pytest.raises(ValueError, match="Job not found"):
client.delete_job("nonexistent-job")
@patch.object(RemoteKaraokeClient, '_request')
def test_retry_job_success(self, mock_request, client):
"""Test retrying a failed job successfully."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"status": "success",
"job_status": "instrumental_selected",
"retry_stage": "video_generation"
}
mock_request.return_value = mock_response
result = client.retry_job("job-123")
assert result["status"] == "success"
assert result["retry_stage"] == "video_generation"
mock_request.assert_called_once_with('POST', '/api/jobs/job-123/retry')
@patch.object(RemoteKaraokeClient, '_request')
def test_retry_job_not_found(self, mock_request, client):
"""Test retrying non-existent job."""
mock_response = MagicMock()
mock_response.status_code = 404
mock_request.return_value = mock_response
with pytest.raises(ValueError, match="Job not found"):
client.retry_job("nonexistent-job")
@patch.object(RemoteKaraokeClient, '_request')
def test_retry_job_not_failed(self, mock_request, client):
"""Test retrying a job that isn't in failed state."""
mock_response = MagicMock()
mock_response.status_code = 400
mock_response.json.return_value = {
"detail": "Only failed jobs can be retried"
}
mock_request.return_value = mock_response
with pytest.raises(RuntimeError, match="Cannot retry job"):
client.retry_job("job-123")
@patch.object(RemoteKaraokeClient, '_request')
def test_get_instrumental_options(self, mock_request, client):
"""Test getting instrumental options."""
@@ -779,6 +820,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -788,11 +830,11 @@ class TestMain:
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_auth.return_value = "token"
result = main()
assert result == 1
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.get_auth_token')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@@ -810,6 +852,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -819,11 +862,11 @@ class TestMain:
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_auth.return_value = "token"
result = main()
assert result == 1
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.get_auth_token')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@@ -841,6 +884,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -850,11 +894,11 @@ class TestMain:
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_auth.return_value = "token"
result = main()
assert result == 1
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.get_auth_token')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@@ -872,6 +916,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None # Added for retry feature
mock_args.delete = None
mock_args.bulk_delete = False # New attribute for bulk delete mode
mock_args.filter_environment = None # New attribute for filtering
@@ -880,9 +925,9 @@ class TestMain:
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_auth.return_value = "token"
result = main()
assert result == 1
mock_parser.print_help.assert_called_once()
@@ -952,6 +997,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = True
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False # New attribute for bulk delete mode
mock_args.filter_environment = None # New attribute for filtering
@@ -960,7 +1006,7 @@ class TestMain:
mock_args.client_id = "" # Job tracking
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.list_jobs.return_value = [
{"job_id": "job-1", "status": "complete", "artist": "Artist 1", "title": "Title 1", "request_metadata": {}},
@@ -993,6 +1039,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = True
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1001,7 +1048,7 @@ class TestMain:
mock_args.client_id = ""
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.list_jobs.return_value = []
mock_client_class.return_value = mock_client
@@ -1052,7 +1099,187 @@ class TestMain:
assert result == 0
mock_client.cancel_job.assert_called_once_with("job-123")
@patch('karaoke_gen.utils.remote_cli.JobMonitor')
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
def test_main_retry_job_mode(
self, mock_client_class, mock_check, mock_create_parser, mock_monitor_class
):
"""Test main in retry job mode with a failed job."""
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.service_url = "https://api.example.com"
mock_args.review_ui_url = "https://review.example.com"
mock_args.poll_interval = 5
mock_args.output_dir = "/tmp"
mock_args.log_level = "info"
mock_args.finalise_only = False
mock_args.edit_lyrics = False
mock_args.test_email_template = False
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = "job-123"
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
mock_args.filter_client_id = None
mock_args.environment = ""
mock_args.client_id = ""
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.get_job.return_value = {
"artist": "Artist",
"title": "Title",
"status": "failed",
"error_message": "Discord webhook failed"
}
mock_client.retry_job.return_value = {
"status": "success",
"retry_stage": "video_generation"
}
mock_client_class.return_value = mock_client
mock_monitor = MagicMock()
mock_monitor.monitor.return_value = 0
mock_monitor_class.return_value = mock_monitor
result = main()
assert result == 0
mock_client.retry_job.assert_called_once_with("job-123")
mock_monitor.monitor.assert_called_once_with("job-123")
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
def test_main_retry_job_not_failed(
self, mock_client_class, mock_check, mock_create_parser
):
"""Test main retry job when job is not in failed state."""
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.service_url = "https://api.example.com"
mock_args.review_ui_url = "https://review.example.com"
mock_args.poll_interval = 5
mock_args.output_dir = "/tmp"
mock_args.log_level = "info"
mock_args.finalise_only = False
mock_args.edit_lyrics = False
mock_args.test_email_template = False
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = "job-123"
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
mock_args.filter_client_id = None
mock_args.environment = ""
mock_args.client_id = ""
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.get_job.return_value = {
"artist": "Artist",
"title": "Title",
"status": "complete" # Not failed
}
mock_client_class.return_value = mock_client
result = main()
assert result == 1 # Should fail because job is not in failed state
mock_client.retry_job.assert_not_called()
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
def test_main_retry_job_not_found(
self, mock_client_class, mock_check, mock_create_parser
):
"""Test main retry job when job not found."""
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.service_url = "https://api.example.com"
mock_args.review_ui_url = "https://review.example.com"
mock_args.poll_interval = 5
mock_args.output_dir = "/tmp"
mock_args.log_level = "info"
mock_args.finalise_only = False
mock_args.edit_lyrics = False
mock_args.test_email_template = False
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = "nonexistent-job"
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
mock_args.filter_client_id = None
mock_args.environment = ""
mock_args.client_id = ""
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.get_job.side_effect = ValueError("Job not found")
mock_client_class.return_value = mock_client
result = main()
assert result == 1
mock_client.retry_job.assert_not_called()
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
def test_main_retry_job_api_error(
self, mock_client_class, mock_check, mock_create_parser
):
"""Test main retry job when API returns an error."""
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.service_url = "https://api.example.com"
mock_args.review_ui_url = "https://review.example.com"
mock_args.poll_interval = 5
mock_args.output_dir = "/tmp"
mock_args.log_level = "info"
mock_args.finalise_only = False
mock_args.edit_lyrics = False
mock_args.test_email_template = False
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = "job-123"
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
mock_args.filter_client_id = None
mock_args.environment = ""
mock_args.client_id = ""
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.get_job.return_value = {
"artist": "Artist",
"title": "Title",
"status": "failed",
"error_message": "Some error"
}
mock_client.retry_job.side_effect = RuntimeError("Cannot retry job: No safe checkpoint found")
mock_client_class.return_value = mock_client
result = main()
assert result == 1
@patch('karaoke_gen.utils.remote_cli.create_parser')
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
@@ -1073,6 +1300,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = "nonexistent"
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1111,6 +1339,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = "job-123"
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1156,6 +1385,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = "job-123"
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1199,6 +1429,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = "job-123"
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1208,7 +1439,7 @@ class TestMain:
mock_args.yes = True # Non-interactive mode
mock_parser.parse_args.return_value = mock_args
mock_create_parser.return_value = mock_parser
mock_client = MagicMock()
mock_client.get_job.return_value = {
"artist": "Artist",
@@ -1245,6 +1476,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1288,6 +1520,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None
@@ -1332,6 +1565,7 @@ class TestMain:
mock_args.resume = None
mock_args.list_jobs = False
mock_args.cancel = None
mock_args.retry = None
mock_args.delete = None
mock_args.bulk_delete = False
mock_args.filter_environment = None