mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-22 05:47:55 +09:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c94c553069 | ||
|
|
d757b0c860 | ||
|
|
c632e68b5b | ||
|
|
9376d53fed | ||
|
|
18ee2df5a2 | ||
|
|
68b8cd25a4 | ||
|
|
cd91b06eb9 |
No files matched your search
@@ -0,0 +1,21 @@
|
||||
---
|
||||
description: Git branching and PR workflow
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Branching Workflow
|
||||
|
||||
**Never commit directly to `main`.** Always work on a feature branch.
|
||||
|
||||
Create descriptive branch names like `feature/add-discord-notifications`, `fix/audio-sync-issue`, or `refactor/cleanup-api-routes`.
|
||||
|
||||
# Creating a Pull Request
|
||||
|
||||
Once all changes are complete, tests pass, and commits are pushed, create a PR with:
|
||||
|
||||
1. **Summary** - Brief overview of the changes
|
||||
2. **Changes Made** - List of specific modifications
|
||||
3. **Testing** - How changes were tested (unit tests, manual testing)
|
||||
4. **Related Issues** - Reference any GitHub issues (e.g., "Fixes #123")
|
||||
|
||||
Only merge after review and CI checks pass.
|
||||
@@ -1,8 +1,7 @@
|
||||
name: Backend Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, replace-modal-with-google-cloud ]
|
||||
push: # Run on all branch pushes
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
@@ -143,8 +142,9 @@ jobs:
|
||||
name: Deploy via Cloud Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [unit-tests, emulator-tests, lint]
|
||||
# Only deploy on push to the branch (not on PRs)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/replace-modal-with-google-cloud'
|
||||
# Deploy on any push where tests pass (pre-alpha: rapid iteration mode)
|
||||
# TODO: Change back to main-only when ready for production: github.ref == 'refs/heads/main'
|
||||
if: github.event_name == 'push'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -722,6 +722,159 @@ async def cancel_job(job_id: str, request: CancelJobRequest) -> dict:
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{job_id}/retry")
|
||||
async def retry_job(
|
||||
job_id: str,
|
||||
background_tasks: BackgroundTasks
|
||||
) -> dict:
|
||||
"""
|
||||
Retry a failed job from the last successful checkpoint.
|
||||
|
||||
This endpoint allows resuming jobs that failed during:
|
||||
- Video generation (re-runs video worker)
|
||||
- Encoding (re-runs video worker)
|
||||
- Packaging (re-runs video worker)
|
||||
|
||||
The retry logic determines the appropriate stage to resume from
|
||||
based on what files/state already exist.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status != JobStatus.FAILED:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Only failed jobs can be retried (current status: {job.status})"
|
||||
)
|
||||
|
||||
try:
|
||||
# Determine retry point based on what's already complete
|
||||
error_details = job.error_details or {}
|
||||
error_stage = error_details.get('stage', 'unknown')
|
||||
|
||||
logger.info(f"Job {job_id}: Retrying from failed stage '{error_stage}'")
|
||||
|
||||
# Check what state exists to determine retry point
|
||||
file_urls = job.file_urls or {}
|
||||
state_data = job.state_data or {}
|
||||
|
||||
# If we have a video with vocals and instrumental selection, retry video generation
|
||||
if (file_urls.get('videos', {}).get('with_vocals') and
|
||||
state_data.get('instrumental_selection')):
|
||||
|
||||
logger.info(f"Job {job_id}: Has rendered video and instrumental selection, retrying video generation")
|
||||
|
||||
# Clear error state
|
||||
job_manager.update_job(job_id, {
|
||||
'error_message': None,
|
||||
'error_details': None,
|
||||
})
|
||||
|
||||
# Reset to INSTRUMENTAL_SELECTED and trigger video worker
|
||||
if not job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.INSTRUMENTAL_SELECTED,
|
||||
progress=65,
|
||||
message=f"Retrying video generation from failed state"
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to transition job status for retry"
|
||||
)
|
||||
|
||||
# Trigger video generation worker
|
||||
background_tasks.add_task(worker_service.trigger_video_worker, job_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "instrumental_selected",
|
||||
"message": "Job retry started from video generation stage",
|
||||
"retry_stage": "video_generation"
|
||||
}
|
||||
|
||||
# If we have corrections and screens but no video, retry render
|
||||
elif (file_urls.get('lyrics', {}).get('corrections') and
|
||||
file_urls.get('screens', {}).get('title')):
|
||||
|
||||
logger.info(f"Job {job_id}: Has corrections and screens, retrying from render stage")
|
||||
|
||||
# Clear error state
|
||||
job_manager.update_job(job_id, {
|
||||
'error_message': None,
|
||||
'error_details': None,
|
||||
})
|
||||
|
||||
# Reset to REVIEW_COMPLETE and trigger render worker
|
||||
if not job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.REVIEW_COMPLETE,
|
||||
progress=70,
|
||||
message=f"Retrying video render from failed state"
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to transition job status for retry"
|
||||
)
|
||||
|
||||
# Trigger render video worker
|
||||
background_tasks.add_task(worker_service.trigger_render_video_worker, job_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "review_complete",
|
||||
"message": "Job retry started from render stage",
|
||||
"retry_stage": "render_video"
|
||||
}
|
||||
|
||||
# If we have stems and corrections, retry from screens generation
|
||||
elif (file_urls.get('stems', {}).get('instrumental_clean') and
|
||||
file_urls.get('lyrics', {}).get('corrections')):
|
||||
|
||||
logger.info(f"Job {job_id}: Has stems and corrections, retrying from screens stage")
|
||||
|
||||
# Clear error state
|
||||
job_manager.update_job(job_id, {
|
||||
'error_message': None,
|
||||
'error_details': None,
|
||||
})
|
||||
|
||||
# Reset to a state before screens and trigger screens worker
|
||||
if not job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.LYRICS_COMPLETE,
|
||||
progress=45,
|
||||
message=f"Retrying from screens generation"
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to transition job status for retry"
|
||||
)
|
||||
|
||||
# Trigger screens worker
|
||||
background_tasks.add_task(worker_service.trigger_screens_worker, job_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "lyrics_complete",
|
||||
"message": "Job retry started from screens generation",
|
||||
"retry_stage": "screens_generation"
|
||||
}
|
||||
|
||||
else:
|
||||
# Can't determine a safe retry point, need to restart from beginning
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot determine safe retry point. Job may need to be resubmitted."
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrying job {job_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/{job_id}/logs")
|
||||
async def get_worker_logs(
|
||||
job_id: str,
|
||||
|
||||
+6
-2
@@ -47,7 +47,10 @@ class Settings(BaseSettings):
|
||||
# Default distribution settings (can be overridden per-request)
|
||||
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
|
||||
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
|
||||
default_discord_webhook_url: Optional[str] = os.getenv("DEFAULT_DISCORD_WEBHOOK_URL")
|
||||
# Strip whitespace/newlines from webhook URL - common issue when env vars are set with trailing newlines
|
||||
default_discord_webhook_url: Optional[str] = (
|
||||
os.getenv("DEFAULT_DISCORD_WEBHOOK_URL", "").strip() or None
|
||||
)
|
||||
|
||||
# Secret Manager cache
|
||||
_secret_cache: Dict[str, str] = {}
|
||||
@@ -90,7 +93,8 @@ class Settings(BaseSettings):
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
name = f"projects/{self.google_cloud_project}/secrets/{secret_id}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
secret_value = response.payload.data.decode('UTF-8')
|
||||
# Strip whitespace/newlines - common issue when secrets are created with trailing newlines
|
||||
secret_value = response.payload.data.decode('UTF-8').strip()
|
||||
|
||||
# Cache it
|
||||
self._secret_cache[secret_id] = secret_value
|
||||
|
||||
@@ -120,9 +120,14 @@ STATE_TRANSITIONS = {
|
||||
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
|
||||
# Terminal states have no transitions
|
||||
# Terminal states - COMPLETE and CANCELLED have no transitions
|
||||
# FAILED allows retry transitions to resume from checkpoints
|
||||
JobStatus.COMPLETE: [],
|
||||
JobStatus.FAILED: [],
|
||||
JobStatus.FAILED: [
|
||||
JobStatus.INSTRUMENTAL_SELECTED, # Retry from video generation
|
||||
JobStatus.REVIEW_COMPLETE, # Retry from render stage
|
||||
JobStatus.LYRICS_COMPLETE, # Retry from screens generation
|
||||
],
|
||||
JobStatus.CANCELLED: [],
|
||||
|
||||
# Legacy states (for backward compatibility)
|
||||
|
||||
@@ -62,6 +62,16 @@ class TestJobStatusTransitions:
|
||||
if status not in excluded:
|
||||
valid_transitions = STATE_TRANSITIONS.get(status, [])
|
||||
assert JobStatus.FAILED in valid_transitions, f"{status} should be able to fail"
|
||||
|
||||
def test_failed_can_transition_for_retry(self):
|
||||
"""Test FAILED status can transition to retry checkpoint states."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.FAILED, [])
|
||||
|
||||
# FAILED should allow retry transitions
|
||||
assert JobStatus.INSTRUMENTAL_SELECTED in valid_transitions, "FAILED should allow retry to INSTRUMENTAL_SELECTED"
|
||||
assert JobStatus.REVIEW_COMPLETE in valid_transitions, "FAILED should allow retry to REVIEW_COMPLETE"
|
||||
assert JobStatus.LYRICS_COMPLETE in valid_transitions, "FAILED should allow retry to LYRICS_COMPLETE"
|
||||
|
||||
|
||||
class TestJobModelSerialization:
|
||||
|
||||
@@ -41,8 +41,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Loggers to capture for video worker
|
||||
# Include the full module path to properly capture KaraokeFinalise logs
|
||||
VIDEO_WORKER_LOGGERS = [
|
||||
"karaoke_gen.karaoke_finalise",
|
||||
"karaoke_gen.karaoke_finalise.karaoke_finalise", # The actual logger name from __name__
|
||||
]
|
||||
|
||||
|
||||
@@ -116,9 +118,9 @@ async def generate_video(job_id: str) -> bool:
|
||||
|
||||
# Download and set up files in the format KaraokeFinalise expects
|
||||
base_name = f"{job.artist} - {job.title}"
|
||||
job_log.info("Downloading files from GCS...")
|
||||
await _setup_working_directory(job_id, job, storage, temp_dir, base_name)
|
||||
job_log.info("Files downloaded successfully")
|
||||
job_log.info("Downloading files from GCS (this may take a few minutes for large files)...")
|
||||
await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log)
|
||||
job_log.info("All files downloaded successfully")
|
||||
|
||||
# Load style config for CDG styles
|
||||
job_log.info("Loading CDG style configuration...")
|
||||
@@ -443,7 +445,8 @@ async def _setup_working_directory(
|
||||
job,
|
||||
storage: StorageService,
|
||||
temp_dir: str,
|
||||
base_name: str
|
||||
base_name: str,
|
||||
job_log=None
|
||||
) -> None:
|
||||
"""
|
||||
Download files from GCS and set up the directory structure KaraokeFinalise expects.
|
||||
@@ -455,25 +458,34 @@ async def _setup_working_directory(
|
||||
- {base_name} (Karaoke).lrc
|
||||
- Instrumental audio file
|
||||
"""
|
||||
logger.info(f"Job {job_id}: Setting up working directory")
|
||||
def log_progress(message: str):
|
||||
"""Log to both module logger and job logger if available."""
|
||||
logger.info(f"Job {job_id}: {message}")
|
||||
if job_log:
|
||||
job_log.info(message)
|
||||
|
||||
log_progress("Setting up working directory")
|
||||
|
||||
# Download title screen
|
||||
log_progress("Downloading title screen...")
|
||||
title_url = job.file_urls['screens']['title']
|
||||
title_path = os.path.join(temp_dir, f"{base_name} (Title).mov")
|
||||
storage.download_file(title_url, title_path)
|
||||
logger.info(f"Job {job_id}: Downloaded title screen")
|
||||
log_progress("Downloaded title screen")
|
||||
|
||||
# Download end screen
|
||||
log_progress("Downloading end screen...")
|
||||
end_url = job.file_urls['screens']['end']
|
||||
end_path = os.path.join(temp_dir, f"{base_name} (End).mov")
|
||||
storage.download_file(end_url, end_path)
|
||||
logger.info(f"Job {job_id}: Downloaded end screen")
|
||||
log_progress("Downloaded end screen")
|
||||
|
||||
# Download lyrics video (with vocals)
|
||||
# Download lyrics video (with vocals) - this is the largest file, ~1-2GB
|
||||
log_progress("Downloading karaoke video (largest file, may take 1-2 minutes)...")
|
||||
lyrics_video_url = job.file_urls['videos']['with_vocals']
|
||||
lyrics_video_path = os.path.join(temp_dir, f"{base_name} (With Vocals).mov")
|
||||
storage.download_file(lyrics_video_url, lyrics_video_path)
|
||||
logger.info(f"Job {job_id}: Downloaded lyrics video")
|
||||
log_progress("Downloaded karaoke video")
|
||||
|
||||
# Download selected instrumental
|
||||
instrumental_selection = job.state_data['instrumental_selection']
|
||||
@@ -481,8 +493,9 @@ async def _setup_working_directory(
|
||||
instrumental_url = job.file_urls['stems'][instrumental_key]
|
||||
instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing"
|
||||
instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac")
|
||||
log_progress(f"Downloading {instrumental_selection} instrumental audio...")
|
||||
storage.download_file(instrumental_url, instrumental_path)
|
||||
logger.info(f"Job {job_id}: Downloaded instrumental ({instrumental_selection})")
|
||||
log_progress(f"Downloaded instrumental ({instrumental_selection})")
|
||||
|
||||
# Download LRC file if available (needed for CDG/TXT)
|
||||
lyrics_urls = job.file_urls.get('lyrics', {})
|
||||
@@ -490,19 +503,19 @@ async def _setup_working_directory(
|
||||
lrc_url = lyrics_urls['lrc']
|
||||
lrc_path = os.path.join(temp_dir, f"{base_name} (Karaoke).lrc")
|
||||
storage.download_file(lrc_url, lrc_path)
|
||||
logger.info(f"Job {job_id}: Downloaded LRC file")
|
||||
log_progress("Downloaded LRC file")
|
||||
|
||||
# Download title/end JPG files (used for YouTube thumbnail)
|
||||
screens = job.file_urls.get('screens', {})
|
||||
if screens.get('title_jpg'):
|
||||
title_jpg_path = os.path.join(temp_dir, f"{base_name} (Title).jpg")
|
||||
storage.download_file(screens['title_jpg'], title_jpg_path)
|
||||
logger.info(f"Job {job_id}: Downloaded title JPG for thumbnail")
|
||||
log_progress("Downloaded title JPG for thumbnail")
|
||||
|
||||
if screens.get('end_jpg'):
|
||||
end_jpg_path = os.path.join(temp_dir, f"{base_name} (End).jpg")
|
||||
storage.download_file(screens['end_jpg'], end_jpg_path)
|
||||
logger.info(f"Job {job_id}: Downloaded end JPG")
|
||||
log_progress("Downloaded end JPG")
|
||||
|
||||
|
||||
async def _upload_results(
|
||||
|
||||
@@ -268,6 +268,8 @@ class KaraokeFinalise:
|
||||
|
||||
# Enable discord notifications if webhook URL is provided and is valid URL
|
||||
if self.discord_webhook_url is not None:
|
||||
# Strip whitespace/newlines that may have been introduced from environment variables or secrets
|
||||
self.discord_webhook_url = self.discord_webhook_url.strip()
|
||||
if not self.discord_webhook_url.startswith("https://discord.com/api/webhooks/"):
|
||||
raise Exception(f"Discord webhook URL is not valid: {self.discord_webhook_url}")
|
||||
|
||||
@@ -858,7 +860,7 @@ class KaraokeFinalise:
|
||||
return env_mov_input, ffmpeg_filter
|
||||
|
||||
def remux_and_encode_output_video_files(self, with_vocals_file, input_files, output_files):
|
||||
self.logger.info(f"Remuxing and encoding output video files...")
|
||||
self.logger.info(f"Remuxing and encoding output video files (4 formats, ~15-20 minutes total)...")
|
||||
|
||||
# Check if output files already exist
|
||||
if os.path.isfile(output_files["final_karaoke_lossless_mp4"]) and os.path.isfile(output_files["final_karaoke_lossless_mkv"]):
|
||||
@@ -869,16 +871,20 @@ class KaraokeFinalise:
|
||||
return
|
||||
|
||||
# Create karaoke version with instrumental audio
|
||||
self.logger.info(f"[Step 1/6] Remuxing video with instrumental audio...")
|
||||
self.remux_with_instrumental(with_vocals_file, input_files["instrumental_audio"], output_files["karaoke_mp4"])
|
||||
|
||||
# Convert the with vocals video to MP4 if needed
|
||||
if not with_vocals_file.endswith(".mp4"):
|
||||
self.logger.info(f"[Step 2/6] Converting karaoke video to MP4...")
|
||||
self.convert_mov_to_mp4(with_vocals_file, output_files["with_vocals_mp4"])
|
||||
|
||||
# Delete the with vocals mov after successfully converting it to mp4
|
||||
if not self.dry_run and os.path.isfile(with_vocals_file):
|
||||
self.logger.info(f"Deleting with vocals MOV file: {with_vocals_file}")
|
||||
os.remove(with_vocals_file)
|
||||
else:
|
||||
self.logger.info(f"[Step 2/6] Skipped - video already in MP4 format")
|
||||
|
||||
# Quote file paths to handle special characters
|
||||
title_mov_file = shlex.quote(os.path.abspath(input_files["title_mov"]))
|
||||
@@ -887,10 +893,17 @@ class KaraokeFinalise:
|
||||
# Prepare concat filter for combining videos
|
||||
env_mov_input, ffmpeg_filter = self.prepare_concat_filter(input_files)
|
||||
|
||||
# Create all output versions
|
||||
# Create all output versions with progress logging
|
||||
self.logger.info(f"[Step 3/6] Encoding lossless 4K MP4 (title + karaoke + end, ~5 minutes)...")
|
||||
self.encode_lossless_mp4(title_mov_file, karaoke_mp4_file, env_mov_input, ffmpeg_filter, output_files["final_karaoke_lossless_mp4"])
|
||||
|
||||
self.logger.info(f"[Step 4/6] Encoding lossy 4K MP4 with AAC audio (~1 minute)...")
|
||||
self.encode_lossy_mp4(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossy_mp4"])
|
||||
|
||||
self.logger.info(f"[Step 5/6] Creating MKV with FLAC audio for YouTube (~1 minute)...")
|
||||
self.encode_lossless_mkv(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossless_mkv"])
|
||||
|
||||
self.logger.info(f"[Step 6/6] Encoding 720p version (~3 minutes)...")
|
||||
self.encode_720p_version(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossy_720p_mp4"])
|
||||
|
||||
# Skip user confirmation in non-interactive mode for Modal deployment
|
||||
@@ -1298,8 +1311,13 @@ class KaraokeFinalise:
|
||||
self.logger.error("YouTube upload failed in non-interactive mode, skipping")
|
||||
|
||||
# Discord notification - runs independently of YouTube upload
|
||||
# Wrapped in try/except so failures don't crash the entire job
|
||||
if self.discord_notication_enabled:
|
||||
self.post_discord_notification()
|
||||
try:
|
||||
self.post_discord_notification()
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to send Discord notification: {e}")
|
||||
self.logger.warning("Continuing without Discord notification - this is non-fatal")
|
||||
|
||||
# Handle folder organization - different logic for server-side vs local mode
|
||||
if self.server_side_mode and self.brand_prefix and self.organised_dir_rclone_root:
|
||||
@@ -1661,11 +1679,17 @@ class KaraokeFinalise:
|
||||
if self.dry_run:
|
||||
self.logger.warning("Dry run enabled. No actions will be performed.")
|
||||
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info("Starting KaraokeFinalise processing pipeline")
|
||||
self.logger.info("=" * 60)
|
||||
|
||||
# Check required input files and parameters exist, get user to confirm features before proceeding
|
||||
self.validate_input_parameters_for_features()
|
||||
|
||||
with_vocals_file = self.find_with_vocals_file()
|
||||
base_name, artist, title = self.get_names_from_withvocals(with_vocals_file)
|
||||
|
||||
self.logger.info(f"Processing: {artist} - {title}")
|
||||
|
||||
# Use the selected instrumental file if provided, otherwise search for one
|
||||
if self.selected_instrumental_file:
|
||||
@@ -1681,13 +1705,20 @@ class KaraokeFinalise:
|
||||
output_files = self.prepare_output_filenames(base_name)
|
||||
|
||||
if self.enable_cdg:
|
||||
self.logger.info("Creating CDG package...")
|
||||
self.create_cdg_zip_file(input_files, output_files, artist, title)
|
||||
self.logger.info("CDG package created successfully")
|
||||
|
||||
if self.enable_txt:
|
||||
self.logger.info("Creating TXT package...")
|
||||
self.create_txt_zip_file(input_files, output_files)
|
||||
self.logger.info("TXT package created successfully")
|
||||
|
||||
self.logger.info("Starting video encoding (this is the longest step, ~15-20 minutes)...")
|
||||
self.remux_and_encode_output_video_files(with_vocals_file, input_files, output_files)
|
||||
self.logger.info("Video encoding completed successfully")
|
||||
|
||||
self.logger.info("Executing distribution features (YouTube, Dropbox, Discord)...")
|
||||
self.execute_optional_features(artist, title, base_name, input_files, output_files, replace_existing)
|
||||
|
||||
result = {
|
||||
|
||||
@@ -90,6 +90,11 @@ def create_parser(prog: str = "karaoke-gen") -> argparse.ArgumentParser:
|
||||
metavar="JOB_ID",
|
||||
help="Cancel a running job (remote mode only). Stops processing but keeps the job record. Example: --cancel abc12345",
|
||||
)
|
||||
workflow_group.add_argument(
|
||||
"--retry",
|
||||
metavar="JOB_ID",
|
||||
help="Retry a failed job from the last successful checkpoint (remote mode only). Example: --retry abc12345",
|
||||
)
|
||||
workflow_group.add_argument(
|
||||
"--delete",
|
||||
metavar="JOB_ID",
|
||||
|
||||
@@ -8,6 +8,7 @@ Set KARAOKE_GEN_URL environment variable to your cloud backend URL.
|
||||
Usage:
|
||||
karaoke-gen-remote <filepath> <artist> <title>
|
||||
karaoke-gen-remote --resume <job_id>
|
||||
karaoke-gen-remote --retry <job_id>
|
||||
karaoke-gen-remote --list
|
||||
karaoke-gen-remote --cancel <job_id>
|
||||
karaoke-gen-remote --delete <job_id>
|
||||
@@ -383,7 +384,25 @@ class RemoteKaraokeClient:
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"Error deleting job: {response.text}")
|
||||
return response.json()
|
||||
|
||||
|
||||
def retry_job(self, job_id: str) -> Dict[str, Any]:
|
||||
"""Retry a failed job from the last successful checkpoint."""
|
||||
response = self._request(
|
||||
'POST',
|
||||
f'/api/jobs/{job_id}/retry'
|
||||
)
|
||||
if response.status_code == 404:
|
||||
raise ValueError(f"Job not found: {job_id}")
|
||||
if response.status_code == 400:
|
||||
try:
|
||||
error_detail = response.json().get('detail', response.text)
|
||||
except Exception:
|
||||
error_detail = response.text
|
||||
raise RuntimeError(f"Cannot retry job: {error_detail}")
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"Error retrying job: {response.text}")
|
||||
return response.json()
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
status: Optional[str] = None,
|
||||
@@ -591,6 +610,40 @@ class JobMonitor:
|
||||
self._last_timeline_index = 0
|
||||
self._last_log_index = 0
|
||||
self._show_worker_logs = True # Enable worker log display
|
||||
self._polls_without_updates = 0 # Track polling activity for heartbeat
|
||||
self._heartbeat_interval = 6 # Show heartbeat every N polls without updates (~30s with 5s poll)
|
||||
|
||||
# Status descriptions for user-friendly logging
|
||||
STATUS_DESCRIPTIONS = {
|
||||
'pending': 'Job queued, waiting to start',
|
||||
'downloading': 'Downloading and preparing input files',
|
||||
'separating_stage1': 'AI audio separation (stage 1 of 2)',
|
||||
'separating_stage2': 'AI audio separation (stage 2 of 2)',
|
||||
'audio_complete': 'Audio separation complete',
|
||||
'transcribing': 'Transcribing lyrics from audio',
|
||||
'correcting': 'Auto-correcting lyrics against reference sources',
|
||||
'lyrics_complete': 'Lyrics processing complete',
|
||||
'generating_screens': 'Creating title and end screens',
|
||||
'applying_padding': 'Adding intro/outro padding',
|
||||
'awaiting_review': 'Waiting for lyrics review',
|
||||
'in_review': 'Lyrics review in progress',
|
||||
'review_complete': 'Review complete, preparing video render',
|
||||
'rendering_video': 'Rendering karaoke video with lyrics',
|
||||
'awaiting_instrumental_selection': 'Waiting for instrumental selection',
|
||||
'instrumental_selected': 'Instrumental selected, preparing final encoding',
|
||||
'generating_video': 'Downloading files for final video encoding',
|
||||
'encoding': 'Encoding final videos (15-20 min, 4 formats)',
|
||||
'packaging': 'Creating CDG/TXT packages',
|
||||
'uploading': 'Uploading to distribution services',
|
||||
'notifying': 'Sending notifications',
|
||||
'complete': 'All processing complete',
|
||||
'failed': 'Job failed',
|
||||
'cancelled': 'Job cancelled',
|
||||
}
|
||||
|
||||
def _get_status_description(self, status: str) -> str:
|
||||
"""Get user-friendly description for a status."""
|
||||
return self.STATUS_DESCRIPTIONS.get(status, status)
|
||||
|
||||
def open_browser(self, url: str) -> None:
|
||||
"""Open URL in the default browser."""
|
||||
@@ -1082,6 +1135,7 @@ class JobMonitor:
|
||||
|
||||
self.logger.info(f"Monitoring job: {job_id}")
|
||||
self.logger.info(f"Service URL: {self.config.service_url}")
|
||||
self.logger.info(f"Polling every {self.config.poll_interval} seconds...")
|
||||
self.logger.info("")
|
||||
|
||||
while True:
|
||||
@@ -1092,16 +1146,40 @@ class JobMonitor:
|
||||
artist = job_data.get('artist', '')
|
||||
title = job_data.get('title', '')
|
||||
|
||||
# Track whether we got any new updates this poll
|
||||
had_updates = False
|
||||
prev_timeline_index = self._last_timeline_index
|
||||
prev_log_index = self._last_log_index
|
||||
|
||||
# Log timeline updates (shows status changes and progress)
|
||||
self.log_timeline_updates(job_data)
|
||||
if self._last_timeline_index > prev_timeline_index:
|
||||
had_updates = True
|
||||
|
||||
# Log worker logs (shows detailed worker output for debugging)
|
||||
self.log_worker_logs(job_id)
|
||||
if self._last_log_index > prev_log_index:
|
||||
had_updates = True
|
||||
|
||||
# Log status changes
|
||||
# Log status changes with user-friendly descriptions
|
||||
if status != last_status:
|
||||
self.logger.info(f"Status changed: {last_status} -> {status}")
|
||||
description = self._get_status_description(status)
|
||||
if last_status:
|
||||
self.logger.info(f"Status: {status} - {description}")
|
||||
else:
|
||||
self.logger.info(f"Current status: {status} - {description}")
|
||||
last_status = status
|
||||
had_updates = True
|
||||
|
||||
# Heartbeat: if no updates for a while, show we're still alive
|
||||
if had_updates:
|
||||
self._polls_without_updates = 0
|
||||
else:
|
||||
self._polls_without_updates += 1
|
||||
if self._polls_without_updates >= self._heartbeat_interval:
|
||||
description = self._get_status_description(status)
|
||||
self.logger.info(f" [Still processing: {description}]")
|
||||
self._polls_without_updates = 0
|
||||
|
||||
# Handle human interaction points
|
||||
if status in ['awaiting_review', 'in_review']:
|
||||
@@ -1388,6 +1466,7 @@ def main():
|
||||
logger.info("")
|
||||
logger.info(f"Total: {len(jobs)} jobs")
|
||||
logger.info("")
|
||||
logger.info("To retry a failed job: karaoke-gen-remote --retry <JOB_ID>")
|
||||
logger.info("To delete a job: karaoke-gen-remote --delete <JOB_ID>")
|
||||
logger.info("To bulk delete: karaoke-gen-remote --bulk-delete --filter-environment=test")
|
||||
logger.info("To cancel a job: karaoke-gen-remote --cancel <JOB_ID>")
|
||||
@@ -1403,24 +1482,24 @@ def main():
|
||||
logger.info("Karaoke Generator (Remote) - Cancel Job")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Job ID: {args.cancel}")
|
||||
|
||||
|
||||
try:
|
||||
# Get job info first
|
||||
job_data = client.get_job(args.cancel)
|
||||
artist = job_data.get('artist', 'Unknown')
|
||||
title = job_data.get('title', 'Unknown')
|
||||
status = job_data.get('status', 'unknown')
|
||||
|
||||
|
||||
logger.info(f"Artist: {artist}")
|
||||
logger.info(f"Title: {title}")
|
||||
logger.info(f"Current status: {status}")
|
||||
logger.info("")
|
||||
|
||||
|
||||
# Cancel the job
|
||||
result = client.cancel_job(args.cancel)
|
||||
logger.info(f"✓ Job cancelled successfully")
|
||||
return 0
|
||||
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(str(e))
|
||||
return 1
|
||||
@@ -1430,7 +1509,54 @@ def main():
|
||||
except Exception as e:
|
||||
logger.error(f"Error cancelling job: {e}")
|
||||
return 1
|
||||
|
||||
|
||||
# Handle retry job mode
|
||||
if args.retry:
|
||||
logger.info("=" * 60)
|
||||
logger.info("Karaoke Generator (Remote) - Retry Failed Job")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Job ID: {args.retry}")
|
||||
|
||||
try:
|
||||
# Get job info first
|
||||
job_data = client.get_job(args.retry)
|
||||
artist = job_data.get('artist', 'Unknown')
|
||||
title = job_data.get('title', 'Unknown')
|
||||
status = job_data.get('status', 'unknown')
|
||||
error_message = job_data.get('error_message', 'No error message')
|
||||
|
||||
logger.info(f"Artist: {artist}")
|
||||
logger.info(f"Title: {title}")
|
||||
logger.info(f"Current status: {status}")
|
||||
if status == 'failed':
|
||||
logger.info(f"Error: {error_message}")
|
||||
logger.info("")
|
||||
|
||||
if status != 'failed':
|
||||
logger.error(f"Only failed jobs can be retried (current status: {status})")
|
||||
return 1
|
||||
|
||||
# Retry the job
|
||||
result = client.retry_job(args.retry)
|
||||
retry_stage = result.get('retry_stage', 'unknown')
|
||||
logger.info(f"✓ Job retry started from stage: {retry_stage}")
|
||||
logger.info("")
|
||||
logger.info(f"Monitoring job progress...")
|
||||
logger.info("")
|
||||
|
||||
# Monitor the retried job
|
||||
return monitor.monitor(args.retry)
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(str(e))
|
||||
return 1
|
||||
except RuntimeError as e:
|
||||
logger.error(str(e))
|
||||
return 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrying job: {e}")
|
||||
return 1
|
||||
|
||||
# Handle delete job mode
|
||||
if args.delete:
|
||||
logger.info("=" * 60)
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.71.16"
|
||||
version = "0.71.19"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
+1
-1
@@ -112,7 +112,7 @@ echo -e "${YELLOW}---BEGIN SERVICE ACCOUNT KEY---${NC}"
|
||||
cat "$KEY_FILE"
|
||||
echo -e "${YELLOW}---END SERVICE ACCOUNT KEY---${NC}"
|
||||
echo ""
|
||||
echo "3. After adding secrets, push to 'replace-modal-with-google-cloud' branch"
|
||||
echo "3. After adding secrets, push to 'main' branch"
|
||||
echo " to trigger an automated deployment!"
|
||||
echo ""
|
||||
echo -e "${RED}⚠️ IMPORTANT SECURITY NOTES:${NC}"
|
||||
|
||||
@@ -47,6 +47,42 @@ def mock_gmail_service():
|
||||
|
||||
# --- Discord Notification Tests ---
|
||||
|
||||
def test_discord_webhook_url_strips_whitespace(mock_logger):
|
||||
"""Test that Discord webhook URL validation strips trailing whitespace/newlines."""
|
||||
# This tests the fix for the %0A issue where webhook URLs have trailing newlines
|
||||
config = MINIMAL_CONFIG.copy()
|
||||
config.update({
|
||||
# URL with trailing newline (simulating env var issue)
|
||||
"discord_webhook_url": "https://discord.com/api/webhooks/123/abc\n",
|
||||
})
|
||||
with patch.object(KaraokeFinalise, 'detect_best_aac_codec', return_value='aac'):
|
||||
finaliser = KaraokeFinalise(logger=mock_logger, **config)
|
||||
|
||||
# Call validate_input_parameters_for_features to trigger the strip
|
||||
with patch('os.getcwd', return_value='/tmp'):
|
||||
finaliser.validate_input_parameters_for_features()
|
||||
|
||||
# The URL should have been stripped
|
||||
assert finaliser.discord_webhook_url == "https://discord.com/api/webhooks/123/abc"
|
||||
assert finaliser.discord_notication_enabled is True
|
||||
|
||||
|
||||
def test_discord_webhook_url_strips_whitespace_and_validates(mock_logger):
|
||||
"""Test that invalid Discord URLs are rejected even after stripping."""
|
||||
config = MINIMAL_CONFIG.copy()
|
||||
config.update({
|
||||
# Invalid URL that starts correctly but is malformed
|
||||
"discord_webhook_url": "https://discord.com/invalid\n",
|
||||
})
|
||||
with patch.object(KaraokeFinalise, 'detect_best_aac_codec', return_value='aac'):
|
||||
finaliser = KaraokeFinalise(logger=mock_logger, **config)
|
||||
|
||||
# Should raise because URL doesn't start with correct prefix after strip
|
||||
with patch('os.getcwd', return_value='/tmp'):
|
||||
with pytest.raises(Exception, match="Discord webhook URL is not valid"):
|
||||
finaliser.validate_input_parameters_for_features()
|
||||
|
||||
|
||||
@patch('requests.post')
|
||||
def test_post_discord_message_success(mock_post, finaliser_for_notify):
|
||||
"""Test successful posting to Discord."""
|
||||
|
||||
@@ -171,6 +171,40 @@ def test_execute_optional_features_youtube_fails(mock_input, mock_discord, mock_
|
||||
assert finaliser_for_process.youtube_url == f"https://www.youtube.com/watch?v=manual_video_id"
|
||||
mock_discord.assert_called_once() # Discord should still be called after manual ID entry
|
||||
|
||||
|
||||
@patch.object(KaraokeFinalise, 'upload_final_mp4_to_youtube_with_title_thumbnail')
|
||||
@patch.object(KaraokeFinalise, 'post_discord_notification')
|
||||
@patch.object(KaraokeFinalise, 'get_next_brand_code', return_value=f"{BRAND_PREFIX}-0001")
|
||||
@patch.object(KaraokeFinalise, 'move_files_to_brand_code_folder')
|
||||
@patch.object(KaraokeFinalise, 'copy_final_files_to_public_share_dirs')
|
||||
@patch.object(KaraokeFinalise, 'sync_public_share_dir_to_rclone_destination')
|
||||
@patch.object(KaraokeFinalise, 'generate_organised_folder_sharing_link')
|
||||
def test_execute_optional_features_discord_failure_continues(
|
||||
mock_gen_link, mock_sync, mock_copy, mock_move, mock_get_next,
|
||||
mock_discord, mock_youtube, finaliser_for_process):
|
||||
"""Test that Discord notification failure doesn't crash the job."""
|
||||
# Simulate Discord notification failure
|
||||
mock_discord.side_effect = Exception("401 Unauthorized: Invalid webhook URL")
|
||||
finaliser_for_process.keep_brand_code = False
|
||||
finaliser_for_process.new_brand_code_dir_path = os.path.join(ORGANISED_DIR, f"{BRAND_PREFIX}-0001 - {ARTIST} - {TITLE}")
|
||||
|
||||
# Should not raise an exception - should continue with other features
|
||||
finaliser_for_process.execute_optional_features(ARTIST, TITLE, BASE_NAME, ALL_INPUT_FILES, ALL_OUTPUT_FILES, False)
|
||||
|
||||
# Discord was called (and failed)
|
||||
mock_discord.assert_called_once()
|
||||
# But other features should still run
|
||||
mock_youtube.assert_called_once()
|
||||
mock_get_next.assert_called_once()
|
||||
mock_move.assert_called_once()
|
||||
mock_copy.assert_called_once()
|
||||
mock_sync.assert_called_once()
|
||||
mock_gen_link.assert_called_once()
|
||||
# Logger should have logged the error (warning)
|
||||
finaliser_for_process.logger.error.assert_called()
|
||||
finaliser_for_process.logger.warning.assert_called()
|
||||
|
||||
|
||||
# --- process Method Tests ---
|
||||
|
||||
@patch.object(KaraokeFinalise, 'validate_input_parameters_for_features')
|
||||
|
||||
+251
-17
@@ -426,10 +426,51 @@ class TestRemoteKaraokeClient:
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 404
|
||||
mock_request.return_value = mock_response
|
||||
|
||||
|
||||
with pytest.raises(ValueError, match="Job not found"):
|
||||
client.delete_job("nonexistent-job")
|
||||
|
||||
|
||||
@patch.object(RemoteKaraokeClient, '_request')
|
||||
def test_retry_job_success(self, mock_request, client):
|
||||
"""Test retrying a failed job successfully."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"status": "success",
|
||||
"job_status": "instrumental_selected",
|
||||
"retry_stage": "video_generation"
|
||||
}
|
||||
mock_request.return_value = mock_response
|
||||
|
||||
result = client.retry_job("job-123")
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["retry_stage"] == "video_generation"
|
||||
mock_request.assert_called_once_with('POST', '/api/jobs/job-123/retry')
|
||||
|
||||
@patch.object(RemoteKaraokeClient, '_request')
|
||||
def test_retry_job_not_found(self, mock_request, client):
|
||||
"""Test retrying non-existent job."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 404
|
||||
mock_request.return_value = mock_response
|
||||
|
||||
with pytest.raises(ValueError, match="Job not found"):
|
||||
client.retry_job("nonexistent-job")
|
||||
|
||||
@patch.object(RemoteKaraokeClient, '_request')
|
||||
def test_retry_job_not_failed(self, mock_request, client):
|
||||
"""Test retrying a job that isn't in failed state."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 400
|
||||
mock_response.json.return_value = {
|
||||
"detail": "Only failed jobs can be retried"
|
||||
}
|
||||
mock_request.return_value = mock_response
|
||||
|
||||
with pytest.raises(RuntimeError, match="Cannot retry job"):
|
||||
client.retry_job("job-123")
|
||||
|
||||
@patch.object(RemoteKaraokeClient, '_request')
|
||||
def test_get_instrumental_options(self, mock_request, client):
|
||||
"""Test getting instrumental options."""
|
||||
@@ -779,6 +820,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -788,11 +830,11 @@ class TestMain:
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
mock_auth.return_value = "token"
|
||||
|
||||
|
||||
result = main()
|
||||
|
||||
|
||||
assert result == 1
|
||||
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.get_auth_token')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@@ -810,6 +852,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -819,11 +862,11 @@ class TestMain:
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
mock_auth.return_value = "token"
|
||||
|
||||
|
||||
result = main()
|
||||
|
||||
|
||||
assert result == 1
|
||||
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.get_auth_token')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@@ -841,6 +884,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -850,11 +894,11 @@ class TestMain:
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
mock_auth.return_value = "token"
|
||||
|
||||
|
||||
result = main()
|
||||
|
||||
|
||||
assert result == 1
|
||||
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.get_auth_token')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@@ -872,6 +916,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None # Added for retry feature
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False # New attribute for bulk delete mode
|
||||
mock_args.filter_environment = None # New attribute for filtering
|
||||
@@ -880,9 +925,9 @@ class TestMain:
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
mock_auth.return_value = "token"
|
||||
|
||||
|
||||
result = main()
|
||||
|
||||
|
||||
assert result == 1
|
||||
mock_parser.print_help.assert_called_once()
|
||||
|
||||
@@ -952,6 +997,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = True
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False # New attribute for bulk delete mode
|
||||
mock_args.filter_environment = None # New attribute for filtering
|
||||
@@ -960,7 +1006,7 @@ class TestMain:
|
||||
mock_args.client_id = "" # Job tracking
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.list_jobs.return_value = [
|
||||
{"job_id": "job-1", "status": "complete", "artist": "Artist 1", "title": "Title 1", "request_metadata": {}},
|
||||
@@ -993,6 +1039,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = True
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1001,7 +1048,7 @@ class TestMain:
|
||||
mock_args.client_id = ""
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.list_jobs.return_value = []
|
||||
mock_client_class.return_value = mock_client
|
||||
@@ -1052,7 +1099,187 @@ class TestMain:
|
||||
|
||||
assert result == 0
|
||||
mock_client.cancel_job.assert_called_once_with("job-123")
|
||||
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.JobMonitor')
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
|
||||
def test_main_retry_job_mode(
|
||||
self, mock_client_class, mock_check, mock_create_parser, mock_monitor_class
|
||||
):
|
||||
"""Test main in retry job mode with a failed job."""
|
||||
mock_parser = MagicMock()
|
||||
mock_args = MagicMock()
|
||||
mock_args.service_url = "https://api.example.com"
|
||||
mock_args.review_ui_url = "https://review.example.com"
|
||||
mock_args.poll_interval = 5
|
||||
mock_args.output_dir = "/tmp"
|
||||
mock_args.log_level = "info"
|
||||
mock_args.finalise_only = False
|
||||
mock_args.edit_lyrics = False
|
||||
mock_args.test_email_template = False
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = "job-123"
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
mock_args.filter_client_id = None
|
||||
mock_args.environment = ""
|
||||
mock_args.client_id = ""
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_job.return_value = {
|
||||
"artist": "Artist",
|
||||
"title": "Title",
|
||||
"status": "failed",
|
||||
"error_message": "Discord webhook failed"
|
||||
}
|
||||
mock_client.retry_job.return_value = {
|
||||
"status": "success",
|
||||
"retry_stage": "video_generation"
|
||||
}
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
mock_monitor = MagicMock()
|
||||
mock_monitor.monitor.return_value = 0
|
||||
mock_monitor_class.return_value = mock_monitor
|
||||
|
||||
result = main()
|
||||
|
||||
assert result == 0
|
||||
mock_client.retry_job.assert_called_once_with("job-123")
|
||||
mock_monitor.monitor.assert_called_once_with("job-123")
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
|
||||
def test_main_retry_job_not_failed(
|
||||
self, mock_client_class, mock_check, mock_create_parser
|
||||
):
|
||||
"""Test main retry job when job is not in failed state."""
|
||||
mock_parser = MagicMock()
|
||||
mock_args = MagicMock()
|
||||
mock_args.service_url = "https://api.example.com"
|
||||
mock_args.review_ui_url = "https://review.example.com"
|
||||
mock_args.poll_interval = 5
|
||||
mock_args.output_dir = "/tmp"
|
||||
mock_args.log_level = "info"
|
||||
mock_args.finalise_only = False
|
||||
mock_args.edit_lyrics = False
|
||||
mock_args.test_email_template = False
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = "job-123"
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
mock_args.filter_client_id = None
|
||||
mock_args.environment = ""
|
||||
mock_args.client_id = ""
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_job.return_value = {
|
||||
"artist": "Artist",
|
||||
"title": "Title",
|
||||
"status": "complete" # Not failed
|
||||
}
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = main()
|
||||
|
||||
assert result == 1 # Should fail because job is not in failed state
|
||||
mock_client.retry_job.assert_not_called()
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
|
||||
def test_main_retry_job_not_found(
|
||||
self, mock_client_class, mock_check, mock_create_parser
|
||||
):
|
||||
"""Test main retry job when job not found."""
|
||||
mock_parser = MagicMock()
|
||||
mock_args = MagicMock()
|
||||
mock_args.service_url = "https://api.example.com"
|
||||
mock_args.review_ui_url = "https://review.example.com"
|
||||
mock_args.poll_interval = 5
|
||||
mock_args.output_dir = "/tmp"
|
||||
mock_args.log_level = "info"
|
||||
mock_args.finalise_only = False
|
||||
mock_args.edit_lyrics = False
|
||||
mock_args.test_email_template = False
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = "nonexistent-job"
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
mock_args.filter_client_id = None
|
||||
mock_args.environment = ""
|
||||
mock_args.client_id = ""
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_job.side_effect = ValueError("Job not found")
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = main()
|
||||
|
||||
assert result == 1
|
||||
mock_client.retry_job.assert_not_called()
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
|
||||
def test_main_retry_job_api_error(
|
||||
self, mock_client_class, mock_check, mock_create_parser
|
||||
):
|
||||
"""Test main retry job when API returns an error."""
|
||||
mock_parser = MagicMock()
|
||||
mock_args = MagicMock()
|
||||
mock_args.service_url = "https://api.example.com"
|
||||
mock_args.review_ui_url = "https://review.example.com"
|
||||
mock_args.poll_interval = 5
|
||||
mock_args.output_dir = "/tmp"
|
||||
mock_args.log_level = "info"
|
||||
mock_args.finalise_only = False
|
||||
mock_args.edit_lyrics = False
|
||||
mock_args.test_email_template = False
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = "job-123"
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
mock_args.filter_client_id = None
|
||||
mock_args.environment = ""
|
||||
mock_args.client_id = ""
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_job.return_value = {
|
||||
"artist": "Artist",
|
||||
"title": "Title",
|
||||
"status": "failed",
|
||||
"error_message": "Some error"
|
||||
}
|
||||
mock_client.retry_job.side_effect = RuntimeError("Cannot retry job: No safe checkpoint found")
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = main()
|
||||
|
||||
assert result == 1
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.create_parser')
|
||||
@patch('karaoke_gen.utils.remote_cli.check_prerequisites')
|
||||
@patch('karaoke_gen.utils.remote_cli.RemoteKaraokeClient')
|
||||
@@ -1073,6 +1300,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = "nonexistent"
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1111,6 +1339,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = "job-123"
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1156,6 +1385,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = "job-123"
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1199,6 +1429,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = "job-123"
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1208,7 +1439,7 @@ class TestMain:
|
||||
mock_args.yes = True # Non-interactive mode
|
||||
mock_parser.parse_args.return_value = mock_args
|
||||
mock_create_parser.return_value = mock_parser
|
||||
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_job.return_value = {
|
||||
"artist": "Artist",
|
||||
@@ -1245,6 +1476,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1288,6 +1520,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
@@ -1332,6 +1565,7 @@ class TestMain:
|
||||
mock_args.resume = None
|
||||
mock_args.list_jobs = False
|
||||
mock_args.cancel = None
|
||||
mock_args.retry = None
|
||||
mock_args.delete = None
|
||||
mock_args.bulk_delete = False
|
||||
mock_args.filter_environment = None
|
||||
|
||||
Reference in new issue
Block a user