mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Problem
Re-rendering Vocal Star job `5fca18dd` (created 2026-08-25) failed in
the encoder:
`404 ... uploads/5fca18dd/audio/S1106-1 Ruby Murray - Softly Softly
Guide.mp3: No such object`.
Jobs created before v0.268.2 kept their uploaded mix and bring-your-own
instrumental only under `uploads/`, which had a 7-day lifecycle delete.
Every pre-October tenant job would fail the same way on re-render (incl.
the theme editor's "Re-render all").
## Fix
New `backend/services/input_recovery.recover_missing_inputs`, called by
the render worker (both GCE + local paths, off the event loop) and the
video worker, right after `ensure_job_inputs_persisted`:
- **Mix** (`input_media_gcs_path`) missing → repoint to the persisted
`jobs/{id}/input/` copy; else reuse a previous recovery; else rebuild
from the job's with-vocals render (`finals/with_vocals_mp4.mp4`,
`videos/with_vocals.*`): its audio track is the original mix (AAC).
Countdown padding (from `state_data.lyrics_metadata` /
`processing_metadata.rendering`) is trimmed; decoded to
`jobs/{id}/input/recovered_mix.flac` (input/ is keep-forever). Tries the
next video if one can't be decoded.
- **Instrumental** (`existing_instrumental_gcs_path`) missing →
persisted copy, else the staged `existing_instrumental.*` /
`custom_instrumental.*` at the job root or in `stems/` (kept by storage
retention).
- Nothing left → the job fails with an actionable message instead of a
raw 404. Recovery is recorded in `state_data.input_recovery`.
Verified on the real job's files: `with_vocals_mp4.mp4` audio = 148.69s,
same as `custom_instrumental.mp3`, same 0.61s leading silence (no
padding on this job).
## Tests
- `backend/tests/test_input_recovery.py` (17): no-op, persisted copy,
real ffmpeg rebuild, countdown trim, padding read from all known
locations, reuse of previous recovery, fall-through on an unreadable
video, actionable errors, instrumental repoint (4 locations), gs://
paths.
- Render-worker tests whose storage mocks report every blob missing stub
the recovery (autouse fixture).
- Full backend suite: 6083 passed.
- CodeRabbit: 3 findings fixed (padding location, to_thread,
fall-through);
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_01Gp5fBmsmWJxgkEHb4JfhXR
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1036 lines
56 KiB
Python
1036 lines
56 KiB
Python
"""
|
||
Render Video Worker
|
||
|
||
Generates the karaoke video with synchronized lyrics AFTER human review.
|
||
|
||
This worker:
|
||
1. Downloads the corrected lyrics data from GCS
|
||
2. Downloads the audio file
|
||
3. Downloads style assets from GCS
|
||
4. Uses LyricsTranscriber's OutputGenerator to render video
|
||
5. Uploads the with_vocals.mkv to GCS
|
||
6. Transitions to INSTRUMENTAL_SELECTED (user already selected during combined review)
|
||
7. Triggers video worker for final encoding
|
||
|
||
Note: Instrumental selection now happens during the combined lyrics + instrumental
|
||
review phase (before this worker runs). The selection is stored in
|
||
state_data['instrumental_selection'] by the review completion endpoint.
|
||
|
||
Key insight: We use OutputGenerator from lyrics_transcriber library
|
||
WITHOUT using its blocking ReviewServer. This allows async operation
|
||
in Cloud Run.
|
||
|
||
Observability:
|
||
- All operations wrapped in tracing spans for Cloud Trace visibility
|
||
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
||
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
||
"""
|
||
import asyncio
|
||
import logging
|
||
import os
|
||
import tempfile
|
||
import time
|
||
import json
|
||
from datetime import datetime, UTC
|
||
from typing import Optional, Dict, Any
|
||
from dataclasses import dataclass
|
||
|
||
from backend.models.job import JobStatus
|
||
from backend.exceptions import InvalidStateTransitionError
|
||
from backend.utils.audio_filenames import local_audio_filename
|
||
from backend.services.job_manager import JobManager
|
||
from backend.services.input_persistence import ensure_job_inputs_persisted
|
||
from backend.services.input_recovery import recover_missing_inputs
|
||
from backend.services.storage_service import StorageService
|
||
from backend.services.lyrics_translation import prepare_job_translations
|
||
from karaoke_gen.lyrics_transcriber.output.translations import apply_translations
|
||
from backend.services.job_health_service import validate_worker_can_run
|
||
from backend.workers.supersede import capture_generation, check_superseded, encoding_worker_job_id
|
||
from backend.config import get_settings
|
||
from backend.workers.registry import worker_registry
|
||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||
from backend.services.encoding_service import get_encoding_service, run_with_lost_job_resubmit
|
||
from backend.services.encoding_errors import (
|
||
EncodingWorkerCapacityError,
|
||
EncodingWorkerStartError,
|
||
)
|
||
|
||
|
||
# Sentinel exception code used when the render worker is parked because Cloud
|
||
# Run is shutting the container down mid-render. Distinct from real GCE
|
||
# capacity codes so log filters can tell them apart.
|
||
RENDER_WORKER_SHUTDOWN_CODE = "WORKER_SHUTDOWN"
|
||
|
||
# Import from lyrics_transcriber (submodule)
|
||
from karaoke_gen.lyrics_transcriber.output.generator import OutputGenerator
|
||
from karaoke_gen.lyrics_transcriber.output.countdown_processor import CountdownProcessor
|
||
from karaoke_gen.lyrics_transcriber.output.timing_validation import (
|
||
LyricsTimingError,
|
||
validate_segment_timing,
|
||
)
|
||
from karaoke_gen.lyrics_transcriber.types import CorrectionResult
|
||
from karaoke_gen.lyrics_transcriber.correction.operations import CorrectionOperations
|
||
from karaoke_gen.lyrics_transcriber.core.config import OutputConfig
|
||
|
||
# Import from the unified style loader
|
||
from karaoke_gen.style_loader import load_styles_from_gcs
|
||
from karaoke_gen.utils import sanitize_filename
|
||
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# Loggers to capture for render video worker
|
||
RENDER_VIDEO_WORKER_LOGGERS = [
|
||
"karaoke_gen.lyrics_transcriber.output",
|
||
"karaoke_gen.lyrics_transcriber.output.generator",
|
||
"karaoke_gen.lyrics_transcriber.output.video",
|
||
"karaoke_gen.lyrics_transcriber.output.ass",
|
||
]
|
||
|
||
|
||
async def process_render_video(job_id: str) -> bool:
|
||
"""
|
||
Render karaoke video with corrected lyrics.
|
||
|
||
Called after human review is complete (REVIEW_COMPLETE state).
|
||
Uses OutputGenerator from lyrics_transcriber to generate the video.
|
||
|
||
Args:
|
||
job_id: Job ID to process
|
||
|
||
Returns:
|
||
True if successful, False otherwise
|
||
"""
|
||
start_time = time.time()
|
||
job_manager = JobManager()
|
||
storage = StorageService()
|
||
settings = get_settings()
|
||
|
||
# Create job logger for remote debugging FIRST
|
||
job_log = create_job_logger(job_id, "render_video")
|
||
|
||
# Log with structured markers for easy Cloud Logging queries
|
||
logger.info(f"[job:{job_id}] WORKER_START worker=render-video")
|
||
job_log.info("=== RENDER VIDEO WORKER STARTED ===")
|
||
job_log.info(f"Job ID: {job_id}")
|
||
|
||
# Register with worker registry so the FastAPI shutdown hook can wait for
|
||
# this worker AND, if Cloud Run forces termination, park the job in
|
||
# RENDER_PENDING_CAPACITY for the auto-retry scheduler to recover.
|
||
await worker_registry.register(job_id, "render-video")
|
||
|
||
# Set up log capture for OutputGenerator
|
||
log_handler = setup_job_logging(job_id, "render_video", *RENDER_VIDEO_WORKER_LOGGERS)
|
||
job_log.info(f"Log handler attached for {len(RENDER_VIDEO_WORKER_LOGGERS)} loggers")
|
||
|
||
job = job_manager.get_job(job_id)
|
||
if not job:
|
||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||
await worker_registry.unregister(job_id, "render-video")
|
||
return False
|
||
|
||
# Copy uploads/{job_id}/** into jobs/{job_id}/input/ (uploads/ expires) and repoint the job.
|
||
await asyncio.to_thread(ensure_job_inputs_persisted, job, storage, job_manager, "render_video")
|
||
|
||
# Hard idempotency gate (worker-side). The legacy HTTP endpoint enforces
|
||
# this via _check_worker_idempotency at dispatch time, but the Cloud Run
|
||
# Job path invokes this worker directly — and Cloud Run may re-run the
|
||
# execution (max_retries=2) after an OOM/timeout/hard kill. The worker
|
||
# must therefore protect itself regardless of dispatch path:
|
||
# - terminal job: never resurrect. A stale or duplicate trigger racing
|
||
# a job that just finished would otherwise re-render and drag a
|
||
# complete job back to instrumental_selected.
|
||
# - already RENDERING_VIDEO: either another execution owns the render
|
||
# (duplicate trigger) or a previous execution died mid-render (Cloud
|
||
# Run Job retry). Never re-enter: a live render must not be disrupted,
|
||
# and a dead one is recovered by the rendering_video_stuck sweep →
|
||
# re-park → auto-retry. Re-entering would hit the same-state
|
||
# transition guard and hard-fail the job with a spurious
|
||
# "Invalid state transition" error.
|
||
# (Legacy path unaffected: the worker enters with status
|
||
# REVIEW_COMPLETE — RENDERING_VIDEO is only set below, by this worker.)
|
||
terminal_statuses = {"failed", "cancelled", "complete", "prep_complete", "error"}
|
||
status_value = job.status.value if hasattr(job.status, "value") else str(job.status)
|
||
if status_value in terminal_statuses:
|
||
logger.info(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=skipped "
|
||
f"reason=terminal_state:{status_value}"
|
||
)
|
||
job_log.info(f"Job in terminal state '{status_value}' — skipping render")
|
||
await worker_registry.unregister(job_id, "render-video")
|
||
return False
|
||
if job.status == JobStatus.RENDERING_VIDEO:
|
||
logger.warning(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=skipped "
|
||
f"reason=already_rendering"
|
||
)
|
||
job_log.warning(
|
||
"Job already in rendering_video — another execution owns this render "
|
||
"(or a previous one died and the rendering_video_stuck sweep will "
|
||
"recover it); not re-entering"
|
||
)
|
||
await worker_registry.unregister(job_id, "render-video")
|
||
return False
|
||
|
||
# The tenant edited its theme after this job's screens were made: regenerate
|
||
# them first (the screens worker re-triggers this worker when done).
|
||
from backend.services.theme_change_service import divert_for_stale_screens
|
||
try:
|
||
diverted = await divert_for_stale_screens(job, job_manager, storage)
|
||
except Exception as e:
|
||
logger.error(f"[job:{job_id}] WORKER_END worker=render-video status=error error={e}")
|
||
job_manager.mark_job_failed(
|
||
job_id=job_id,
|
||
error_message=f"Screen regeneration for the updated theme failed: {e}",
|
||
error_details={"stage": "render_video", "error": str(e)},
|
||
)
|
||
await worker_registry.unregister(job_id, "render-video")
|
||
return False
|
||
if diverted:
|
||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=skipped reason=theme_screens_regen")
|
||
await worker_registry.unregister(job_id, "render-video")
|
||
return True
|
||
|
||
# Capture the supersession fence at start. If an admin reset (or a newer
|
||
# render trigger) advances the generation while we render, our result is
|
||
# stale and must be discarded instead of overwriting fresh outputs or
|
||
# failing the job. See backend/workers/supersede.py.
|
||
captured_generation = capture_generation(job)
|
||
|
||
# Validate job status is appropriate for render video worker
|
||
# This helps catch bugs where the worker is triggered incorrectly
|
||
status_error = validate_worker_can_run("render_video_worker", job)
|
||
if status_error:
|
||
logger.warning(f"[job:{job_id}] {status_error}")
|
||
job_log.warning(status_error)
|
||
# Continue anyway - this is a safety net warning, not a hard failure
|
||
|
||
job_log.info(f"Starting video render for {job.artist} - {job.title}")
|
||
logger.info(f"[job:{job_id}] Starting video render (post-review)")
|
||
|
||
try:
|
||
# Wrap entire worker in a tracing span
|
||
with job_span("render-video-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
|
||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||
# This ensures logs from third-party libraries (lyrics_transcriber.output) are only
|
||
# captured by this job's handler, not handlers from other concurrent jobs
|
||
with job_logging_context(job_id):
|
||
# Check if GCE encoding is enabled
|
||
encoding_service = get_encoding_service()
|
||
use_gce = encoding_service.is_enabled
|
||
|
||
async def prepare_translations() -> Optional[str]:
|
||
"""Translate the final (reviewed) lyrics once, after the job is in
|
||
RENDERING_VIDEO. Never blocks the render (failure = no translations)."""
|
||
try:
|
||
path = await asyncio.to_thread(
|
||
prepare_job_translations, job_id, job, storage, job_manager
|
||
)
|
||
except Exception as exc:
|
||
logger.error(f"[job:{job_id}] Translated lyrics skipped: {exc}", exc_info=True)
|
||
return None
|
||
if path:
|
||
job_log.info(f"Rendering with translated lyrics ({job.translation_language})")
|
||
return path
|
||
|
||
if use_gce:
|
||
# ============ GCE RENDER VIDEO PATH ============
|
||
job_log.info("GCE encoding enabled - delegating render_video to encoding worker")
|
||
logger.info(f"[job:{job_id}] Using GCE encoding worker for render_video")
|
||
|
||
# Transition to RENDERING_VIDEO
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.RENDERING_VIDEO,
|
||
progress=75,
|
||
message="Rendering karaoke video on encoding worker"
|
||
)
|
||
translations_gcs = await prepare_translations()
|
||
|
||
# Old jobs lost their uploaded mix (uploads/ expired): rebuild it first,
|
||
# or the encoder 404s on input_media_gcs_path.
|
||
await asyncio.to_thread(recover_missing_inputs, job, storage, job_manager)
|
||
|
||
# Build render config from job metadata (no downloads needed on Cloud Run!)
|
||
bucket_name = settings.gcs_bucket_name
|
||
|
||
# Check for updated corrections
|
||
updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json"
|
||
has_updated = storage.file_exists(updated_corrections_gcs)
|
||
|
||
# Read is_duet from state_data (set by review completion endpoint)
|
||
is_duet = bool(job.state_data.get("is_duet", False))
|
||
|
||
render_config = {
|
||
"original_corrections_gcs_path": f"gs://{bucket_name}/jobs/{job_id}/lyrics/corrections.json",
|
||
"audio_gcs_path": f"gs://{bucket_name}/{job.input_media_gcs_path}",
|
||
"output_gcs_prefix": f"gs://{bucket_name}/jobs/{job_id}/",
|
||
"artist": job.artist,
|
||
"title": job.title,
|
||
"subtitle_offset_ms": getattr(job, 'subtitle_offset_ms', 0) or 0,
|
||
"video_resolution": "4k",
|
||
"is_duet": is_duet,
|
||
}
|
||
|
||
if translations_gcs:
|
||
render_config["translations_gcs_path"] = f"gs://{bucket_name}/{translations_gcs}"
|
||
|
||
if has_updated:
|
||
render_config["updated_corrections_gcs_path"] = f"gs://{bucket_name}/{updated_corrections_gcs}"
|
||
|
||
if job.style_params_gcs_path:
|
||
render_config["style_params_gcs_path"] = f"gs://{bucket_name}/{job.style_params_gcs_path}" if not job.style_params_gcs_path.startswith("gs://") else job.style_params_gcs_path
|
||
|
||
if job.style_assets:
|
||
render_config["style_assets"] = {
|
||
k: f"gs://{bucket_name}/{v}" if not v.startswith("gs://") else v
|
||
for k, v in job.style_assets.items()
|
||
}
|
||
|
||
capacity_marker_cleared = [False]
|
||
|
||
def progress_callback(progress: int):
|
||
# Update progress in place — the job is already in
|
||
# RENDERING_VIDEO when this fires, and going through
|
||
# transition_to_state would (1) reject the same-state
|
||
# transition with an InvalidStateTransitionError that
|
||
# gets logged via the error monitor, and (2) flood the
|
||
# timeline with redundant "transitioned" events for
|
||
# every tick.
|
||
#
|
||
# Supersession guard: if the job was reset/cancelled or a
|
||
# newer render took over while this (now stale) render is
|
||
# still emitting ticks, don't stamp our progress onto the
|
||
# job — that would drag a reset job's progress bar back to
|
||
# ~75-82% while it sits in awaiting_review.
|
||
if check_superseded(
|
||
job_manager, job_id, captured_generation, {JobStatus.RENDERING_VIDEO}
|
||
):
|
||
return
|
||
scaled = 75 + int(progress * 0.07) # Map 0-100 to 75-82
|
||
job_manager.update_job(job_id, {
|
||
'progress': scaled,
|
||
'updated_at': datetime.now(UTC),
|
||
})
|
||
# The worker is really rendering now, so drop the marker from
|
||
# earlier parked attempts; otherwise the job card keeps saying
|
||
# "waiting for encoding server availability" for the whole render.
|
||
if progress and not capacity_marker_cleared[0]:
|
||
capacity_marker_cleared[0] = True
|
||
if (job.state_data or {}).get('render_pending_capacity'):
|
||
job_manager.delete_state_data_key(job_id, 'render_pending_capacity')
|
||
|
||
with job_span("gce-render-video", job_id) as render_span:
|
||
render_start = time.time()
|
||
|
||
# Resubmit under a fresh worker job id if the encoding
|
||
# worker loses the render mid-run (OOM/deploy restart wipes
|
||
# its in-memory registry). The output GCS prefix is fixed in
|
||
# render_config, so a new worker id doesn't move outputs.
|
||
async def _submit_render(render_job_id: str):
|
||
return await encoding_service.render_video_on_gce(
|
||
render_job_id, render_config, progress_callback=progress_callback
|
||
)
|
||
|
||
result = await run_with_lost_job_resubmit(
|
||
_submit_render,
|
||
encoding_worker_job_id(job_id, captured_generation),
|
||
log=job_log,
|
||
)
|
||
render_duration = time.time() - render_start
|
||
render_span.set_attribute("duration_seconds", render_duration)
|
||
|
||
job_log.info(f"GCE render_video complete in {render_duration:.1f}s")
|
||
logger.info(f"[job:{job_id}] GCE render_video complete in {render_duration:.1f}s")
|
||
|
||
# Supersession fence: if the job was reset/cancelled or a newer
|
||
# render was triggered while we rendered, discard this stale
|
||
# result rather than recording its file_urls / transitioning
|
||
# (which would fail on an invalid terminal transition below).
|
||
#
|
||
# Residual (accepted): the GCE render already wrote the with_vocals
|
||
# object under the shared jobs/{job_id}/ prefix before this check, so
|
||
# a stale run can leave stale bytes there. A newer generation always
|
||
# re-renders and overwrites the same object, so the worst case is a
|
||
# transient stale object — never a failed job or a bad downstream
|
||
# trigger (those are blocked here + by the InvalidStateTransition
|
||
# guard). Fully closing the GCS window needs generation-scoped output
|
||
# prefixes + a conditional promote on the encoder side (follow-up).
|
||
superseded = check_superseded(
|
||
job_manager, job_id, captured_generation, {JobStatus.RENDERING_VIDEO}
|
||
)
|
||
if superseded:
|
||
job_log.warning(f"Render superseded, discarding stale GCE result: {superseded}")
|
||
logger.info(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=superseded "
|
||
f"reason={superseded}"
|
||
)
|
||
return False
|
||
|
||
# Parse response and update Firestore
|
||
output_files = result.get("output_files", [])
|
||
metadata = result.get("metadata", {})
|
||
|
||
# Update file URLs from GCE response
|
||
for gcs_path in output_files:
|
||
blob_path = gcs_path
|
||
if blob_path.startswith("gs://"):
|
||
blob_path = blob_path.split("/", 3)[3] if blob_path.count("/") >= 3 else blob_path
|
||
if "with_vocals.mkv" in blob_path:
|
||
job_manager.update_file_url(job_id, 'videos', 'with_vocals', blob_path)
|
||
elif "karaoke.ass" in blob_path:
|
||
job_manager.update_file_url(job_id, 'lyrics', 'ass', blob_path)
|
||
elif "karaoke.lrc" in blob_path:
|
||
job_manager.update_file_url(job_id, 'lyrics', 'lrc', blob_path)
|
||
elif "corrected.txt" in blob_path:
|
||
job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', blob_path)
|
||
|
||
# Handle countdown padding metadata
|
||
padding_added = metadata.get("countdown_padding_added", False)
|
||
padding_seconds = metadata.get("countdown_padding_seconds", 0)
|
||
if padding_added:
|
||
existing_lyrics_metadata = job.state_data.get('lyrics_metadata', {})
|
||
existing_lyrics_metadata['has_countdown_padding'] = True
|
||
existing_lyrics_metadata['countdown_padding_seconds'] = padding_seconds
|
||
job_manager.update_state_data(job_id, 'lyrics_metadata', existing_lyrics_metadata)
|
||
job_log.info(f"Countdown padding: {padding_seconds}s")
|
||
|
||
# Store render timing
|
||
job_manager.update_processing_metadata(job_id, "rendering", {
|
||
"render_duration_seconds": round(render_duration, 1),
|
||
"countdown_padding_added": padding_added,
|
||
"countdown_padding_seconds": padding_seconds if padding_added else 0,
|
||
"rendered_on": "gce",
|
||
})
|
||
job_manager.update_processing_metadata(job_id, "timing.render_video_worker_seconds", round(render_duration, 1))
|
||
|
||
# Transition state (same as local path)
|
||
if getattr(job, 'prep_only', False):
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.PREP_COMPLETE,
|
||
progress=100,
|
||
message="Prep phase complete - download outputs to continue locally"
|
||
)
|
||
job_log.info("=== RENDER VIDEO WORKER COMPLETE (PREP ONLY, GCE) ===")
|
||
duration = time.time() - start_time
|
||
root_span.set_attribute("duration_seconds", duration)
|
||
root_span.set_attribute("rendered_on", "gce")
|
||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s gce=true prep_only=true")
|
||
else:
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.INSTRUMENTAL_SELECTED,
|
||
progress=82,
|
||
message="Video rendered on encoding worker, starting final encoding"
|
||
)
|
||
job_log.info("=== RENDER VIDEO WORKER COMPLETE (GCE) ===")
|
||
duration = time.time() - start_time
|
||
root_span.set_attribute("duration_seconds", duration)
|
||
root_span.set_attribute("rendered_on", "gce")
|
||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s gce=true")
|
||
|
||
from backend.services.worker_service import get_worker_service
|
||
worker_service = get_worker_service()
|
||
job_log.info("Triggering video worker for final encoding...")
|
||
video_triggered = await worker_service.trigger_video_worker(job_id)
|
||
if not video_triggered:
|
||
# No exception path reaches here (trigger returns False
|
||
# instead of raising) — surface it loudly, or the job
|
||
# sits at instrumental_selected with no error and no
|
||
# sweep watching that state.
|
||
job_log.error(
|
||
"Video worker dispatch FAILED after render complete — "
|
||
"job stalls at instrumental_selected until manually retried"
|
||
)
|
||
logger.error(
|
||
f"[job:{job_id}] Video worker dispatch failed after GCE "
|
||
f"render complete (job stalled at instrumental_selected)"
|
||
)
|
||
|
||
job_manager.update_state_data(job_id, 'render_progress', {'stage': 'complete'})
|
||
return True
|
||
|
||
else:
|
||
# ============ LOCAL RENDER VIDEO PATH ============
|
||
# Transition to RENDERING_VIDEO
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.RENDERING_VIDEO,
|
||
progress=75,
|
||
message="Rendering karaoke video with corrected lyrics"
|
||
)
|
||
translations_gcs = await prepare_translations()
|
||
|
||
with tempfile.TemporaryDirectory() as temp_dir:
|
||
job_log.info(f"Created temp directory: {temp_dir}")
|
||
|
||
# Try updated corrections first (from human review), fall back to original
|
||
corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json"
|
||
corrections_gcs_original = f"jobs/{job_id}/lyrics/corrections.json"
|
||
|
||
# Get GCS path from file_urls
|
||
corrections_url = job.file_urls.get('lyrics', {}).get('corrections_updated')
|
||
if not corrections_url:
|
||
corrections_url = job.file_urls.get('lyrics', {}).get('corrections')
|
||
|
||
if not corrections_url:
|
||
# Try direct GCS paths
|
||
if storage.file_exists(corrections_gcs_updated):
|
||
corrections_gcs = corrections_gcs_updated
|
||
elif storage.file_exists(corrections_gcs_original):
|
||
corrections_gcs = corrections_gcs_original
|
||
else:
|
||
raise FileNotFoundError(f"No corrections file found for job {job_id}")
|
||
else:
|
||
# Extract GCS path from URL
|
||
corrections_gcs = _extract_gcs_path(corrections_url)
|
||
|
||
# 2. Load the ORIGINAL corrections (has full structure)
|
||
original_corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
|
||
original_corrections_path = os.path.join(temp_dir, "corrections_original.json")
|
||
|
||
job_log.info(f"Downloading original corrections from {original_corrections_gcs}")
|
||
logger.info(f"Job {job_id}: Downloading original corrections from {original_corrections_gcs}")
|
||
storage.download_file(original_corrections_gcs, original_corrections_path)
|
||
|
||
with open(original_corrections_path, 'r', encoding='utf-8') as f:
|
||
original_data = json.load(f)
|
||
|
||
# 3. Load base CorrectionResult from original data
|
||
base_result = CorrectionResult.from_dict(original_data)
|
||
|
||
# 4. Apply user corrections if available (from review UI)
|
||
# Uses the same method as preview generation to ensure
|
||
# identical CorrectionResult construction (DRY).
|
||
updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json"
|
||
|
||
if storage.file_exists(updated_corrections_gcs):
|
||
job_log.info("Found updated corrections from review, applying via shared method")
|
||
logger.info(f"Job {job_id}: Found updated corrections, applying via shared method")
|
||
updated_path = os.path.join(temp_dir, "corrections_updated.json")
|
||
storage.download_file(updated_corrections_gcs, updated_path)
|
||
|
||
with open(updated_path, 'r', encoding='utf-8') as f:
|
||
updated_data = json.load(f)
|
||
|
||
if isinstance(updated_data, dict) and "corrections" in updated_data:
|
||
# Same construction path as preview — prevents divergence
|
||
correction_result = CorrectionOperations.update_correction_result_with_data(
|
||
base_result, updated_data
|
||
)
|
||
job_log.info("Applied user corrections via CorrectionOperations")
|
||
logger.info(f"Job {job_id}: Applied user corrections via CorrectionOperations")
|
||
else:
|
||
job_log.warning("corrections_updated.json exists but has no 'corrections' key, using base result")
|
||
logger.warning(f"Job {job_id}: corrections_updated.json has no 'corrections' key, using base result")
|
||
correction_result = base_result
|
||
else:
|
||
correction_result = base_result
|
||
|
||
job_log.info(f"Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
|
||
logger.info(f"Job {job_id}: Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
|
||
|
||
# 4b. Reject untimed lyrics BEFORE any expensive work or encoder
|
||
# dispatch. Custom/replaced lyrics that were never tap-synced carry
|
||
# None start/end times and otherwise crash the encoder with a cryptic
|
||
# "'<' not supported between instances of 'NoneType' and 'float'".
|
||
# Failing here gives the user an actionable message and never sends a
|
||
# broken job to the GCE encoder (which runs its own pinned wheel).
|
||
if translations_gcs:
|
||
try:
|
||
apply_translations(
|
||
correction_result.corrected_segments, storage.download_json(translations_gcs)
|
||
)
|
||
except Exception as exc:
|
||
job_log.error(f"Could not apply translated lyrics, rendering without: {exc}")
|
||
|
||
try:
|
||
validate_segment_timing(correction_result.corrected_segments)
|
||
except LyricsTimingError as timing_err:
|
||
job_log.error(f"Render rejected — lyrics not synchronized: {timing_err}")
|
||
logger.error(f"Job {job_id}: Render rejected — lyrics not synchronized: {timing_err}")
|
||
raise ValueError(str(timing_err)) from timing_err
|
||
|
||
# 5. Download audio file (rebuilt first if the upload expired)
|
||
await asyncio.to_thread(recover_missing_inputs, job, storage, job_manager)
|
||
audio_gcs_path = job.input_media_gcs_path
|
||
audio_path = os.path.join(temp_dir, local_audio_filename(audio_gcs_path or "", "audio"))
|
||
|
||
if not audio_gcs_path:
|
||
raise FileNotFoundError(f"No input audio path for job {job_id}")
|
||
|
||
job_log.info(f"Downloading audio from {audio_gcs_path}")
|
||
logger.info(f"Job {job_id}: Downloading audio from {audio_gcs_path}")
|
||
storage.download_file(audio_gcs_path, audio_path)
|
||
job_log.info(f"Audio downloaded: {os.path.getsize(audio_path)} bytes")
|
||
|
||
# Process countdown intro if needed (for songs that start too quickly)
|
||
# This adds "3... 2... 1..." segment and pads audio with 3s silence
|
||
# Note: Countdown is deferred to this stage (not during lyrics transcription)
|
||
# so the review UI can show the original timing without the 3s shift
|
||
countdown_processor = CountdownProcessor(cache_dir=temp_dir, logger=logger)
|
||
correction_result, audio_path, padding_added, padding_seconds = countdown_processor.process(
|
||
correction_result=correction_result,
|
||
audio_filepath=audio_path,
|
||
)
|
||
if padding_added:
|
||
job_log.info(f"Countdown added: {padding_seconds}s padding applied to audio and timestamps shifted")
|
||
else:
|
||
job_log.info("No countdown needed - song starts after 3 seconds")
|
||
|
||
# Update lyrics_metadata with countdown info so video_worker knows to pad instrumental
|
||
# This is critical for audio sync - without this, instrumental won't match padded vocals
|
||
if padding_added:
|
||
existing_lyrics_metadata = job.state_data.get('lyrics_metadata', {})
|
||
existing_lyrics_metadata['has_countdown_padding'] = True
|
||
existing_lyrics_metadata['countdown_padding_seconds'] = padding_seconds
|
||
job_manager.update_state_data(job_id, 'lyrics_metadata', existing_lyrics_metadata)
|
||
job_log.info(f"Updated lyrics_metadata: has_countdown_padding=True, countdown_padding_seconds={padding_seconds}")
|
||
|
||
# Read is_duet from state_data (set by review completion endpoint)
|
||
is_duet = bool(job.state_data.get("is_duet", False))
|
||
if is_duet:
|
||
job_log.info("Duet mode enabled: using two-singer subtitle layout")
|
||
|
||
# 6. Get or create styles using the unified style loader
|
||
job_log.info("Loading style configuration...")
|
||
job_log.info(f" job.style_params_gcs_path: {job.style_params_gcs_path}")
|
||
job_log.info(f" job.style_assets: {list(job.style_assets.keys()) if job.style_assets else 'None'}")
|
||
|
||
styles_path, style_data = load_styles_from_gcs(
|
||
style_params_gcs_path=job.style_params_gcs_path,
|
||
style_assets=job.style_assets,
|
||
temp_dir=temp_dir,
|
||
download_func=storage.download_file,
|
||
logger=job_log,
|
||
)
|
||
|
||
# 7. Configure OutputGenerator
|
||
output_dir = os.path.join(temp_dir, "output")
|
||
cache_dir = os.path.join(temp_dir, "cache")
|
||
os.makedirs(output_dir, exist_ok=True)
|
||
os.makedirs(cache_dir, exist_ok=True)
|
||
|
||
job_log.info(f"Using styles from: {styles_path}")
|
||
|
||
# Get subtitle offset from job (user-specified timing adjustment)
|
||
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
|
||
if subtitle_offset != 0:
|
||
job_log.info(f"Applying subtitle offset: {subtitle_offset}ms")
|
||
|
||
config = OutputConfig(
|
||
output_dir=output_dir,
|
||
cache_dir=cache_dir,
|
||
output_styles_json=styles_path,
|
||
render_video=True,
|
||
generate_cdg=False, # CDG optional, generated separately
|
||
generate_plain_text=True,
|
||
generate_lrc=True,
|
||
video_resolution="4k",
|
||
subtitle_offset_ms=subtitle_offset,
|
||
is_duet=is_duet,
|
||
)
|
||
|
||
job_log.info(f"OutputConfig: output_styles_json={config.output_styles_json}, render_video={config.render_video}")
|
||
|
||
output_generator = OutputGenerator(config, logger)
|
||
|
||
# 8. Generate outputs (video, LRC, ASS, etc.)
|
||
# Sanitize artist/title to handle Unicode characters (curly quotes, em dashes, etc.)
|
||
safe_artist = sanitize_filename(job.artist) if job.artist else "Unknown"
|
||
safe_title = sanitize_filename(job.title) if job.title else "Unknown"
|
||
output_prefix = f"{safe_artist} - {safe_title}"
|
||
job_log.info(f"Generating outputs with prefix '{output_prefix}'")
|
||
logger.info(f"Job {job_id}: Generating outputs with prefix '{output_prefix}'")
|
||
|
||
outputs = output_generator.generate_outputs(
|
||
transcription_corrected=correction_result,
|
||
lyrics_results={}, # Reference lyrics already in correction_result
|
||
output_prefix=output_prefix,
|
||
audio_filepath=audio_path,
|
||
artist=job.artist,
|
||
title=job.title
|
||
)
|
||
|
||
# Supersession fence: bail before overwriting outputs if
|
||
# the job was reset/cancelled or a newer render started
|
||
# while we rendered (mirrors the GCE path above).
|
||
#
|
||
# Same accepted residual as the GCE path: earlier writes in
|
||
# this path (lyrics_metadata countdown padding, GCS objects
|
||
# under the shared jobs/{job_id}/ prefix) can still land on a
|
||
# newer generation in the narrow window before this check. A
|
||
# newer render re-renders and overwrites them, and the
|
||
# InvalidStateTransition guard blocks any bad downstream
|
||
# trigger, so the worst case is transient stale bytes/metadata.
|
||
# Fully closing it needs generation-scoped output paths +
|
||
# conditional promote (deferred follow-up). NB: prod renders
|
||
# via the GCE path; this local path is dev/fallback only.
|
||
superseded = check_superseded(
|
||
job_manager, job_id, captured_generation, {JobStatus.RENDERING_VIDEO}
|
||
)
|
||
if superseded:
|
||
job_log.warning(f"Render superseded, discarding stale local result: {superseded}")
|
||
logger.info(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=superseded "
|
||
f"reason={superseded}"
|
||
)
|
||
return False
|
||
|
||
# 9. Upload video to GCS
|
||
if outputs.video and os.path.exists(outputs.video):
|
||
video_size = os.path.getsize(outputs.video)
|
||
job_log.info(f"Video generated: {video_size} bytes")
|
||
video_gcs_path = f"jobs/{job_id}/videos/with_vocals.mkv"
|
||
video_url = storage.upload_file(outputs.video, video_gcs_path)
|
||
job_manager.update_file_url(job_id, 'videos', 'with_vocals', video_url)
|
||
job_log.info(f"Uploaded with_vocals.mkv to GCS")
|
||
logger.info(f"Job {job_id}: Uploaded with_vocals.mkv ({video_size} bytes)")
|
||
else:
|
||
job_log.error("Video generation failed - no output file produced!")
|
||
raise Exception("Video generation failed - no output file produced")
|
||
|
||
# 10. Upload LRC file
|
||
if outputs.lrc and os.path.exists(outputs.lrc):
|
||
lrc_gcs_path = f"jobs/{job_id}/lyrics/karaoke.lrc"
|
||
lrc_url = storage.upload_file(outputs.lrc, lrc_gcs_path)
|
||
job_manager.update_file_url(job_id, 'lyrics', 'lrc', lrc_url)
|
||
job_log.info("Uploaded karaoke.lrc")
|
||
logger.info(f"Job {job_id}: Uploaded karaoke.lrc")
|
||
|
||
# 11. Upload ASS subtitle file
|
||
if outputs.ass and os.path.exists(outputs.ass):
|
||
ass_gcs_path = f"jobs/{job_id}/lyrics/karaoke.ass"
|
||
ass_url = storage.upload_file(outputs.ass, ass_gcs_path)
|
||
job_manager.update_file_url(job_id, 'lyrics', 'ass', ass_url)
|
||
job_log.info("Uploaded karaoke.ass")
|
||
logger.info(f"Job {job_id}: Uploaded karaoke.ass")
|
||
|
||
# 12. Upload corrected text files
|
||
if outputs.corrected_txt and os.path.exists(outputs.corrected_txt):
|
||
txt_gcs_path = f"jobs/{job_id}/lyrics/corrected.txt"
|
||
txt_url = storage.upload_file(outputs.corrected_txt, txt_gcs_path)
|
||
job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', txt_url)
|
||
job_log.info("Uploaded corrected.txt")
|
||
logger.info(f"Job {job_id}: Uploaded corrected.txt")
|
||
|
||
# Store render timing in processing metadata
|
||
render_duration = time.time() - start_time
|
||
job_manager.update_processing_metadata(job_id, "rendering", {
|
||
"render_duration_seconds": round(render_duration, 1),
|
||
"countdown_padding_added": padding_added,
|
||
"countdown_padding_seconds": padding_seconds if padding_added else 0,
|
||
})
|
||
job_manager.update_processing_metadata(job_id, "timing.render_video_worker_seconds", round(render_duration, 1))
|
||
|
||
# 13. Transition based on prep_only flag
|
||
# Note: Instrumental selection was already made during combined review
|
||
# (stored in state_data['instrumental_selection'])
|
||
if getattr(job, 'prep_only', False):
|
||
# Prep-only mode: stop here and mark as prep complete
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.PREP_COMPLETE,
|
||
progress=100,
|
||
message="Prep phase complete - download outputs to continue locally"
|
||
)
|
||
job_log.info("=== RENDER VIDEO WORKER COMPLETE (PREP ONLY) ===")
|
||
duration = time.time() - start_time
|
||
root_span.set_attribute("duration_seconds", duration)
|
||
root_span.set_attribute("prep_only", True)
|
||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s prep_only=true")
|
||
else:
|
||
# Normal mode: instrumental was selected during combined review
|
||
# Transition directly to INSTRUMENTAL_SELECTED and trigger video worker
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.INSTRUMENTAL_SELECTED,
|
||
progress=82,
|
||
message="Video rendered, starting final encoding"
|
||
)
|
||
job_log.info("=== RENDER VIDEO WORKER COMPLETE ===")
|
||
duration = time.time() - start_time
|
||
root_span.set_attribute("duration_seconds", duration)
|
||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s")
|
||
|
||
# Trigger video worker for final encoding
|
||
from backend.services.worker_service import get_worker_service
|
||
|
||
worker_service = get_worker_service()
|
||
job_log.info("Triggering video worker for final encoding...")
|
||
video_triggered = await worker_service.trigger_video_worker(job_id)
|
||
if not video_triggered:
|
||
job_log.error(
|
||
"Video worker dispatch FAILED after render complete — "
|
||
"job stalls at instrumental_selected until manually retried"
|
||
)
|
||
logger.error(
|
||
f"[job:{job_id}] Video worker dispatch failed after local "
|
||
f"render complete (job stalled at instrumental_selected)"
|
||
)
|
||
|
||
# Mark render progress as complete for idempotency
|
||
# This allows the worker to be re-triggered after admin reset
|
||
job_manager.update_state_data(job_id, 'render_progress', {'stage': 'complete'})
|
||
return True
|
||
|
||
except EncodingWorkerStartError as e:
|
||
# Catches both EncodingWorkerCapacityError (zone exhausted) and
|
||
# generic EncodingWorkerStartError (e.g. 503 SERVICE_UNAVAILABLE from
|
||
# the GCE backend) — both are transient and the auto-retry scheduler
|
||
# should handle them. Hard-failing here would lose the job.
|
||
duration = time.time() - start_time
|
||
is_capacity = isinstance(e, EncodingWorkerCapacityError)
|
||
reason = "capacity exhausted" if is_capacity else "VM start failed (transient)"
|
||
job_log.warning(
|
||
f"GCE encoding worker {reason}, parking job for auto-retry: {e}"
|
||
)
|
||
logger.warning(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=pending_capacity "
|
||
f"duration={duration:.1f}s code={e.code} zone={e.zone} kind={'capacity' if is_capacity else 'start_error'}"
|
||
)
|
||
_park_job_for_capacity_retry(job_manager, job_id, e)
|
||
return False
|
||
except InvalidStateTransitionError as e:
|
||
# Safety net for the reset race: if our terminal transition became illegal
|
||
# because the job was reset/cancelled out from under us mid-render (e.g.
|
||
# admin "Review" reset while rendering_video -> the transition to
|
||
# instrumental_selected is no longer valid), the operator moved the job
|
||
# deliberately. Discard our stale result quietly instead of clobbering
|
||
# their reset with a spurious "failed" (the original 7f457087 incident).
|
||
#
|
||
# But an InvalidStateTransitionError does NOT by itself prove supersession
|
||
# — it also fires for genuine workflow defects (a bad initial transition,
|
||
# a state-machine bug). CONFIRM supersession before swallowing the error;
|
||
# otherwise fall through to the real failure path so a non-superseded job
|
||
# surfaces instead of silently stalling.
|
||
duration = time.time() - start_time
|
||
supersede_reason = check_superseded(
|
||
job_manager, job_id, captured_generation, {JobStatus.RENDERING_VIDEO}
|
||
)
|
||
if supersede_reason:
|
||
job_log.warning(
|
||
f"Render superseded mid-transition, discarding stale result: "
|
||
f"{supersede_reason} ({e})"
|
||
)
|
||
logger.info(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=superseded "
|
||
f"duration={duration:.1f}s reason={supersede_reason}"
|
||
)
|
||
return False
|
||
# Not superseded: this is a real defect, not the reset race. Fail loudly.
|
||
error_text = str(e) or repr(e)
|
||
job_log.error(f"Video render failed (unexpected invalid transition): {error_text}")
|
||
logger.error(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=error "
|
||
f"duration={duration:.1f}s error={error_text}"
|
||
)
|
||
job_manager.fail_job(job_id, f"Video render failed: {error_text}")
|
||
return False
|
||
except Exception as e:
|
||
duration = time.time() - start_time
|
||
# Use repr() so empty-string exceptions still produce a useful message;
|
||
# f"{e}" silently produced "" for bare TimeoutError() instances and
|
||
# users saw blank failure messages.
|
||
error_text = str(e) or repr(e)
|
||
job_log.error(f"Video render failed: {error_text}")
|
||
logger.error(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=error "
|
||
f"duration={duration:.1f}s error={error_text}"
|
||
)
|
||
job_manager.fail_job(job_id, f"Video render failed: {error_text}")
|
||
return False
|
||
finally:
|
||
# Always unregister — if we don't, the shutdown hook will think this
|
||
# render is still in flight and try to park the (already-completed)
|
||
# job, which would produce a no-op transition + warning log spam.
|
||
await worker_registry.unregister(job_id, "render-video")
|
||
|
||
|
||
def _park_job_for_capacity_retry(
|
||
job_manager: JobManager,
|
||
job_id: str,
|
||
error: EncodingWorkerStartError,
|
||
) -> None:
|
||
"""Transition a job to RENDER_PENDING_CAPACITY for auto-retry by the scheduler.
|
||
|
||
The user-facing message tells the user this is temporary and will retry
|
||
automatically — no action needed. Auto-retry is driven by the
|
||
/api/internal/retry-pending-render-jobs endpoint, fired every 10 min by
|
||
Cloud Scheduler.
|
||
"""
|
||
now = datetime.now(UTC).isoformat()
|
||
job = job_manager.get_job(job_id)
|
||
existing = (job.state_data or {}).get('render_pending_capacity', {}) if job else {}
|
||
|
||
pending_meta = {
|
||
'first_seen_at': existing.get('first_seen_at') or now,
|
||
'last_attempt_at': now,
|
||
'attempt_count': int(existing.get('attempt_count', 0)) + 1,
|
||
'last_code': error.code or '',
|
||
'last_zone': error.zone or '',
|
||
'last_vm': error.vm_name or '',
|
||
}
|
||
|
||
job_manager.update_state_data(job_id, 'render_pending_capacity', pending_meta)
|
||
|
||
user_message = (
|
||
"Encoding capacity is temporarily unavailable. Your job will retry "
|
||
"automatically — no action needed. Most jobs recover within 5–30 minutes."
|
||
)
|
||
|
||
job_manager.transition_to_state(
|
||
job_id=job_id,
|
||
new_status=JobStatus.RENDER_PENDING_CAPACITY,
|
||
progress=70,
|
||
message=user_message,
|
||
timeline_metadata={'code': error.code, 'zone': error.zone, 'vm': error.vm_name},
|
||
raise_on_invalid=False,
|
||
)
|
||
|
||
|
||
def _extract_gcs_path(url: str) -> str:
|
||
"""Extract GCS path from a URL or return as-is if already a path."""
|
||
if url.startswith('gs://'):
|
||
# Already a GCS path
|
||
return url.replace('gs://', '').split('/', 1)[1] if '/' in url else url
|
||
if url.startswith('https://storage.googleapis.com/'):
|
||
# Signed URL - extract path
|
||
path = url.replace('https://storage.googleapis.com/', '')
|
||
# Remove query params
|
||
if '?' in path:
|
||
path = path.split('?')[0]
|
||
# Skip bucket name
|
||
parts = path.split('/', 1)
|
||
return parts[1] if len(parts) > 1 else path
|
||
return url
|
||
|
||
|
||
def park_active_render_jobs_for_shutdown() -> int:
|
||
"""Park any in-flight render jobs in RENDER_PENDING_CAPACITY before exit.
|
||
|
||
Called by the FastAPI shutdown hook when Cloud Run is forcing the container
|
||
down (autoscaling, deploy, preemption) and one or more render workers
|
||
didn't finish in time. Without this, jobs sit at `rendering_video`
|
||
indefinitely until an operator manually retries.
|
||
|
||
Parking transitions the job to RENDER_PENDING_CAPACITY, which the
|
||
`/api/internal/retry-pending-render-jobs` Cloud Scheduler job picks up
|
||
every 10 minutes.
|
||
|
||
Only jobs currently in the RENDERING_VIDEO state are parked — if the
|
||
worker completed concurrently (or the job was already moved past this
|
||
stage by some other code path) we leave it alone.
|
||
|
||
Returns the number of jobs parked.
|
||
"""
|
||
active = worker_registry.get_active_workers()
|
||
render_job_ids = [
|
||
job_id for job_id, worker_types in active.items()
|
||
if "render-video" in worker_types
|
||
]
|
||
if not render_job_ids:
|
||
return 0
|
||
|
||
job_manager = JobManager()
|
||
parked = 0
|
||
|
||
for job_id in render_job_ids:
|
||
try:
|
||
job = job_manager.get_job(job_id)
|
||
if not job:
|
||
logger.warning(f"[job:{job_id}] Cannot park for shutdown: job not found")
|
||
continue
|
||
if job.status != JobStatus.RENDERING_VIDEO:
|
||
# Worker completed or job moved past this stage between the
|
||
# active-set snapshot and our park attempt — nothing to do.
|
||
logger.info(
|
||
f"[job:{job_id}] Skipping shutdown park: status is "
|
||
f"{job.status} (not RENDERING_VIDEO)"
|
||
)
|
||
continue
|
||
|
||
shutdown_marker = EncodingWorkerStartError(
|
||
"Render worker terminated by Cloud Run shutdown — auto-retry will resume",
|
||
code=RENDER_WORKER_SHUTDOWN_CODE,
|
||
vm_name="",
|
||
zone="",
|
||
)
|
||
_park_job_for_capacity_retry(job_manager, job_id, shutdown_marker)
|
||
parked += 1
|
||
logger.warning(
|
||
f"[job:{job_id}] WORKER_END worker=render-video status=shutdown_parked"
|
||
)
|
||
except Exception as exc: # pragma: no cover - defensive
|
||
# Don't let one failed park block the others; we have a tight
|
||
# deadline before SIGKILL.
|
||
logger.error(
|
||
f"[job:{job_id}] Failed to park for shutdown: {exc!r}"
|
||
)
|
||
|
||
return parked
|
||
|
||
|
||
# For compatibility with worker service
|
||
render_video_worker = process_render_video
|
||
|
||
|
||
# ==================== CLI Entry Point for Cloud Run Jobs ====================
|
||
# Allows the render worker to run as a standalone Cloud Run Job execution
|
||
# (video-encoding-job with an args override), immune to Cloud Run Service
|
||
# deployment rollouts. As a BackgroundTask on the service it was killed ~10s
|
||
# after SIGTERM whenever a deploy landed mid-render (incident 2026-09-13).
|
||
# Usage: python -m backend.workers.render_video_worker --job-id <job_id>
|
||
|
||
def main():
|
||
"""CLI entry point for running the render worker as a Cloud Run Job.
|
||
|
||
Exit-code contract (differs from video_worker deliberately): the Cloud Run
|
||
Job template has max_retries=2, and ``process_render_video`` handles every
|
||
expected failure internally — it parks the job (capacity/shutdown), fails
|
||
it with a user-facing message, or discards a superseded result. Re-running
|
||
the worker after any of those would re-enter a job that has already moved
|
||
to a non-render state. So a clean return (True OR False) exits 0, and only
|
||
an escaped exception (crash before state was handled) exits 1 to request a
|
||
Cloud Run retry.
|
||
"""
|
||
import argparse
|
||
import asyncio
|
||
import sys
|
||
|
||
parser = argparse.ArgumentParser(
|
||
description="Post-review render video worker for karaoke generation"
|
||
)
|
||
parser.add_argument("--job-id", required=True, help="Job ID to process")
|
||
args = parser.parse_args()
|
||
job_id = args.job_id
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||
)
|
||
|
||
logger.info(f"Starting render video worker CLI for job {job_id}")
|
||
|
||
try:
|
||
success = asyncio.run(process_render_video(job_id))
|
||
if success:
|
||
logger.info(f"Render video completed successfully for job {job_id}")
|
||
else:
|
||
logger.error(
|
||
f"Render video did not complete for job {job_id} "
|
||
"(job state already handled by the worker — parked, failed, or superseded)"
|
||
)
|
||
sys.exit(0)
|
||
except Exception as e:
|
||
logger.error(f"Render video worker crashed: {e}", exc_info=True)
|
||
sys.exit(1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|