mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary
In prod (2026-08-31 to 10-29), about a third of jobs were last scored
**before stems existed**. People opened the review link 15–40 s after
AWAITING_REVIEW. The audio worker's "second chance" re-score then hit
IN_REVIEW (`wrong_status`), so:
- 134 jobs never got a backing-vocals verdict;
- 7 jobs whose lyrics were in the auto tier went to a human anyway (e.g.
6c4be7d8, ed0b0c33, c4d8a256).
This is the first quick win from the full-auto v2 re-evaluation, which
targets >50% zero-touch jobs.
## Changes
- **`backend/services/auto_approval/review_gate.py` (new).**
- When the text verdict is AUTO and the only blocker is
`audio_incomplete`, the screens worker parks the job in
`GENERATING_SCREENS` with `state_data.auto_review_waiting_for_stems`. No
review link or notification goes out.
- The hold is claimed atomically (`claim_job_state_flag`) by one of:
- the audio worker once stems land (normal path);
- the screens worker itself, if stems landed while it was parking;
- `recover_stuck_jobs`, for holds older than 20 min.
- The claimant re-scores (`trigger="audio_hold"`), then either
auto-completes or opens review exactly like before, with server-side
pre-apply.
- If the post-claim re-score fails, the job falls back to human review.
If that also fails, the hold marker is restored so the sweep retries. A
job is never stranded.
- **Executor:** `hold_for_audio` is recorded in the verdict. It never
applies with `always_review`, `made_for_you`,
`backing_preference=review`, unconfident lyrics, or the kill switch on.
- **Kill switch:** `AUTO_APPROVAL_AUDIO_HOLD_ENABLED` (code default
true, because the executor runs in both the service and the
audio-separation job).
- Docs: LESSONS-LEARNED, README status row.
## Testing
- New: `test_review_gate.py` (11), `test_screens_worker_review_hold.py`
(2), executor hold cases (7), and recover-stuck-jobs hold sweep (3).
- Full backend suite locally: 6962 passed. The only failures are
`test_emulator_integration`, which needs the Firestore emulator and also
fails on pristine main.
- CodeRabbit CLI ran locally. Its one finding (stranded job on
post-claim failure) is fixed; the re-review is clean.
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_012RYLVVVPsvPbXFhRYY5VSJ
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
839 lines
41 KiB
Python
839 lines
41 KiB
Python
"""
|
|
Audio separation worker.
|
|
|
|
Handles the audio processing track of parallel processing:
|
|
1. Stage 1: Clean instrumental separation (GPU, 3-5 min)
|
|
2. Stage 2: Backing vocals separation (GPU, 2-3 min)
|
|
3. Post-processing: Combine instrumentals, normalize
|
|
|
|
Supports two modes:
|
|
- Local GPU: Runs Separator directly with L4 GPU (MODEL_DIR env var)
|
|
- Remote API: Calls audio-separator Cloud Run Service (AUDIO_SEPARATOR_API_URL env var)
|
|
|
|
Observability:
|
|
- All operations wrapped in tracing spans for Cloud Trace visibility
|
|
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
|
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import time
|
|
from typing import Optional, Dict, Any
|
|
|
|
from backend.services.job_manager import JobManager, with_notification_drain
|
|
from backend.services.input_persistence import ensure_job_inputs_persisted
|
|
from backend.services.storage_service import StorageService
|
|
from backend.services.audio_search_service import DownloadError
|
|
from backend.services.job_health_service import validate_worker_can_run
|
|
from backend.utils.audio_filenames import local_audio_filename
|
|
from backend.config import get_settings
|
|
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
|
from backend.workers.registry import worker_registry
|
|
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
|
from backend.services.metrics import metrics
|
|
from backend.services.quick_version.service import (
|
|
STATUS_FAILED as QUICK_STATUS_FAILED,
|
|
set_quick_state,
|
|
start_quick_version,
|
|
)
|
|
|
|
# Upper bound on how long the audio worker waits (at exit) for the quick-version
|
|
# render thread. The render normally finishes long before the ensemble does.
|
|
QUICK_VERSION_JOIN_TIMEOUT_SECONDS = 300
|
|
|
|
# Import from karaoke_gen package
|
|
from karaoke_gen.audio_processor import AudioProcessor
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Default ensemble presets — resolved by audio-separator package into models + algorithm.
|
|
# To change models, update ensemble_presets.json in audio-separator and release a new version.
|
|
DEFAULT_INSTRUMENTAL_PRESET = "instrumental_clean" # Fv7z + Resurrection, SDR ~17.5
|
|
DEFAULT_KARAOKE_PRESET = "karaoke" # 3 karaoke models ensemble, SDR ~10.6
|
|
|
|
# Legacy model name constants — kept for backward compatibility with per-job overrides
|
|
DEFAULT_CLEAN_MODEL = "model_bs_roformer_ep_317_sdr_12.9755.ckpt"
|
|
DEFAULT_BACKING_MODELS = ["mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt"]
|
|
DEFAULT_OTHER_MODELS = [] # Demucs 6-stem dropped — no longer separating individual instruments
|
|
|
|
|
|
# Loggers to capture for audio worker
|
|
AUDIO_WORKER_LOGGERS = [
|
|
"karaoke_gen.audio_processor",
|
|
]
|
|
|
|
|
|
def create_audio_processor(
|
|
temp_dir: str,
|
|
model_file_dir: Optional[str] = None,
|
|
clean_instrumental_model: Optional[str] = None,
|
|
backing_vocals_models: Optional[list] = None,
|
|
other_stems_models: Optional[list] = None,
|
|
instrumental_preset: Optional[str] = None,
|
|
karaoke_preset: Optional[str] = None,
|
|
) -> AudioProcessor:
|
|
"""
|
|
Create an AudioProcessor instance configured for audio processing.
|
|
|
|
Supports two modes:
|
|
- Local GPU mode: When model_file_dir is set, uses the local Separator class
|
|
with GPU models loaded from that directory. No remote API calls.
|
|
- Remote API mode (default): When model_file_dir is None, delegates separation
|
|
to a remote audio-separator API server.
|
|
|
|
Within each mode, separation can be configured via:
|
|
- Preset mode (default): Uses ensemble presets for higher quality separation.
|
|
Presets are resolved by the audio-separator API server (remote) or local
|
|
preset definitions (local GPU).
|
|
- Model mode (legacy): Uses explicit model filenames for per-job overrides.
|
|
|
|
Args:
|
|
temp_dir: Temporary directory for processing
|
|
model_file_dir: Directory containing local model files for GPU separation.
|
|
When set, enables local GPU mode. When None (default), uses remote API.
|
|
clean_instrumental_model: Model for clean instrumental (legacy, overrides preset)
|
|
backing_vocals_models: Models for backing vocals (legacy, overrides preset)
|
|
other_stems_models: Models for other stems (legacy, default empty)
|
|
instrumental_preset: Ensemble preset for instrumental separation
|
|
karaoke_preset: Ensemble preset for karaoke/BV separation
|
|
|
|
Returns:
|
|
Configured AudioProcessor instance
|
|
"""
|
|
audio_logger = logging.getLogger("karaoke_gen.audio_processor")
|
|
audio_logger.setLevel(logging.INFO)
|
|
|
|
# Determine effective configuration
|
|
# If explicit models are provided, use them (legacy per-job override)
|
|
# Otherwise use ensemble presets (default)
|
|
effective_clean_model = clean_instrumental_model or DEFAULT_CLEAN_MODEL
|
|
effective_backing_models = backing_vocals_models or DEFAULT_BACKING_MODELS
|
|
effective_other_models = other_stems_models or DEFAULT_OTHER_MODELS
|
|
|
|
ffmpeg_base_command = "ffmpeg -hide_banner -loglevel error -nostats -y"
|
|
|
|
processor = AudioProcessor(
|
|
logger=audio_logger,
|
|
log_level=logging.INFO,
|
|
log_formatter=None,
|
|
model_file_dir=model_file_dir,
|
|
lossless_output_format="FLAC",
|
|
clean_instrumental_model=effective_clean_model,
|
|
backing_vocals_models=effective_backing_models,
|
|
other_stems_models=effective_other_models,
|
|
ffmpeg_base_command=ffmpeg_base_command,
|
|
)
|
|
|
|
# Set preset configuration (used by both remote and local paths)
|
|
processor.instrumental_preset = instrumental_preset or DEFAULT_INSTRUMENTAL_PRESET
|
|
processor.karaoke_preset = karaoke_preset or DEFAULT_KARAOKE_PRESET
|
|
|
|
return processor
|
|
|
|
|
|
async def process_audio_separation(job_id: str) -> bool:
|
|
"""
|
|
Process audio separation for a job using karaoke_gen.AudioProcessor.
|
|
|
|
This is the main entry point for the audio worker.
|
|
Called asynchronously from the job submission endpoint.
|
|
|
|
Workflow:
|
|
1. Download audio from GCS
|
|
2. Stage 1: Separate with clean instrumental + other stems models (GPU)
|
|
3. Stage 2: Separate vocals for backing vocals (GPU)
|
|
4. Post-process: Combine instrumentals, normalize audio
|
|
5. Upload all stems to GCS
|
|
6. Mark job as AUDIO_COMPLETE
|
|
|
|
Args:
|
|
job_id: Job ID to process
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
start_time = time.time()
|
|
job_manager = JobManager()
|
|
storage = StorageService()
|
|
settings = get_settings()
|
|
|
|
# Create job logger for remote debugging FIRST
|
|
job_log = create_job_logger(job_id, "audio")
|
|
|
|
# Log with structured markers for easy Cloud Logging queries
|
|
logger.info(f"[job:{job_id}] WORKER_START worker=audio")
|
|
job_log.info("=== AUDIO WORKER STARTED ===")
|
|
job_log.info(f"Job ID: {job_id}")
|
|
|
|
# Register with worker registry to prevent premature container shutdown
|
|
# This ensures Cloud Run waits for this worker to complete before terminating
|
|
await worker_registry.register(job_id, "audio")
|
|
|
|
# Set up log capture for AudioProcessor
|
|
log_handler = setup_job_logging(job_id, "audio", *AUDIO_WORKER_LOGGERS)
|
|
job_log.info(f"Log handler attached for {len(AUDIO_WORKER_LOGGERS)} loggers")
|
|
|
|
job = job_manager.get_job(job_id)
|
|
if not job:
|
|
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
|
job_log.error(f"Job {job_id} not found in Firestore!")
|
|
return False
|
|
|
|
# Copy uploads/{job_id}/** into jobs/{job_id}/input/ (uploads/ expires) and repoint the job.
|
|
await asyncio.to_thread(ensure_job_inputs_persisted, job, storage, job_manager, "audio")
|
|
|
|
# Validate job status is appropriate for audio worker
|
|
# This helps catch bugs where the worker is triggered incorrectly
|
|
status_error = validate_worker_can_run("audio_worker", job)
|
|
if status_error:
|
|
logger.warning(f"[job:{job_id}] {status_error}")
|
|
job_log.warning(status_error)
|
|
# Continue anyway - this is a safety net warning, not a hard failure
|
|
|
|
quick_render = None
|
|
|
|
# Restore mode: storage retention purged this completed job's stems and the
|
|
# screens worker asked for them back (backend/services/stems_restore.py).
|
|
# Separate + upload as usual, then hand back to the screens worker; skip the
|
|
# first-run-only steps (quick version, prep-phase advance, auto-approval).
|
|
from backend.services.stems_restore import complete_stems_restore, is_restore_run, mark_restore_failed
|
|
restore_mode = is_restore_run(job)
|
|
if restore_mode:
|
|
job_log.info("RESTORE MODE: re-separating stems purged by storage retention")
|
|
|
|
# Create temporary working directory
|
|
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_{job_id}_")
|
|
job_log.info(f"Created temp directory: {temp_dir}")
|
|
|
|
try:
|
|
# Wrap entire worker in a tracing span
|
|
with job_span("audio-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
|
|
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
|
# This ensures logs from third-party libraries (karaoke_gen.audio_processor) are
|
|
# only captured by this job's handler, not handlers from other concurrent jobs
|
|
with job_logging_context(job_id):
|
|
job_log.info(f"Starting audio separation for {job.artist} - {job.title}")
|
|
logger.info(f"[job:{job_id}] Starting audio separation for {job.artist} - {job.title}")
|
|
|
|
# Determine separation mode: local GPU or remote API
|
|
api_url = os.environ.get("AUDIO_SEPARATOR_API_URL")
|
|
model_dir = os.environ.get("MODEL_DIR")
|
|
if api_url:
|
|
job_log.info(f"Audio separator mode: remote API at {api_url}")
|
|
add_span_attribute("audio_separator_mode", "remote")
|
|
add_span_attribute("audio_separator_api", api_url)
|
|
elif model_dir:
|
|
job_log.info(f"Audio separator mode: local GPU (models at {model_dir})")
|
|
add_span_attribute("audio_separator_mode", "local_gpu")
|
|
add_span_attribute("model_dir", model_dir)
|
|
else:
|
|
raise Exception(
|
|
"Audio separation not configured. Set either AUDIO_SEPARATOR_API_URL "
|
|
"(remote API) or MODEL_DIR (local GPU with baked-in models)."
|
|
)
|
|
|
|
# Download audio file from GCS or URL
|
|
with job_span("download-audio", job_id) as download_span:
|
|
job_log.info("Downloading audio file...")
|
|
# download_audio raises DownloadError with a specific reason
|
|
# on failure (no silent None → generic-error fallback).
|
|
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager)
|
|
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
|
|
download_span.set_attribute("audio_file", os.path.basename(audio_path))
|
|
download_span.set_attribute("source", "url" if job.url else "gcs")
|
|
|
|
# Capture input audio properties via ffprobe (before any processing)
|
|
_store_audio_source_metadata(job_manager, job_id, audio_path, job_log)
|
|
|
|
# kjbox make-it jobs: a quick single-model separation on the warm
|
|
# GPU, then a background render of a scrolling-lyrics draft video
|
|
# while the ensemble below runs. Never raises; joined in `finally`.
|
|
if model_dir and not restore_mode:
|
|
quick_render = start_quick_version(
|
|
job, audio_path, temp_dir, model_dir, job_manager, storage, job_log
|
|
)
|
|
|
|
# Update progress using state_data (don't change status during parallel processing)
|
|
# The status is managed at a higher level - workers just track their progress
|
|
job_manager.update_state_data(job_id, 'audio_progress', {
|
|
'stage': 'separating_stage1',
|
|
'progress': 10,
|
|
'message': 'Starting audio separation (Stage 1: Clean instrumental)'
|
|
})
|
|
|
|
# Create AudioProcessor instance (reuses karaoke_gen code)
|
|
# Use model configuration from job if provided, otherwise use defaults
|
|
job_log.info("Creating AudioProcessor instance...")
|
|
if job.clean_instrumental_model:
|
|
job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}")
|
|
add_span_attribute("clean_model", job.clean_instrumental_model)
|
|
if job.backing_vocals_models:
|
|
job_log.info(f" Using backing vocals models: {job.backing_vocals_models}")
|
|
if job.other_stems_models:
|
|
job_log.info(f" Using other stems models: {job.other_stems_models}")
|
|
|
|
audio_processor = create_audio_processor(
|
|
temp_dir,
|
|
clean_instrumental_model=job.clean_instrumental_model,
|
|
backing_vocals_models=job.backing_vocals_models,
|
|
other_stems_models=job.other_stems_models,
|
|
model_file_dir=model_dir,
|
|
)
|
|
audio_processor.job_id = job_id # Used as filename prefix (avoids artist/title encoding issues)
|
|
|
|
# Pass GCS URI so the separator can fetch directly from GCS (avoids 413 for large files)
|
|
if job.input_media_gcs_path:
|
|
audio_processor.input_gcs_uri = f"gs://{settings.gcs_bucket_name}/{job.input_media_gcs_path}"
|
|
job_log.info(f" GCS URI for separator: {audio_processor.input_gcs_uri}")
|
|
|
|
# Store effective model names in state_data for video_worker to use in file naming
|
|
# This ensures output filenames match local CLI behavior (e.g., "Instrumental model_bs_roformer_ep_317_sdr_12.9755.ckpt")
|
|
effective_model_names = {
|
|
'clean_instrumental_model': job.clean_instrumental_model or DEFAULT_CLEAN_MODEL,
|
|
'backing_vocals_models': job.backing_vocals_models or DEFAULT_BACKING_MODELS,
|
|
'other_stems_models': job.other_stems_models or DEFAULT_OTHER_MODELS,
|
|
}
|
|
job_manager.update_state_data(job_id, 'model_names', effective_model_names)
|
|
job_log.info(f"Stored effective model names: clean={effective_model_names['clean_instrumental_model']}")
|
|
|
|
# Format artist-title for file naming (matches CLI behavior)
|
|
# Sanitize to handle Unicode characters (curly quotes, em dashes, etc.)
|
|
# that cause HTTP header encoding issues with the remote API
|
|
from karaoke_gen.utils import sanitize_filename
|
|
safe_artist = sanitize_filename(job.artist) if job.artist else "Unknown"
|
|
safe_title = sanitize_filename(job.title) if job.title else "Unknown"
|
|
artist_title = f"{safe_artist} - {safe_title}"
|
|
|
|
# Run audio separation (local GPU or remote API)
|
|
# This returns a dict with paths to all separated stems
|
|
with job_span("audio-separation", job_id) as sep_span:
|
|
sep_start = time.time()
|
|
job_log.info("Starting audio separation (this may take 5-10 minutes)...")
|
|
job_log.info(" Stage 1: Clean instrumental separation (MDX models)")
|
|
job_log.info(" Stage 2: Backing vocals separation (Demucs model)")
|
|
add_span_event("separation_started")
|
|
logger.info(f"[job:{job_id}] Starting audio separation")
|
|
|
|
# Retry with exponential backoff. The audio-separator GPU service
|
|
# may be busy processing other jobs for 5-10+ minutes each, so we
|
|
# need patience: 60s, 120s, 240s, 480s = ~15 min total wait.
|
|
max_retries = 5
|
|
for attempt in range(1, max_retries + 1):
|
|
try:
|
|
with metrics.time_external_api("audio_separation", job_id):
|
|
separation_result = audio_processor.process_audio_separation(
|
|
audio_file=audio_path,
|
|
artist_title=artist_title,
|
|
track_output_dir=temp_dir
|
|
)
|
|
break # Success
|
|
except Exception as e:
|
|
error_str = str(e)
|
|
if attempt < max_retries:
|
|
wait_time = 60 * (2 ** (attempt - 1)) # 60s, 120s, 240s, 480s
|
|
job_log.warning(
|
|
f"Audio separation attempt {attempt}/{max_retries} failed: {error_str}. "
|
|
f"Retrying in {wait_time}s..."
|
|
)
|
|
logger.warning(f"[job:{job_id}] Separation attempt {attempt} failed: {error_str}")
|
|
time.sleep(wait_time)
|
|
else:
|
|
job_log.error(f"Audio separation failed after {max_retries} attempts: {error_str}")
|
|
raise
|
|
|
|
sep_duration = time.time() - sep_start
|
|
sep_span.set_attribute("duration_seconds", sep_duration)
|
|
sep_span.set_attribute("stem_count", len(separation_result))
|
|
add_span_event("separation_completed", {"duration_seconds": sep_duration})
|
|
|
|
job_log.info("Audio separation complete!")
|
|
job_log.info(f" Generated {len(separation_result)} stem files")
|
|
logger.info(f"[job:{job_id}] Audio separation complete, organizing results")
|
|
|
|
# Update progress using state_data (don't change status during parallel processing)
|
|
job_manager.update_state_data(job_id, 'audio_progress', {
|
|
'stage': 'audio_complete',
|
|
'progress': 45,
|
|
'message': 'Audio separation complete, uploading stems'
|
|
})
|
|
|
|
# Upload all stems to GCS
|
|
with job_span("upload-stems", job_id) as upload_span:
|
|
await upload_separation_results(job_id, separation_result, storage, job_manager)
|
|
upload_span.set_attribute("stem_count", len(separation_result))
|
|
|
|
logger.info(f"[job:{job_id}] All stems uploaded successfully")
|
|
|
|
# Transcode the review-playable stems to OGG now. screens_worker's
|
|
# eager pass runs on lyrics completion — usually before separation —
|
|
# so otherwise the first preview-modal play waits on an on-demand
|
|
# ffmpeg run in the API (7-15 s of silence).
|
|
await _transcode_review_stems(job_id, job_manager, storage, job_log)
|
|
|
|
# Store processing metadata for separation provenance
|
|
duration = time.time() - start_time
|
|
job_manager.update_processing_metadata(job_id, "separation_restore" if restore_mode else "separation", {
|
|
"provider": "local_gpu" if model_dir else "remote_api",
|
|
"clean_model": effective_model_names['clean_instrumental_model'],
|
|
"backing_models": effective_model_names['backing_vocals_models'],
|
|
"other_models": effective_model_names['other_stems_models'],
|
|
"stems_generated": list(separation_result.keys()),
|
|
"duration_seconds": round(sep_duration, 1),
|
|
})
|
|
job_manager.update_processing_metadata(job_id, "timing.audio_worker_seconds", round(duration, 1))
|
|
|
|
# Analyze backing vocals for instrumental selection recommendation
|
|
# This must run here (after stems are uploaded) rather than in screens_worker,
|
|
# because screens_worker triggers on lyrics completion and audio may not be done yet
|
|
from backend.workers.screens_worker import _analyze_backing_vocals
|
|
try:
|
|
await _analyze_backing_vocals(job_id, job_manager, storage, job_log)
|
|
except Exception as e:
|
|
logger.warning(f"[job:{job_id}] Backing vocals analysis failed (non-fatal): {e}")
|
|
|
|
if restore_mode:
|
|
await complete_stems_restore(job_id, job_manager)
|
|
duration = time.time() - start_time
|
|
logger.info(f"[job:{job_id}] WORKER_END worker=audio mode=restore status=success duration={duration:.1f}s")
|
|
return True
|
|
|
|
# Mark audio processing complete
|
|
job_manager.mark_audio_complete(job_id)
|
|
|
|
# Fallback screens trigger: if lyrics already finished but its
|
|
# (primary) screens dispatch was lost, advancing here rescues the
|
|
# job. No-op unless lyrics are done AND the job is still at
|
|
# `downloading`, so the audio/review decoupling is preserved and a
|
|
# normal run never double-triggers.
|
|
await job_manager.advance_to_screens_if_ready(job_id)
|
|
|
|
# If review already finished and the video worker parked the job
|
|
# waiting for these stems, resume the final encode now.
|
|
try:
|
|
if await job_manager.resume_video_if_waiting_for_stems(job_id):
|
|
job_log.info("Stems ready: resumed the parked final encode")
|
|
except Exception as e:
|
|
# recover_stuck_jobs re-triggers a parked job within ~10 min
|
|
logger.error(f"[job:{job_id}] Failed to resume parked video worker: {e}")
|
|
|
|
# Auto-approval second chance: when lyrics/screens finished BEFORE
|
|
# audio, the job is already parked in AWAITING_REVIEW and the
|
|
# screens-worker scoring ran without backing analysis. Now that
|
|
# stems + analysis exist, re-score and auto-complete the review if
|
|
# fully confident. No-op (fail-safe) in every other state.
|
|
# A job the screens worker held out of review (lyrics confident,
|
|
# stems pending) is released here: re-scored, then auto-completed
|
|
# or opened for review (review_gate.py).
|
|
try:
|
|
from backend.services.auto_approval.executor import maybe_auto_complete_review
|
|
from backend.services.auto_approval.review_gate import release_audio_hold
|
|
released = await release_audio_hold(job_id, reason="audio_worker")
|
|
if released == "auto_completed":
|
|
job_log.info("Auto-approved after audio completion: review skipped")
|
|
elif released is None:
|
|
auto_result = await maybe_auto_complete_review(job_id, trigger="audio_worker")
|
|
if auto_result.get("outcome") == "auto_completed":
|
|
job_log.info("Auto-approved after audio completion: review skipped")
|
|
except Exception as e:
|
|
logger.warning(f"[job:{job_id}] auto-approval attempt failed (non-fatal): {e}")
|
|
|
|
root_span.set_attribute("duration_seconds", duration)
|
|
logger.info(f"[job:{job_id}] WORKER_END worker=audio status=success duration={duration:.1f}s")
|
|
return True
|
|
|
|
except DownloadError as e:
|
|
duration = time.time() - start_time
|
|
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}", exc_info=True)
|
|
if restore_mode:
|
|
mark_restore_failed(job_id, job_manager)
|
|
job_manager.mark_job_failed(
|
|
job_id=job_id,
|
|
error_message=f"Audio download failed: {str(e)}",
|
|
error_details={"stage": "download_audio", "error": str(e)}
|
|
)
|
|
return False
|
|
|
|
except Exception as e:
|
|
duration = time.time() - start_time
|
|
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}", exc_info=True)
|
|
if restore_mode:
|
|
mark_restore_failed(job_id, job_manager)
|
|
job_manager.mark_job_failed(
|
|
job_id=job_id,
|
|
error_message=f"Audio separation failed: {str(e)}",
|
|
error_details={"stage": "audio_separation", "error": str(e)}
|
|
)
|
|
return False
|
|
|
|
finally:
|
|
# Let the quick-version render (reads files in temp_dir) finish before
|
|
# cleanup — even when the full separation failed.
|
|
if quick_render is not None:
|
|
finished = await asyncio.to_thread(quick_render.join, QUICK_VERSION_JOIN_TIMEOUT_SECONDS)
|
|
if not finished:
|
|
logger.warning(f"[job:{job_id}] Quick version render still running at worker exit; abandoning")
|
|
set_quick_state(job_manager, job_id, QUICK_STATUS_FAILED, error="render timed out")
|
|
|
|
# Unregister from worker registry to signal completion
|
|
await worker_registry.unregister(job_id, "audio")
|
|
|
|
# Remove log handler to avoid duplicate logging on future runs
|
|
for logger_name in AUDIO_WORKER_LOGGERS:
|
|
try:
|
|
logging.getLogger(logger_name).removeHandler(log_handler)
|
|
except Exception:
|
|
pass
|
|
|
|
# Cleanup temporary directory
|
|
if os.path.exists(temp_dir):
|
|
shutil.rmtree(temp_dir)
|
|
logger.debug(f"[job:{job_id}] Cleaned up temp directory: {temp_dir}")
|
|
|
|
|
|
async def download_audio(
|
|
job_id: str,
|
|
temp_dir: str,
|
|
storage: StorageService,
|
|
job,
|
|
job_manager_instance: JobManager = None
|
|
) -> str:
|
|
"""
|
|
Fetch the job's audio file from GCS to a local temp directory.
|
|
|
|
All inputs are expected to already be in GCS by the time this worker runs:
|
|
- Uploaded files: at input_media_gcs_path
|
|
- URL jobs: downloaded by the flacfetch service in create_job_from_url
|
|
before any worker is triggered
|
|
|
|
This worker never downloads from a URL itself (no yt-dlp on Cloud Run); a
|
|
URL job arriving without a GCS file is treated as an upstream bug.
|
|
|
|
Args:
|
|
job_id: Job ID
|
|
temp_dir: Temporary directory to save to
|
|
storage: StorageService instance
|
|
job: Job object with URL or GCS path
|
|
job_manager_instance: Optional JobManager instance for updating job metadata
|
|
|
|
Returns:
|
|
Path to the downloaded audio file.
|
|
|
|
Raises:
|
|
DownloadError: with a specific reason if the audio can't be obtained.
|
|
We fail loudly with the real cause rather than returning None and
|
|
surfacing a generic "Failed to download audio file" to the user.
|
|
"""
|
|
try:
|
|
from karaoke_gen.utils import sanitize_filename
|
|
|
|
# Case 1: File was uploaded to GCS
|
|
if job.input_media_gcs_path:
|
|
logger.info(f"Job {job_id}: Downloading uploaded file from GCS: {job.input_media_gcs_path}")
|
|
# Sanitize filename to handle Unicode chars that cause HTTP header encoding issues
|
|
safe_filename = sanitize_filename(job.filename) if job.filename else "input.flac"
|
|
local_path = os.path.join(temp_dir, safe_filename)
|
|
storage.download_file(job.input_media_gcs_path, local_path)
|
|
logger.info(f"Job {job_id}: Downloaded uploaded file to {local_path}")
|
|
|
|
# Persist uploaded audio to jobs/ prefix (uploads/ is a lifecycle-expiring staging area; see input_persistence)
|
|
if job.input_media_gcs_path.startswith("uploads/"):
|
|
try:
|
|
persistent_path = f"jobs/{job_id}/input/{safe_filename}"
|
|
storage.upload_file(local_path, persistent_path)
|
|
jm = job_manager_instance or JobManager()
|
|
jm.update_job(job_id, {'input_media_gcs_path': persistent_path})
|
|
jm.update_file_url(job_id, 'input', 'audio', persistent_path)
|
|
logger.info(f"Job {job_id}: Persisted upload to {persistent_path}")
|
|
except Exception as e:
|
|
logger.warning(f"Job {job_id}: Failed to persist upload to jobs/ prefix: {e}")
|
|
|
|
return local_path
|
|
|
|
# Case 2: URL download (from file_urls if already downloaded, or from job.url)
|
|
if job.file_urls and job.file_urls.get('input'):
|
|
# Already downloaded and stored in GCS
|
|
input_url = job.file_urls.get('input')
|
|
local_path = os.path.join(temp_dir, local_audio_filename(str(input_url), "input"))
|
|
storage.download_file(input_url, local_path)
|
|
logger.info(f"Job {job_id}: Downloaded audio from GCS: {input_url}")
|
|
return local_path
|
|
|
|
# A URL job that reached this worker WITHOUT a pre-downloaded GCS file is
|
|
# an upstream bug: all URLs are downloaded by the flacfetch service in
|
|
# create_job_from_url before any worker runs. We never run yt-dlp on
|
|
# Cloud Run, so there is no local fallback — fail clearly instead.
|
|
if job.url:
|
|
logger.error(
|
|
f"Job {job_id}: URL job reached the audio worker without a "
|
|
f"downloaded input_media_gcs_path (url={job.url}). URLs must be "
|
|
"fetched via the flacfetch service before processing; refusing to "
|
|
"download locally."
|
|
)
|
|
raise DownloadError(
|
|
"URL job reached the audio worker without a downloaded input file. "
|
|
"URLs must be fetched via the flacfetch service before processing; "
|
|
"we never download locally on Cloud Run."
|
|
)
|
|
|
|
logger.error(f"Job {job_id}: No input source found (no GCS path or file_urls)")
|
|
raise DownloadError(
|
|
"No input audio source found for this job (no GCS path and no file_urls)."
|
|
)
|
|
|
|
except DownloadError:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Job {job_id}: Failed to download audio: {e}", exc_info=True)
|
|
raise DownloadError(f"Failed to download audio from storage: {e}") from e
|
|
|
|
|
|
async def upload_separation_results(
|
|
job_id: str,
|
|
separation_result: Dict[str, Any],
|
|
storage: StorageService,
|
|
job_manager: JobManager
|
|
) -> None:
|
|
"""
|
|
Upload all audio separation results to GCS.
|
|
|
|
The separation_result dict from AudioProcessor.process_audio_separation() contains:
|
|
- clean_instrumental: Dict with 'vocals' and 'instrumental' paths
|
|
- other_stems: Dict with stem paths (bass, drums, guitar, piano, other)
|
|
- backing_vocals: Dict with model-keyed paths to lead_vocals and backing_vocals
|
|
- combined_instrumentals: Dict with model-keyed paths to instrumental+BV files
|
|
|
|
Args:
|
|
job_id: Job ID
|
|
separation_result: Result dict from AudioProcessor
|
|
storage: StorageService instance
|
|
job_manager: JobManager instance
|
|
"""
|
|
logger.info(f"Job {job_id}: Uploading separation results to GCS")
|
|
|
|
# Upload clean instrumental + vocals (Stage 1, clean model)
|
|
if separation_result.get("clean_instrumental"):
|
|
clean = separation_result["clean_instrumental"]
|
|
|
|
if clean.get("instrumental") and os.path.exists(clean["instrumental"]):
|
|
gcs_path = f"jobs/{job_id}/stems/instrumental_clean.flac"
|
|
url = storage.upload_file(clean["instrumental"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'instrumental_clean', url)
|
|
logger.info(f"Job {job_id}: Uploaded clean instrumental")
|
|
|
|
if clean.get("vocals") and os.path.exists(clean["vocals"]):
|
|
gcs_path = f"jobs/{job_id}/stems/vocals_clean.flac"
|
|
url = storage.upload_file(clean["vocals"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'vocals_clean', url)
|
|
logger.info(f"Job {job_id}: Uploaded clean vocals")
|
|
|
|
# Upload other stems (Stage 1, htdemucs 6-stem)
|
|
if separation_result.get("other_stems"):
|
|
for stem_name, stem_value in separation_result["other_stems"].items():
|
|
# Handle both string paths and nested dicts
|
|
if isinstance(stem_value, str):
|
|
stem_path = stem_value
|
|
elif isinstance(stem_value, dict):
|
|
# Some models return nested dicts like {"path": "/path/to/file"}
|
|
stem_path = stem_value.get("path") or stem_value.get("file")
|
|
logger.debug(f"Job {job_id}: other_stems[{stem_name}] is dict: {stem_value}")
|
|
else:
|
|
logger.warning(f"Job {job_id}: Unexpected type for other_stems[{stem_name}]: {type(stem_value)}")
|
|
continue
|
|
|
|
if stem_path and isinstance(stem_path, str) and os.path.exists(stem_path):
|
|
gcs_path = f"jobs/{job_id}/stems/{stem_name.lower()}.flac"
|
|
url = storage.upload_file(stem_path, gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', stem_name.lower(), url)
|
|
logger.info(f"Job {job_id}: Uploaded {stem_name} stem")
|
|
|
|
# Upload backing vocals separation (Stage 2)
|
|
if separation_result.get("backing_vocals"):
|
|
# backing_vocals is a dict keyed by model name
|
|
for model_name, bv_stems in separation_result["backing_vocals"].items():
|
|
if bv_stems.get("lead_vocals") and os.path.exists(bv_stems["lead_vocals"]):
|
|
gcs_path = f"jobs/{job_id}/stems/lead_vocals.flac"
|
|
url = storage.upload_file(bv_stems["lead_vocals"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'lead_vocals', url)
|
|
logger.info(f"Job {job_id}: Uploaded lead vocals")
|
|
|
|
if bv_stems.get("backing_vocals") and os.path.exists(bv_stems["backing_vocals"]):
|
|
gcs_path = f"jobs/{job_id}/stems/backing_vocals.flac"
|
|
url = storage.upload_file(bv_stems["backing_vocals"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'backing_vocals', url)
|
|
logger.info(f"Job {job_id}: Uploaded backing vocals")
|
|
|
|
# Only process first model (we typically only use one backing vocals model)
|
|
break
|
|
|
|
# Upload combined instrumentals (instrumental + backing vocals)
|
|
if separation_result.get("combined_instrumentals"):
|
|
# combined_instrumentals is a dict keyed by model name
|
|
for model_name, combined_path in separation_result["combined_instrumentals"].items():
|
|
if combined_path and os.path.exists(combined_path):
|
|
gcs_path = f"jobs/{job_id}/stems/instrumental_with_backing.flac"
|
|
url = storage.upload_file(combined_path, gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'instrumental_with_backing', url)
|
|
logger.info(f"Job {job_id}: Uploaded instrumental with backing vocals")
|
|
|
|
# Only process first model
|
|
break
|
|
|
|
# Store instrumental options in state_data for later selection
|
|
instrumental_options = {}
|
|
if separation_result.get("clean_instrumental", {}).get("instrumental"):
|
|
instrumental_options["clean"] = f"jobs/{job_id}/stems/instrumental_clean.flac"
|
|
if separation_result.get("combined_instrumentals"):
|
|
instrumental_options["with_backing"] = f"jobs/{job_id}/stems/instrumental_with_backing.flac"
|
|
|
|
if instrumental_options:
|
|
job_manager.update_state_data(job_id, 'instrumental_options', instrumental_options)
|
|
logger.info(f"Job {job_id}: Stored instrumental options: {list(instrumental_options.keys())}")
|
|
|
|
|
|
async def _transcode_review_stems(job_id: str, job_manager: JobManager, storage: StorageService, job_log) -> None:
|
|
"""Pre-transcode review stems to OGG (non-fatal: the API transcodes lazily on miss)."""
|
|
from backend.services.audio_transcoding_service import AudioTranscodingService
|
|
|
|
try:
|
|
job = job_manager.get_job(job_id)
|
|
if not job:
|
|
return
|
|
written = await asyncio.to_thread(
|
|
AudioTranscodingService(storage_service=storage).transcode_review_stems, job
|
|
)
|
|
job_log.info(f"Review stem audio transcoded: {len(written)} files")
|
|
except Exception as e:
|
|
job_log.warning(f"Review stem transcoding failed (non-fatal): {e}")
|
|
|
|
|
|
def _store_audio_source_metadata(
|
|
job_manager: JobManager,
|
|
job_id: str,
|
|
audio_path: str,
|
|
job_log,
|
|
) -> None:
|
|
"""Run ffprobe on input audio and store properties in processing_metadata.
|
|
|
|
Captures format, codec, sample rate, channels, duration, file size, and bitrate.
|
|
Non-fatal: failures are logged but don't stop the job.
|
|
"""
|
|
import subprocess
|
|
import json as _json
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
[
|
|
"ffprobe", "-v", "quiet", "-print_format", "json",
|
|
"-show_format", "-show_streams", audio_path,
|
|
],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
if result.returncode != 0:
|
|
job_log.warning(f"ffprobe failed (rc={result.returncode}): {result.stderr[:200]}")
|
|
return
|
|
|
|
probe = _json.loads(result.stdout)
|
|
fmt = probe.get("format", {})
|
|
# Find audio stream
|
|
audio_stream = None
|
|
for stream in probe.get("streams", []):
|
|
if stream.get("codec_type") == "audio":
|
|
audio_stream = stream
|
|
break
|
|
|
|
metadata = {
|
|
"input_format": fmt.get("format_name", "").split(",")[0],
|
|
"input_duration_seconds": round(float(fmt.get("duration", 0)), 1),
|
|
"input_file_size_bytes": int(fmt.get("size", 0)),
|
|
"input_bitrate_kbps": round(int(fmt.get("bit_rate", 0)) / 1000) if fmt.get("bit_rate") else None,
|
|
}
|
|
|
|
if audio_stream:
|
|
metadata.update({
|
|
"input_codec": audio_stream.get("codec_name"),
|
|
"input_sample_rate": int(audio_stream.get("sample_rate", 0)) or None,
|
|
"input_channels": audio_stream.get("channels"),
|
|
"input_bit_depth": int(audio_stream.get("bits_per_raw_sample", 0)) or None,
|
|
})
|
|
|
|
job_manager.update_processing_metadata(job_id, "audio_source", metadata)
|
|
job_log.info(f"Audio source: {metadata.get('input_codec', '?')} "
|
|
f"{metadata.get('input_sample_rate', '?')}Hz "
|
|
f"{metadata.get('input_channels', '?')}ch "
|
|
f"{metadata.get('input_duration_seconds', '?')}s")
|
|
|
|
except Exception as e:
|
|
job_log.warning(f"Failed to capture audio source metadata (non-fatal): {e}")
|
|
logger.warning(f"Job {job_id}: ffprobe metadata capture failed: {e}")
|
|
|
|
|
|
# ==================== CLI Entry Point for Cloud Run Jobs ====================
|
|
# This allows the audio worker to be run as a standalone Cloud Run Job.
|
|
# Usage: python -m backend.workers.audio_worker --job-id <job_id>
|
|
|
|
def main():
|
|
"""
|
|
CLI entry point for running audio worker as a Cloud Run Job.
|
|
|
|
Cloud Run Jobs run to completion without HTTP request lifecycle concerns,
|
|
avoiding the instance termination issue where Cloud Run would shut down
|
|
instances mid-processing when using BackgroundTasks.
|
|
|
|
Usage:
|
|
python -m backend.workers.audio_worker --job-id abc123
|
|
|
|
Environment Variables:
|
|
GOOGLE_CLOUD_PROJECT: GCP project ID (required)
|
|
GCS_BUCKET_NAME: Storage bucket name (required)
|
|
AUDIO_SEPARATOR_API_URL: Remote API URL (or MODEL_DIR for local GPU)
|
|
"""
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
|
|
# Set up argument parser
|
|
parser = argparse.ArgumentParser(
|
|
description="Audio separation worker for karaoke generation"
|
|
)
|
|
parser.add_argument(
|
|
"--job-id",
|
|
required=True,
|
|
help="Job ID to process"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
job_id = args.job_id
|
|
|
|
# Configure logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
)
|
|
|
|
logger.info(f"Starting audio worker CLI for job {job_id}")
|
|
|
|
# Run the async worker function
|
|
try:
|
|
# Drain: push/email sends scheduled by the job's last state change must
|
|
# finish before asyncio.run cancels pending tasks on exit.
|
|
success = asyncio.run(with_notification_drain(process_audio_separation(job_id)))
|
|
if success:
|
|
logger.info(f"Audio separation completed successfully for job {job_id}")
|
|
sys.exit(0)
|
|
else:
|
|
logger.error(f"Audio separation failed for job {job_id}")
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
logger.error(f"Audio worker crashed: {e}", exc_info=True)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|