diff --git a/backend/config.py b/backend/config.py index d5afe8e4..d26bc6ce 100644 --- a/backend/config.py +++ b/backend/config.py @@ -45,7 +45,10 @@ class Settings(BaseSettings): # When enabled, uses Gemini via Vertex AI for intelligent lyrics correction use_agentic_ai: bool = os.getenv("USE_AGENTIC_AI", "true").lower() in ("true", "1", "yes") agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "vertexai/gemini-3-flash-preview") - + # Timeout for agentic correction in seconds. If correction takes longer, abort and + # use uncorrected transcription - human review will fix any issues. + agentic_correction_timeout_seconds: int = int(os.getenv("AGENTIC_CORRECTION_TIMEOUT_SECONDS", "180")) + # Cloud Tasks (for scalable worker coordination) # When enabled, workers are triggered via Cloud Tasks for guaranteed delivery # When disabled (default), workers are triggered via direct HTTP (for development) diff --git a/backend/workers/lyrics_worker.py b/backend/workers/lyrics_worker.py index e618fb91..f64b2de9 100644 --- a/backend/workers/lyrics_worker.py +++ b/backend/workers/lyrics_worker.py @@ -44,9 +44,15 @@ from backend.config import get_settings logger = logging.getLogger(__name__) -# Timeout for transcription (10 minutes) - used by asyncio.wait_for to prevent hanging +# Timeout for non-agentic transcription (10 minutes) - prevents hanging on AudioShake API TRANSCRIPTION_TIMEOUT_SECONDS = 600 +# Default agentic correction timeout (3 minutes) +# Configurable via AGENTIC_CORRECTION_TIMEOUT_SECONDS environment variable +# If agentic AI takes longer than this, abort and use uncorrected transcription +# Human review will correct any issues +DEFAULT_AGENTIC_TIMEOUT_SECONDS = 180 + def _configure_agentic_ai(): """Configure environment variables for agentic AI correction. @@ -290,31 +296,67 @@ async def process_lyrics_transcription(job_id: str) -> bool: logger.info(f"[job:{job_id}] Calling lyrics_processor.transcribe_lyrics()") # Run transcription + correction with timeout - # AudioShake typically takes 1-2 minutes, but can hang. Uses TRANSCRIPTION_TIMEOUT_SECONDS. + # AudioShake typically takes 1-2 minutes, but can hang. with job_span("audioshake-transcription", job_id) as trans_span: trans_start = time.time() add_span_event("transcription_started") - # Run transcription in thread pool to avoid blocking the event loop - # (transcribe_lyrics is synchronous and takes 1-2 minutes) - # Note: transcribe_lyrics internally calls AudioShake and lyrics APIs - try: - with metrics.time_external_api("audioshake", job_id): - result = await asyncio.wait_for( - asyncio.to_thread( - lyrics_processor.transcribe_lyrics, - input_audio_wav=audio_path, - artist=job.artist, # Original artist for file naming - title=job.title, # Original title for file naming - track_output_dir=temp_dir, - lyrics_artist=lyrics_search_artist, # Override for lyrics search - lyrics_title=lyrics_search_title # Override for lyrics search - ), - timeout=TRANSCRIPTION_TIMEOUT_SECONDS - ) - except asyncio.TimeoutError: - raise Exception(f"Transcription timed out after {TRANSCRIPTION_TIMEOUT_SECONDS} seconds") - + # Calculate deadline for agentic correction + # If agentic AI is enabled, the correction loop will check this deadline + # and abort early if exceeded, returning uncorrected transcription for human review + settings = get_settings() + agentic_deadline = None + timeout_seconds = settings.agentic_correction_timeout_seconds + if settings.use_agentic_ai: + agentic_deadline = time.time() + timeout_seconds + job_log.info( + f"Agentic AI enabled with {timeout_seconds}s timeout" + ) + + # Run transcription in thread pool + # When agentic AI is enabled, wrap with outer timeout as safety net + # Inner deadline check in corrector.py will break out of gap loop gracefully + # Outer timeout catches case where single LLM call hangs for too long + transcription_coro = asyncio.to_thread( + lyrics_processor.transcribe_lyrics, + input_audio_wav=audio_path, + artist=job.artist, # Original artist for file naming + title=job.title, # Original title for file naming + track_output_dir=temp_dir, + lyrics_artist=lyrics_search_artist, # Override for lyrics search + lyrics_title=lyrics_search_title, # Override for lyrics search + agentic_deadline=agentic_deadline, # Deadline for agentic timeout + ) + + with metrics.time_external_api("audioshake", job_id): + if settings.use_agentic_ai: + # Apply agentic-specific timeout (shorter, with inner deadline check) + outer_timeout = timeout_seconds + 60 # Extra buffer for cleanup + try: + result = await asyncio.wait_for( + transcription_coro, + timeout=outer_timeout + ) + except asyncio.TimeoutError: + # This should rarely trigger since inner deadline check breaks first + # But provides hard stop if e.g. single LLM call takes >3 min + job_log.error( + f"HARD TIMEOUT: Transcription exceeded {outer_timeout}s. " + "This indicates the inner deadline check failed to trigger." + ) + raise RuntimeError( + f"Lyrics transcription timed out after {outer_timeout}s" + ) from None + else: + # Non-agentic mode: use general timeout (10 minutes) + try: + result = await asyncio.wait_for( + transcription_coro, + timeout=TRANSCRIPTION_TIMEOUT_SECONDS + ) + except asyncio.TimeoutError: + raise Exception(f"Transcription timed out after {TRANSCRIPTION_TIMEOUT_SECONDS} seconds") + trans_duration = time.time() - trans_start trans_span.set_attribute("duration_seconds", trans_duration) add_span_event("transcription_completed", {"duration_seconds": trans_duration}) diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 42130db2..b9cf1f4b 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -304,6 +304,36 @@ Default 5-minute timeout may not be enough for video encoding. Consider: - Breaking encoding into smaller chunks - Increasing timeout for specific operations +### Two-Layer Timeout Pattern for Long-Running Loops + +**Problem**: A loop processing many items (e.g., 74 LLM calls at 15s each = 18+ minutes) can block indefinitely. Single outer timeout isn't enough because it may interrupt mid-operation and leave state inconsistent. + +**Solution**: Use two layers of timeout protection: + +```python +# Layer 1: Inner deadline check (cooperative, clean exit) +deadline = time.time() + timeout_seconds +for item in items: + if time.time() > deadline: + logger.warning("Deadline exceeded, breaking early") + break # Exit gracefully with partial/no results + process(item) + +# Layer 2: Outer asyncio.wait_for (safety net for hung operations) +result = await asyncio.wait_for( + asyncio.to_thread(run_loop_with_deadline), + timeout=timeout_seconds + 60 # Buffer for cleanup +) +``` + +**Why both layers**: +- Inner check: Handles normal case (many items). Exits cleanly between iterations. +- Outer timeout: Catches edge case (single operation hangs). Hard stop if inner check never runs. + +**Design choice**: Use `break` not `raise` for inner timeout. This returns partial/uncorrected results rather than failing the job, allowing downstream human review to fix issues. + +See `docs/archive/2025-12-31-agentic-timeout-implementation.md` for implementation details. + ### Secret Manager Access Cloud Run service account needs `Secret Manager Secret Accessor` role. This is managed via Pulumi in `infrastructure/`. diff --git a/docs/README.md b/docs/README.md index 42381817..8cabfaf0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ ## Recent Changes +- **Agentic Correction Timeout** (2025-12-31): Added 3-minute configurable timeout for agentic AI lyrics correction. Prevents stuck jobs when songs have many gaps (74+ gaps could take 30+ minutes). On timeout, skips correction and proceeds to human review with raw transcription. See [archive/2025-12-31-agentic-timeout-implementation.md](archive/2025-12-31-agentic-timeout-implementation.md) - **Job Failure Fixes** (2025-12-31): Fixed 4 critical issues causing job failures: (1) user_email not set on jobs - users couldn't see their jobs, (2) YouTube URL race condition - lyrics worker started before audio downloaded, (3) Audio search cache not persisting across Cloud Run instances, (4) Jobs stuck in downloading state - added 10min transcription timeout. See [archive/2024-12-31-job-failure-investigation.md](archive/2024-12-31-job-failure-investigation.md) - **Vertex AI Auth Fix** (2025-12-31): Fixed `ChatGoogleGenerativeAI` to use Vertex AI backend with service account auth on Cloud Run. PR #145 broke cloud deployments by requiring `GOOGLE_API_KEY`. Solution: pass `project` parameter to trigger Vertex AI backend with ADC. See [archive/2025-12-31-vertex-ai-auth-fix.md](archive/2025-12-31-vertex-ai-auth-fix.md) - **LangChain Provider Migration** (2025-12-30): Migrated from `langchain-google-vertexai` (gRPC) to `langchain-google-genai` (REST) to fix silent hangs during model initialization. Added initialization and warm-up timeouts using ThreadPoolExecutor for fail-fast behavior. See [archive/2025-12-30-langchain-vertexai-to-genai-migration.md](archive/2025-12-30-langchain-vertexai-to-genai-migration.md) diff --git a/docs/archive/2025-12-31-agentic-timeout-implementation.md b/docs/archive/2025-12-31-agentic-timeout-implementation.md new file mode 100644 index 00000000..92696481 --- /dev/null +++ b/docs/archive/2025-12-31-agentic-timeout-implementation.md @@ -0,0 +1,68 @@ +# Agentic AI Correction Timeout Implementation + +**Date**: 2025-12-31 +**PR**: #149 +**Version**: 0.85.0 + +## Problem + +Agentic AI lyrics correction was causing stuck jobs in production. Songs with many gaps (74+) could take 30+ minutes to process since each gap takes 10-30 seconds for LLM inference. This blocked the entire job pipeline. + +Example from production logs: +``` +Processing 22/74 gaps (~15s each) = ~18 minutes elapsed +Estimated total: 74 gaps × 15s = 18+ minutes +``` + +## Solution + +Implemented a configurable 3-minute timeout (default) with two layers: + +### Layer 1: Inner Deadline Check (Cooperative) +In `corrector.py`, check deadline before processing each gap: +```python +if deadline and time.time() > deadline: + self.logger.warning("AGENTIC TIMEOUT: Deadline exceeded...") + break # Return uncorrected transcription for human review +``` + +### Layer 2: Outer asyncio.wait_for (Safety Net) +In `lyrics_worker.py`, wrap the entire transcription call: +```python +result = await asyncio.wait_for( + asyncio.to_thread(lyrics_processor.transcribe_lyrics, ...), + timeout=timeout_seconds + 60 # Extra buffer +) +``` + +## Behavior + +| Scenario | What Happens | +|----------|--------------| +| Completes in <3 min | Normal flow, corrections applied | +| Gap loop exceeds deadline | Inner check breaks, returns uncorrected | +| LLM hangs indefinitely | Outer timeout fires (rare), job fails | +| Any timeout | Job proceeds to human review with raw transcription | + +## Files Changed + +- `backend/config.py` - Added `agentic_correction_timeout_seconds` config +- `backend/workers/lyrics_worker.py` - Deadline calculation, outer timeout wrapper +- `karaoke_gen/lyrics_processor.py` - Pass deadline to controller +- `lyrics_transcriber_temp/lyrics_transcriber/core/controller.py` - Accept deadline parameter +- `lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py` - Deadline check in gap loop + +## Configuration + +Environment variable: `AGENTIC_CORRECTION_TIMEOUT_SECONDS` (default: 180) + +## Key Design Decisions + +1. **Break, don't raise**: Inner timeout uses `break` to exit the gap loop gracefully, returning whatever we have (uncorrected). This is better than raising an exception because: + - No retry logic needed + - Job continues to human review + - User can manually correct any issues + +2. **Two-layer protection**: The inner check handles normal cases (many gaps). The outer timeout catches edge cases (single LLM call hanging). Both are needed for reliability. + +3. **Configurable timeout**: Default 3 minutes balances correction quality vs. user experience. It can be adjusted via environment variable per deployment. diff --git a/karaoke_gen/lyrics_processor.py b/karaoke_gen/lyrics_processor.py index 20b20364..f2bda35c 100644 --- a/karaoke_gen/lyrics_processor.py +++ b/karaoke_gen/lyrics_processor.py @@ -3,6 +3,7 @@ import re import logging import shutil import json +from typing import Optional from lyrics_transcriber import LyricsTranscriber, OutputConfig, TranscriberConfig, LyricsConfig from lyrics_transcriber.core.controller import LyricsControllerResult from dotenv import load_dotenv @@ -252,10 +253,11 @@ class LyricsProcessor: "See README.md 'Transcription Providers' section for detailed setup instructions." ) - def transcribe_lyrics(self, input_audio_wav, artist, title, track_output_dir, lyrics_artist=None, lyrics_title=None): + def transcribe_lyrics(self, input_audio_wav, artist, title, track_output_dir, lyrics_artist=None, lyrics_title=None, + agentic_deadline: Optional[float] = None): """ Transcribe lyrics for a track. - + Args: input_audio_wav: Path to the audio file artist: Original artist name (used for filename generation) @@ -263,7 +265,9 @@ class LyricsProcessor: track_output_dir: Output directory path lyrics_artist: Artist name for lyrics processing (defaults to artist if None) lyrics_title: Title for lyrics processing (defaults to title if None) - + agentic_deadline: Optional Unix timestamp. If agentic correction is still + running after this time, it will abort and return uncorrected results. + Raises: ValueError: If transcription is enabled but no providers are configured """ @@ -423,8 +427,8 @@ class LyricsProcessor: logger=self.logger, ) - # Process and get results - results: LyricsControllerResult = transcriber.process() + # Process and get results (pass deadline for agentic timeout) + results: LyricsControllerResult = transcriber.process(agentic_deadline=agentic_deadline) self.logger.info(f"Transcriber Results Filepaths:") for key, value in results.__dict__.items(): if key.endswith("_filepath"): diff --git a/lyrics_transcriber_temp/lyrics_transcriber/core/controller.py b/lyrics_transcriber_temp/lyrics_transcriber/core/controller.py index cf12b68d..64b3fd25 100644 --- a/lyrics_transcriber_temp/lyrics_transcriber/core/controller.py +++ b/lyrics_transcriber_temp/lyrics_transcriber/core/controller.py @@ -289,8 +289,13 @@ class LyricsTranscriber: """Initialize output generation service.""" return OutputGenerator(config=self.output_config, logger=self.logger) - def process(self) -> LyricsControllerResult: - """Main processing method that orchestrates the entire workflow.""" + def process(self, agentic_deadline: Optional[float] = None) -> LyricsControllerResult: + """Main processing method that orchestrates the entire workflow. + + Args: + agentic_deadline: Optional Unix timestamp. If agentic correction is still + running after this time, it will abort and return uncorrected results. + """ self.logger.info(f"LyricsTranscriber controller beginning processing for {self.artist} - {self.title}") @@ -390,7 +395,7 @@ class LyricsTranscriber: # Step 3: Process and correct lyrics if enabled AND we have transcription results if self.output_config.run_correction and self.results.transcription_results: - self.correct_lyrics() + self.correct_lyrics(agentic_deadline=agentic_deadline) elif self.output_config.run_correction: self.logger.info("Skipping lyrics correction - no transcription results available") @@ -494,8 +499,13 @@ class LyricsTranscriber: else: self.logger.debug(" - LocalWhisper: DISABLED (enable_local_whisper=False)") - def correct_lyrics(self) -> None: - """Run lyrics correction using transcription and internet lyrics.""" + def correct_lyrics(self, agentic_deadline: Optional[float] = None) -> None: + """Run lyrics correction using transcription and internet lyrics. + + Args: + agentic_deadline: Optional Unix timestamp. If agentic correction is still + running after this time, it will abort and return uncorrected results. + """ self.logger.info("Starting lyrics correction process") # Check if we have reference lyrics to work with @@ -553,6 +563,7 @@ class LyricsTranscriber: transcription_results=self.results.transcription_results, lyrics_results=self.results.lyrics_results, metadata=metadata, + agentic_deadline=agentic_deadline, ) # Store corrected results diff --git a/lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py b/lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py index 0897af75..903faab0 100644 --- a/lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py +++ b/lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py @@ -4,6 +4,7 @@ from pathlib import Path from copy import deepcopy import os import shortuuid +import time from lyrics_transcriber.correction.handlers.levenshtein import LevenshteinHandler from lyrics_transcriber.correction.handlers.no_space_punct_match import NoSpacePunctuationMatchHandler @@ -107,8 +108,24 @@ class LyricsCorrector: transcription_results: List[TranscriptionResult], lyrics_results: Dict[str, LyricsData], metadata: Optional[Dict[str, Any]] = None, + agentic_deadline: Optional[float] = None, ) -> CorrectionResult: - """Execute the correction process.""" + """Execute the correction process. + + Args: + transcription_results: List of transcription results to correct. + lyrics_results: Dictionary of lyrics data from various sources. + metadata: Optional metadata including artist, title, audio file hash. + agentic_deadline: Optional Unix timestamp (from time.time()). If agentic + correction is still running after this time, it will abort and return + uncorrected results for human review. + + Note: + The deadline is checked between gap iterations, not during LLM processing. + A single long-running LLM call may exceed the deadline. The caller should + wrap this method with an outer timeout (e.g., asyncio.wait_for) as a safety + net for hung operations. + """ # Optional agentic routing flag from environment; default off for safety agentic_enabled = os.getenv("USE_AGENTIC_AI", "").lower() in {"1", "true", "yes"} self.logger.info(f"🤖 AGENTIC MODE: {'ENABLED' if agentic_enabled else 'DISABLED'} (USE_AGENTIC_AI={os.getenv('USE_AGENTIC_AI', 'NOT_SET')})") @@ -132,9 +149,9 @@ class LyricsCorrector: # Store anchor sequences for use in correction handlers self._anchor_sequences = anchor_sequences - # Process corrections with metadata + # Process corrections with metadata and optional deadline for agentic timeout corrections, corrected_segments, correction_steps, word_id_map, segment_id_map = self._process_corrections( - primary_transcription.segments, gap_sequences, metadata=metadata + primary_transcription.segments, gap_sequences, metadata=metadata, deadline=agentic_deadline ) # Calculate correction ratio @@ -178,10 +195,22 @@ class LyricsCorrector: return leading_space + new_word.strip() + trailing_space def _process_corrections( - self, segments: List[LyricsSegment], gap_sequences: List[GapSequence], metadata: Optional[Dict[str, Any]] = None + self, segments: List[LyricsSegment], gap_sequences: List[GapSequence], metadata: Optional[Dict[str, Any]] = None, + deadline: Optional[float] = None ) -> Tuple[List[WordCorrection], List[LyricsSegment], List[CorrectionStep], Dict[str, str], Dict[str, str]]: """Process corrections using handlers. + Args: + segments: List of lyrics segments to process. + gap_sequences: List of gap sequences to correct. + metadata: Optional metadata including artist, title, audio file hash. + deadline: Optional Unix timestamp (from time.time()). When agentic mode is + enabled and this deadline is exceeded, remaining gaps are skipped and + the method returns with whatever corrections have been made (likely none). + + Returns: + Tuple of (corrections, corrected_segments, correction_steps, word_id_map, segment_id_map). + The correction flow works as follows: 1. First pass: Process all gaps - Iterate through each gap sequence @@ -415,6 +444,16 @@ class LyricsCorrector: # === END TEMPORARY CODE === for i, gap in enumerate(gap_sequences, 1): + # Check deadline before processing each gap (agentic mode only) + # This allows us to abort early and return uncorrected results for human review + if deadline and use_agentic_env and time.time() > deadline: + self.logger.warning( + f"⏰ AGENTIC TIMEOUT: Deadline exceeded after processing {i-1}/{len(gap_sequences)} gaps. " + "Skipping remaining gaps - human review will correct any issues." + ) + # Break out of loop - continue with whatever corrections we have (likely none) + break + self.logger.info(f"Processing gap {i}/{len(gap_sequences)} at position {gap.transcription_position}") # Get the actual words for logging