feat(agentic): Add 3-minute timeout for lyrics correction (#149)

## Summary

- Adds configurable timeout (default 3 minutes) for agentic AI lyrics
correction
- When timeout occurs, skips remaining gaps and returns uncorrected
transcription
- Job proceeds to human review where user can manually correct lyrics
- Prevents stuck jobs in production (songs with 74+ gaps could take 30+
minutes)

## Implementation

Two-layer timeout approach:
1. **Inner deadline check** in `corrector.py` gap loop - cooperative
exit before each gap
2. **Outer `asyncio.wait_for()`** in `lyrics_worker.py` - safety net
catches hung LLM calls

## Files Changed

- `backend/config.py` - Add `agentic_correction_timeout_seconds` config
- `backend/workers/lyrics_worker.py` - Add deadline calculation and
outer timeout
- `karaoke_gen/lyrics_processor.py` - Pass deadline through to
controller
- `lyrics_transcriber_temp/lyrics_transcriber/core/controller.py` -
Accept deadline parameter
- `lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py` -
Add deadline check in gap loop
- `pyproject.toml` - Bump version to 0.85.0

## Test plan

- [x] Unit tests pass (812 passed, 29 skipped)
- [ ] Deploy to production and monitor agentic correction jobs
- [ ] Verify timeout triggers after ~3 minutes on long-running
corrections
- [ ] Verify job proceeds to `awaiting_review` with uncorrected lyrics

🤖 Generated with [Claude Code](https://claude.com/claude-code)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

**New Features**
- Agentic AI lyrics correction now includes a configurable timeout
(default 3 minutes). If correction exceeds the timeout, jobs proceed
with raw transcription available for human review.

**Documentation**
- Added implementation notes on timeout behavior and the two-layer
timeout pattern architecture.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.5 authored and GitHub committed 2025-12-31 16:13:46 -05:00
1 parent ed2305d446
commit c68c221ba3
8 files changed
+235 -37

No files matched your search

+4 -1
View File
@@ -45,7 +45,10 @@ class Settings(BaseSettings):
# When enabled, uses Gemini via Vertex AI for intelligent lyrics correction
use_agentic_ai: bool = os.getenv("USE_AGENTIC_AI", "true").lower() in ("true", "1", "yes")
agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "vertexai/gemini-3-flash-preview")
# Timeout for agentic correction in seconds. If correction takes longer, abort and
# use uncorrected transcription - human review will fix any issues.
agentic_correction_timeout_seconds: int = int(os.getenv("AGENTIC_CORRECTION_TIMEOUT_SECONDS", "180"))
# Cloud Tasks (for scalable worker coordination)
# When enabled, workers are triggered via Cloud Tasks for guaranteed delivery
# When disabled (default), workers are triggered via direct HTTP (for development)
+64 -22
View File
@@ -44,9 +44,15 @@ from backend.config import get_settings
logger = logging.getLogger(__name__)
# Timeout for transcription (10 minutes) - used by asyncio.wait_for to prevent hanging
# Timeout for non-agentic transcription (10 minutes) - prevents hanging on AudioShake API
TRANSCRIPTION_TIMEOUT_SECONDS = 600
# Default agentic correction timeout (3 minutes)
# Configurable via AGENTIC_CORRECTION_TIMEOUT_SECONDS environment variable
# If agentic AI takes longer than this, abort and use uncorrected transcription
# Human review will correct any issues
DEFAULT_AGENTIC_TIMEOUT_SECONDS = 180
def _configure_agentic_ai():
"""Configure environment variables for agentic AI correction.
@@ -290,31 +296,67 @@ async def process_lyrics_transcription(job_id: str) -> bool:
logger.info(f"[job:{job_id}] Calling lyrics_processor.transcribe_lyrics()")
# Run transcription + correction with timeout
# AudioShake typically takes 1-2 minutes, but can hang. Uses TRANSCRIPTION_TIMEOUT_SECONDS.
# AudioShake typically takes 1-2 minutes, but can hang.
with job_span("audioshake-transcription", job_id) as trans_span:
trans_start = time.time()
add_span_event("transcription_started")
# Run transcription in thread pool to avoid blocking the event loop
# (transcribe_lyrics is synchronous and takes 1-2 minutes)
# Note: transcribe_lyrics internally calls AudioShake and lyrics APIs
try:
with metrics.time_external_api("audioshake", job_id):
result = await asyncio.wait_for(
asyncio.to_thread(
lyrics_processor.transcribe_lyrics,
input_audio_wav=audio_path,
artist=job.artist, # Original artist for file naming
title=job.title, # Original title for file naming
track_output_dir=temp_dir,
lyrics_artist=lyrics_search_artist, # Override for lyrics search
lyrics_title=lyrics_search_title # Override for lyrics search
),
timeout=TRANSCRIPTION_TIMEOUT_SECONDS
)
except asyncio.TimeoutError:
raise Exception(f"Transcription timed out after {TRANSCRIPTION_TIMEOUT_SECONDS} seconds")
# Calculate deadline for agentic correction
# If agentic AI is enabled, the correction loop will check this deadline
# and abort early if exceeded, returning uncorrected transcription for human review
settings = get_settings()
agentic_deadline = None
timeout_seconds = settings.agentic_correction_timeout_seconds
if settings.use_agentic_ai:
agentic_deadline = time.time() + timeout_seconds
job_log.info(
f"Agentic AI enabled with {timeout_seconds}s timeout"
)
# Run transcription in thread pool
# When agentic AI is enabled, wrap with outer timeout as safety net
# Inner deadline check in corrector.py will break out of gap loop gracefully
# Outer timeout catches case where single LLM call hangs for too long
transcription_coro = asyncio.to_thread(
lyrics_processor.transcribe_lyrics,
input_audio_wav=audio_path,
artist=job.artist, # Original artist for file naming
title=job.title, # Original title for file naming
track_output_dir=temp_dir,
lyrics_artist=lyrics_search_artist, # Override for lyrics search
lyrics_title=lyrics_search_title, # Override for lyrics search
agentic_deadline=agentic_deadline, # Deadline for agentic timeout
)
with metrics.time_external_api("audioshake", job_id):
if settings.use_agentic_ai:
# Apply agentic-specific timeout (shorter, with inner deadline check)
outer_timeout = timeout_seconds + 60 # Extra buffer for cleanup
try:
result = await asyncio.wait_for(
transcription_coro,
timeout=outer_timeout
)
except asyncio.TimeoutError:
# This should rarely trigger since inner deadline check breaks first
# But provides hard stop if e.g. single LLM call takes >3 min
job_log.error(
f"HARD TIMEOUT: Transcription exceeded {outer_timeout}s. "
"This indicates the inner deadline check failed to trigger."
)
raise RuntimeError(
f"Lyrics transcription timed out after {outer_timeout}s"
) from None
else:
# Non-agentic mode: use general timeout (10 minutes)
try:
result = await asyncio.wait_for(
transcription_coro,
timeout=TRANSCRIPTION_TIMEOUT_SECONDS
)
except asyncio.TimeoutError:
raise Exception(f"Transcription timed out after {TRANSCRIPTION_TIMEOUT_SECONDS} seconds")
trans_duration = time.time() - trans_start
trans_span.set_attribute("duration_seconds", trans_duration)
add_span_event("transcription_completed", {"duration_seconds": trans_duration})
+30
View File
@@ -304,6 +304,36 @@ Default 5-minute timeout may not be enough for video encoding. Consider:
- Breaking encoding into smaller chunks
- Increasing timeout for specific operations
### Two-Layer Timeout Pattern for Long-Running Loops
**Problem**: A loop processing many items (e.g., 74 LLM calls at 15s each = 18+ minutes) can block indefinitely. Single outer timeout isn't enough because it may interrupt mid-operation and leave state inconsistent.
**Solution**: Use two layers of timeout protection:
```python
# Layer 1: Inner deadline check (cooperative, clean exit)
deadline = time.time() + timeout_seconds
for item in items:
if time.time() > deadline:
logger.warning("Deadline exceeded, breaking early")
break # Exit gracefully with partial/no results
process(item)
# Layer 2: Outer asyncio.wait_for (safety net for hung operations)
result = await asyncio.wait_for(
asyncio.to_thread(run_loop_with_deadline),
timeout=timeout_seconds + 60 # Buffer for cleanup
)
```
**Why both layers**:
- Inner check: Handles normal case (many items). Exits cleanly between iterations.
- Outer timeout: Catches edge case (single operation hangs). Hard stop if inner check never runs.
**Design choice**: Use `break` not `raise` for inner timeout. This returns partial/uncorrected results rather than failing the job, allowing downstream human review to fix issues.
See `docs/archive/2025-12-31-agentic-timeout-implementation.md` for implementation details.
### Secret Manager Access
Cloud Run service account needs `Secret Manager Secret Accessor` role. This is managed via Pulumi in `infrastructure/`.
+1
View File
@@ -34,6 +34,7 @@
## Recent Changes
- **Agentic Correction Timeout** (2025-12-31): Added 3-minute configurable timeout for agentic AI lyrics correction. Prevents stuck jobs when songs have many gaps (74+ gaps could take 30+ minutes). On timeout, skips correction and proceeds to human review with raw transcription. See [archive/2025-12-31-agentic-timeout-implementation.md](archive/2025-12-31-agentic-timeout-implementation.md)
- **Job Failure Fixes** (2025-12-31): Fixed 4 critical issues causing job failures: (1) user_email not set on jobs - users couldn't see their jobs, (2) YouTube URL race condition - lyrics worker started before audio downloaded, (3) Audio search cache not persisting across Cloud Run instances, (4) Jobs stuck in downloading state - added 10min transcription timeout. See [archive/2024-12-31-job-failure-investigation.md](archive/2024-12-31-job-failure-investigation.md)
- **Vertex AI Auth Fix** (2025-12-31): Fixed `ChatGoogleGenerativeAI` to use Vertex AI backend with service account auth on Cloud Run. PR #145 broke cloud deployments by requiring `GOOGLE_API_KEY`. Solution: pass `project` parameter to trigger Vertex AI backend with ADC. See [archive/2025-12-31-vertex-ai-auth-fix.md](archive/2025-12-31-vertex-ai-auth-fix.md)
- **LangChain Provider Migration** (2025-12-30): Migrated from `langchain-google-vertexai` (gRPC) to `langchain-google-genai` (REST) to fix silent hangs during model initialization. Added initialization and warm-up timeouts using ThreadPoolExecutor for fail-fast behavior. See [archive/2025-12-30-langchain-vertexai-to-genai-migration.md](archive/2025-12-30-langchain-vertexai-to-genai-migration.md)
@@ -0,0 +1,68 @@
# Agentic AI Correction Timeout Implementation
**Date**: 2025-12-31
**PR**: #149
**Version**: 0.85.0
## Problem
Agentic AI lyrics correction was causing stuck jobs in production. Songs with many gaps (74+) could take 30+ minutes to process since each gap takes 10-30 seconds for LLM inference. This blocked the entire job pipeline.
Example from production logs:
```
Processing 22/74 gaps (~15s each) = ~18 minutes elapsed
Estimated total: 74 gaps × 15s = 18+ minutes
```
## Solution
Implemented a configurable 3-minute timeout (default) with two layers:
### Layer 1: Inner Deadline Check (Cooperative)
In `corrector.py`, check deadline before processing each gap:
```python
if deadline and time.time() > deadline:
self.logger.warning("AGENTIC TIMEOUT: Deadline exceeded...")
break # Return uncorrected transcription for human review
```
### Layer 2: Outer asyncio.wait_for (Safety Net)
In `lyrics_worker.py`, wrap the entire transcription call:
```python
result = await asyncio.wait_for(
asyncio.to_thread(lyrics_processor.transcribe_lyrics, ...),
timeout=timeout_seconds + 60 # Extra buffer
)
```
## Behavior
| Scenario | What Happens |
|----------|--------------|
| Completes in <3 min | Normal flow, corrections applied |
| Gap loop exceeds deadline | Inner check breaks, returns uncorrected |
| LLM hangs indefinitely | Outer timeout fires (rare), job fails |
| Any timeout | Job proceeds to human review with raw transcription |
## Files Changed
- `backend/config.py` - Added `agentic_correction_timeout_seconds` config
- `backend/workers/lyrics_worker.py` - Deadline calculation, outer timeout wrapper
- `karaoke_gen/lyrics_processor.py` - Pass deadline to controller
- `lyrics_transcriber_temp/lyrics_transcriber/core/controller.py` - Accept deadline parameter
- `lyrics_transcriber_temp/lyrics_transcriber/correction/corrector.py` - Deadline check in gap loop
## Configuration
Environment variable: `AGENTIC_CORRECTION_TIMEOUT_SECONDS` (default: 180)
## Key Design Decisions
1. **Break, don't raise**: Inner timeout uses `break` to exit the gap loop gracefully, returning whatever we have (uncorrected). This is better than raising an exception because:
- No retry logic needed
- Job continues to human review
- User can manually correct any issues
2. **Two-layer protection**: The inner check handles normal cases (many gaps). The outer timeout catches edge cases (single LLM call hanging). Both are needed for reliability.
3. **Configurable timeout**: Default 3 minutes balances correction quality vs. user experience. It can be adjusted via environment variable per deployment.
+9 -5
View File
@@ -3,6 +3,7 @@ import re
import logging
import shutil
import json
from typing import Optional
from lyrics_transcriber import LyricsTranscriber, OutputConfig, TranscriberConfig, LyricsConfig
from lyrics_transcriber.core.controller import LyricsControllerResult
from dotenv import load_dotenv
@@ -252,10 +253,11 @@ class LyricsProcessor:
"See README.md 'Transcription Providers' section for detailed setup instructions."
)
def transcribe_lyrics(self, input_audio_wav, artist, title, track_output_dir, lyrics_artist=None, lyrics_title=None):
def transcribe_lyrics(self, input_audio_wav, artist, title, track_output_dir, lyrics_artist=None, lyrics_title=None,
agentic_deadline: Optional[float] = None):
"""
Transcribe lyrics for a track.
Args:
input_audio_wav: Path to the audio file
artist: Original artist name (used for filename generation)
@@ -263,7 +265,9 @@ class LyricsProcessor:
track_output_dir: Output directory path
lyrics_artist: Artist name for lyrics processing (defaults to artist if None)
lyrics_title: Title for lyrics processing (defaults to title if None)
agentic_deadline: Optional Unix timestamp. If agentic correction is still
running after this time, it will abort and return uncorrected results.
Raises:
ValueError: If transcription is enabled but no providers are configured
"""
@@ -423,8 +427,8 @@ class LyricsProcessor:
logger=self.logger,
)
# Process and get results
results: LyricsControllerResult = transcriber.process()
# Process and get results (pass deadline for agentic timeout)
results: LyricsControllerResult = transcriber.process(agentic_deadline=agentic_deadline)
self.logger.info(f"Transcriber Results Filepaths:")
for key, value in results.__dict__.items():
if key.endswith("_filepath"):
@@ -289,8 +289,13 @@ class LyricsTranscriber:
"""Initialize output generation service."""
return OutputGenerator(config=self.output_config, logger=self.logger)
def process(self) -> LyricsControllerResult:
"""Main processing method that orchestrates the entire workflow."""
def process(self, agentic_deadline: Optional[float] = None) -> LyricsControllerResult:
"""Main processing method that orchestrates the entire workflow.
Args:
agentic_deadline: Optional Unix timestamp. If agentic correction is still
running after this time, it will abort and return uncorrected results.
"""
self.logger.info(f"LyricsTranscriber controller beginning processing for {self.artist} - {self.title}")
@@ -390,7 +395,7 @@ class LyricsTranscriber:
# Step 3: Process and correct lyrics if enabled AND we have transcription results
if self.output_config.run_correction and self.results.transcription_results:
self.correct_lyrics()
self.correct_lyrics(agentic_deadline=agentic_deadline)
elif self.output_config.run_correction:
self.logger.info("Skipping lyrics correction - no transcription results available")
@@ -494,8 +499,13 @@ class LyricsTranscriber:
else:
self.logger.debug(" - LocalWhisper: DISABLED (enable_local_whisper=False)")
def correct_lyrics(self) -> None:
"""Run lyrics correction using transcription and internet lyrics."""
def correct_lyrics(self, agentic_deadline: Optional[float] = None) -> None:
"""Run lyrics correction using transcription and internet lyrics.
Args:
agentic_deadline: Optional Unix timestamp. If agentic correction is still
running after this time, it will abort and return uncorrected results.
"""
self.logger.info("Starting lyrics correction process")
# Check if we have reference lyrics to work with
@@ -553,6 +563,7 @@ class LyricsTranscriber:
transcription_results=self.results.transcription_results,
lyrics_results=self.results.lyrics_results,
metadata=metadata,
agentic_deadline=agentic_deadline,
)
# Store corrected results
@@ -4,6 +4,7 @@ from pathlib import Path
from copy import deepcopy
import os
import shortuuid
import time
from lyrics_transcriber.correction.handlers.levenshtein import LevenshteinHandler
from lyrics_transcriber.correction.handlers.no_space_punct_match import NoSpacePunctuationMatchHandler
@@ -107,8 +108,24 @@ class LyricsCorrector:
transcription_results: List[TranscriptionResult],
lyrics_results: Dict[str, LyricsData],
metadata: Optional[Dict[str, Any]] = None,
agentic_deadline: Optional[float] = None,
) -> CorrectionResult:
"""Execute the correction process."""
"""Execute the correction process.
Args:
transcription_results: List of transcription results to correct.
lyrics_results: Dictionary of lyrics data from various sources.
metadata: Optional metadata including artist, title, audio file hash.
agentic_deadline: Optional Unix timestamp (from time.time()). If agentic
correction is still running after this time, it will abort and return
uncorrected results for human review.
Note:
The deadline is checked between gap iterations, not during LLM processing.
A single long-running LLM call may exceed the deadline. The caller should
wrap this method with an outer timeout (e.g., asyncio.wait_for) as a safety
net for hung operations.
"""
# Optional agentic routing flag from environment; default off for safety
agentic_enabled = os.getenv("USE_AGENTIC_AI", "").lower() in {"1", "true", "yes"}
self.logger.info(f"🤖 AGENTIC MODE: {'ENABLED' if agentic_enabled else 'DISABLED'} (USE_AGENTIC_AI={os.getenv('USE_AGENTIC_AI', 'NOT_SET')})")
@@ -132,9 +149,9 @@ class LyricsCorrector:
# Store anchor sequences for use in correction handlers
self._anchor_sequences = anchor_sequences
# Process corrections with metadata
# Process corrections with metadata and optional deadline for agentic timeout
corrections, corrected_segments, correction_steps, word_id_map, segment_id_map = self._process_corrections(
primary_transcription.segments, gap_sequences, metadata=metadata
primary_transcription.segments, gap_sequences, metadata=metadata, deadline=agentic_deadline
)
# Calculate correction ratio
@@ -178,10 +195,22 @@ class LyricsCorrector:
return leading_space + new_word.strip() + trailing_space
def _process_corrections(
self, segments: List[LyricsSegment], gap_sequences: List[GapSequence], metadata: Optional[Dict[str, Any]] = None
self, segments: List[LyricsSegment], gap_sequences: List[GapSequence], metadata: Optional[Dict[str, Any]] = None,
deadline: Optional[float] = None
) -> Tuple[List[WordCorrection], List[LyricsSegment], List[CorrectionStep], Dict[str, str], Dict[str, str]]:
"""Process corrections using handlers.
Args:
segments: List of lyrics segments to process.
gap_sequences: List of gap sequences to correct.
metadata: Optional metadata including artist, title, audio file hash.
deadline: Optional Unix timestamp (from time.time()). When agentic mode is
enabled and this deadline is exceeded, remaining gaps are skipped and
the method returns with whatever corrections have been made (likely none).
Returns:
Tuple of (corrections, corrected_segments, correction_steps, word_id_map, segment_id_map).
The correction flow works as follows:
1. First pass: Process all gaps
- Iterate through each gap sequence
@@ -415,6 +444,16 @@ class LyricsCorrector:
# === END TEMPORARY CODE ===
for i, gap in enumerate(gap_sequences, 1):
# Check deadline before processing each gap (agentic mode only)
# This allows us to abort early and return uncorrected results for human review
if deadline and use_agentic_env and time.time() > deadline:
self.logger.warning(
f"⏰ AGENTIC TIMEOUT: Deadline exceeded after processing {i-1}/{len(gap_sequences)} gaps. "
"Skipping remaining gaps - human review will correct any issues."
)
# Break out of loop - continue with whatever corrections we have (likely none)
break
self.logger.info(f"Processing gap {i}/{len(gap_sequences)} at position {gap.transcription_position}")
# Get the actual words for logging