fix(agentic): Use Gemini 3 Flash with global region and fix response parsing

Changes:
- Reverted to gemini-3-flash-preview (latest model with best capabilities)
- Changed GCP_LOCATION default from us-central1 to global (required for Gemini 3)
- Fixed multimodal response parsing for Gemini 3 models (returns list of content parts)
- Added local test script for agentic correction workflow

The test script (scripts/test_agentic_correction.py) allows testing the agentic
correction locally without deploying to Cloud Run, making iteration faster.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Andrew BeveridgeandClaude Opus 4.5 committed 2025-12-30 14:52:36 -05:00
1 parent 6e1f263ecd
commit 73e933d3b5
6 files changed
+372 -16

No files matched your search

+75
View File
@@ -0,0 +1,75 @@
[
{
"test_name": "Sound-alike error (wurld -> world)",
"success": true,
"proposals": [
{
"action": "ReplaceWord",
"word_id": null,
"word_ids": [
"w1"
],
"replacement_text": "world",
"confidence": 0.75,
"reason": "Sound-alike error. Reference suggests: 'world'. The transcription 'wurld' is a phonetic misspelling or sound-alike of the word 'world' which appears in both reference sources (Genius and LRCLIB).",
"gap_category": "GapCategory.SOUND_ALIKE",
"requires_human_review": false
}
]
},
{
"test_name": "Punctuation difference",
"success": true,
"proposals": [
{
"action": "NoAction",
"word_id": null,
"word_ids": [
"w2"
],
"replacement_text": null,
"confidence": 0.99,
"reason": "Transcription matches at least one reference source. The gap text 'Hello' followed by the context 'is it me you're looking for' matches the Genius reference lyrics exactly, excluding punctuation. According to the guidelines, if any reference source matches the transcription text (ignoring punctuation), it should be classified as NO_ERROR.",
"gap_category": "GapCategory.NO_ERROR",
"requires_human_review": false
}
]
},
{
"test_name": "Extra filler word",
"success": true,
"proposals": [
{
"action": "DeleteWord",
"word_id": "w3",
"word_ids": null,
"replacement_text": null,
"confidence": 0.8,
"reason": "Extra filler word 'And' at sentence start not in reference. The transcription includes the filler word 'And' at the beginning of the phrase, which is not present in the reference lyrics. This matches the definition of EXTRA_WORDS.",
"gap_category": "GapCategory.EXTRA_WORDS",
"requires_human_review": false
}
]
},
{
"test_name": "Complex multi-word error",
"success": true,
"proposals": [
{
"action": "NoAction",
"word_id": null,
"word_ids": [
"w7",
"w8",
"w9",
"w10"
],
"replacement_text": null,
"confidence": 0.99,
"reason": "Transcription matches at least one reference source. The transcription text 'gonna give you up' matches the Genius reference lyrics exactly when ignoring punctuation and capitalization. In context, the full phrase 'never gonna give you up' is correctly transcribed and aligns perfectly with the known lyrics and title of the song.",
"gap_category": "GapCategory.NO_ERROR",
"requires_human_review": false
}
]
}
]
@@ -19,8 +19,9 @@ class ProviderConfig:
cache_dir: str
# GCP/Vertex AI settings
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
gcp_project_id: Optional[str] = None
gcp_location: str = "us-central1"
gcp_location: str = "global"
request_timeout_seconds: float = 30.0
max_retries: int = 2
@@ -51,7 +52,7 @@ class ProviderConfig:
privacy_mode=os.getenv("PRIVACY_MODE", "false").lower() in {"1", "true", "yes"},
cache_dir=cache_dir,
gcp_project_id=os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID"),
gcp_location=os.getenv("GCP_LOCATION", "us-central1"),
gcp_location=os.getenv("GCP_LOCATION", "global"),
request_timeout_seconds=float(os.getenv("AGENTIC_TIMEOUT_SECONDS", "30.0")),
max_retries=int(os.getenv("AGENTIC_MAX_RETRIES", "2")),
retry_backoff_base_seconds=float(os.getenv("AGENTIC_BACKOFF_BASE_SECONDS", "0.2")),
@@ -187,26 +187,41 @@ class LangChainBridge(BaseAIProvider):
def _invoke_model(self, prompt: str) -> str:
"""Invoke the chat model with a prompt.
This is a simple wrapper that can be passed to the retry executor.
Args:
prompt: The prompt to send
Returns:
Response content as string
Raises:
Exception: Any error from the model invocation
"""
from langchain_core.messages import HumanMessage
# Prepare config with session_id in metadata (Langfuse format)
config = {}
if hasattr(self, '_session_id') and self._session_id:
config["metadata"] = {"langfuse_session_id": self._session_id}
logger.debug(f"🤖 [LangChain] Invoking with session_id: {self._session_id}")
response = self._chat_model.invoke([HumanMessage(content=prompt)], config=config)
return response.content
content = response.content
# Handle multimodal response format from Gemini 3+ models
# Response can be a list of content parts: [{'type': 'text', 'text': '...'}]
if isinstance(content, list):
# Extract text from the first text content part
for part in content:
if isinstance(part, dict) and part.get('type') == 'text':
return part.get('text', '')
# Fallback: concatenate all text parts
return ''.join(
part.get('text', '') if isinstance(part, dict) else str(part)
for part in content
)
return content
@@ -5,10 +5,9 @@ from typing import Dict, Any
from .providers.config import ProviderConfig
# Default model for cloud deployments - Gemini 2.0 Flash via Vertex AI
# Note: gemini-3-flash-preview requires 'global' location, not us-central1
# Using gemini-2.0-flash for broader regional availability
DEFAULT_CLOUD_MODEL = "vertexai/gemini-2.0-flash"
# Default model for cloud deployments - Gemini 3 Flash via Vertex AI
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
DEFAULT_CLOUD_MODEL = "vertexai/gemini-3-flash-preview"
class ModelRouter:
@@ -21,7 +20,7 @@ class ModelRouter:
"""Choose appropriate model based on gap characteristics.
Returns model identifier in format "provider/model" for LangChain:
- "vertexai/gemini-2.0-flash" for Gemini via Vertex AI (default)
- "vertexai/gemini-3-flash-preview" for Gemini via Vertex AI (default)
- "ollama/llama3.2:latest" for local Ollama models
- "openai/gpt-4" for OpenAI models
- "anthropic/claude-3-sonnet-20240229" for Anthropic models
@@ -35,7 +34,7 @@ class ModelRouter:
if self._config.privacy_mode:
return "ollama/llama3.2:latest"
# Default to Gemini 2.0 Flash for all cases (fast, cost-effective)
# Default to Gemini 3 Flash for all cases (fast, cost-effective, latest capabilities)
return DEFAULT_CLOUD_MODEL
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.81.4"
version = "0.81.5"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+266
View File
@@ -0,0 +1,266 @@
#!/usr/bin/env python3
"""
Test script for the agentic correction workflow.
This script allows testing the agentic correction locally without deploying
to Cloud Run or running a full karaoke job.
Usage:
# Test with default Vertex AI (requires gcloud auth):
python scripts/test_agentic_correction.py
# Test with a specific model:
AGENTIC_AI_MODEL=openai/gpt-4o python scripts/test_agentic_correction.py
AGENTIC_AI_MODEL=anthropic/claude-3-5-sonnet-20241022 python scripts/test_agentic_correction.py
# Test with local Ollama:
PRIVACY_MODE=1 python scripts/test_agentic_correction.py
Environment variables:
GOOGLE_CLOUD_PROJECT: GCP project ID (required for Vertex AI)
GCP_LOCATION: Vertex AI location (default: global)
AGENTIC_AI_MODEL: Override the model (e.g., openai/gpt-4o)
PRIVACY_MODE: Set to 1 to use local Ollama
LANGFUSE_PUBLIC_KEY: Langfuse public key for tracing
LANGFUSE_SECRET_KEY: Langfuse secret key for tracing
"""
import os
import sys
import json
import logging
from typing import List, Dict, Any
# Add the project to the path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Configure logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
datefmt="%H:%M:%S"
)
logger = logging.getLogger(__name__)
# Sample test cases representing different gap categories
TEST_CASES = [
{
"name": "Sound-alike error (wurld -> world)",
"gap_id": "test_gap_1",
"gap_words": [
{"id": "w1", "text": "wurld", "start_time": 0.5, "end_time": 1.0}
],
"preceding_words": "hello",
"following_words": "this is a test",
"reference_contexts": {
"genius": "hello world this is a test",
"lrclib": "Hello world, this is a test"
},
"artist": "Test Artist",
"title": "Test Song"
},
{
"name": "Punctuation difference",
"gap_id": "test_gap_2",
"gap_words": [
{"id": "w2", "text": "Hello", "start_time": 0.0, "end_time": 0.5}
],
"preceding_words": "",
"following_words": "is it me you're looking for",
"reference_contexts": {
"genius": "Hello, is it me you're looking for?"
},
"artist": "Lionel Richie",
"title": "Hello"
},
{
"name": "Extra filler word",
"gap_id": "test_gap_3",
"gap_words": [
{"id": "w3", "text": "And", "start_time": 2.0, "end_time": 2.2},
{"id": "w4", "text": "I", "start_time": 2.2, "end_time": 2.4},
{"id": "w5", "text": "love", "start_time": 2.4, "end_time": 2.6},
{"id": "w6", "text": "you", "start_time": 2.6, "end_time": 2.8}
],
"preceding_words": "baby",
"following_words": "so much",
"reference_contexts": {
"genius": "baby I love you so much"
},
"artist": "Test Artist",
"title": "Love Song"
},
{
"name": "Complex multi-word error",
"gap_id": "test_gap_4",
"gap_words": [
{"id": "w7", "text": "gonna", "start_time": 3.0, "end_time": 3.2},
{"id": "w8", "text": "give", "start_time": 3.2, "end_time": 3.4},
{"id": "w9", "text": "you", "start_time": 3.4, "end_time": 3.6},
{"id": "w10", "text": "up", "start_time": 3.6, "end_time": 3.8}
],
"preceding_words": "never",
"following_words": "never gonna let you down",
"reference_contexts": {
"genius": "Never gonna give you up, never gonna let you down"
},
"artist": "Rick Astley",
"title": "Never Gonna Give You Up"
}
]
def run_single_test(test_case: Dict[str, Any], agent) -> Dict[str, Any]:
"""Run a single test case and return results."""
logger.info(f"\n{'='*60}")
logger.info(f"Test: {test_case['name']}")
logger.info(f"Gap words: {[w['text'] for w in test_case['gap_words']]}")
logger.info(f"Reference: {list(test_case['reference_contexts'].values())[0][:50]}...")
logger.info(f"{'='*60}")
try:
proposals = agent.propose_for_gap(
gap_id=test_case["gap_id"],
gap_words=test_case["gap_words"],
preceding_words=test_case["preceding_words"],
following_words=test_case["following_words"],
reference_contexts=test_case["reference_contexts"],
artist=test_case.get("artist"),
title=test_case.get("title")
)
result = {
"test_name": test_case["name"],
"success": True,
"proposals": []
}
if not proposals:
logger.info("No proposals returned (gap may be correct)")
result["proposals"] = []
else:
for p in proposals:
proposal_dict = {
"action": p.action,
"word_id": p.word_id,
"word_ids": p.word_ids,
"replacement_text": p.replacement_text,
"confidence": p.confidence,
"reason": p.reason,
"gap_category": str(p.gap_category) if p.gap_category else None,
"requires_human_review": p.requires_human_review
}
result["proposals"].append(proposal_dict)
logger.info(f"\nProposal:")
logger.info(f" Action: {p.action}")
logger.info(f" Word ID(s): {p.word_id or p.word_ids}")
if p.replacement_text:
logger.info(f" Replacement: {p.replacement_text}")
logger.info(f" Confidence: {p.confidence:.2f}")
logger.info(f" Category: {p.gap_category}")
logger.info(f" Reason: {p.reason}")
if p.requires_human_review:
logger.info(f" ⚠️ Requires human review")
return result
except Exception as e:
logger.error(f"Test failed with error: {e}")
import traceback
traceback.print_exc()
return {
"test_name": test_case["name"],
"success": False,
"error": str(e)
}
def main():
"""Main entry point."""
# Check environment
project_id = os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID")
location = os.getenv("GCP_LOCATION", "global")
model_override = os.getenv("AGENTIC_AI_MODEL")
privacy_mode = os.getenv("PRIVACY_MODE", "").lower() in ("1", "true", "yes")
logger.info("=" * 60)
logger.info("AGENTIC CORRECTION TEST SCRIPT")
logger.info("=" * 60)
logger.info(f"GCP Project: {project_id or 'not set'}")
logger.info(f"GCP Location: {location}")
logger.info(f"Model Override: {model_override or 'none (using default)'}")
logger.info(f"Privacy Mode: {privacy_mode}")
logger.info(f"Langfuse: {'enabled' if os.getenv('LANGFUSE_PUBLIC_KEY') else 'disabled'}")
# Import after path setup
try:
from lyrics_transcriber.correction.agentic.agent import AgenticCorrector
from lyrics_transcriber.correction.agentic.router import ModelRouter
from lyrics_transcriber.correction.agentic.providers.config import ProviderConfig
except ImportError as e:
logger.error(f"Failed to import agentic correction modules: {e}")
logger.error("Make sure you're running from the karaoke-gen directory")
sys.exit(1)
# Determine which model to use
config = ProviderConfig.from_env()
router = ModelRouter(config)
model = router.choose_model("unknown", 0.5)
logger.info(f"Selected model: {model}")
# Validate environment for Vertex AI
if model.startswith("vertexai/") and not project_id:
logger.error("GOOGLE_CLOUD_PROJECT environment variable is required for Vertex AI")
logger.error("Set it with: export GOOGLE_CLOUD_PROJECT=your-project-id")
logger.error("Or use a different model: AGENTIC_AI_MODEL=openai/gpt-4o")
sys.exit(1)
# Create the agent
logger.info(f"\nCreating AgenticCorrector with model: {model}")
try:
agent = AgenticCorrector.from_model(
model=model,
session_id="test_session_local",
cache_dir=os.path.expanduser("~/lyrics-transcriber-cache")
)
logger.info("Agent created successfully!")
except Exception as e:
logger.error(f"Failed to create agent: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
# Run test cases
results = []
for test_case in TEST_CASES:
result = run_single_test(test_case, agent)
results.append(result)
# Summary
logger.info("\n" + "=" * 60)
logger.info("TEST SUMMARY")
logger.info("=" * 60)
passed = sum(1 for r in results if r["success"])
failed = len(results) - passed
for r in results:
status = "✅ PASS" if r["success"] else "❌ FAIL"
logger.info(f"{status}: {r['test_name']}")
if not r["success"]:
logger.info(f" Error: {r.get('error', 'unknown')}")
logger.info(f"\nResults: {passed}/{len(results)} passed")
# Output JSON results
output_file = "agentic_test_results.json"
with open(output_file, "w") as f:
json.dump(results, f, indent=2)
logger.info(f"\nDetailed results saved to: {output_file}")
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(main())