mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(agentic): Use Gemini 3 Flash with global region and fix response parsing
Changes: - Reverted to gemini-3-flash-preview (latest model with best capabilities) - Changed GCP_LOCATION default from us-central1 to global (required for Gemini 3) - Fixed multimodal response parsing for Gemini 3 models (returns list of content parts) - Added local test script for agentic correction workflow The test script (scripts/test_agentic_correction.py) allows testing the agentic correction locally without deploying to Cloud Run, making iteration faster. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6e1f263ecd
commit
73e933d3b5
6 files changed
+366
-10
No files matched your search
@@ -0,0 +1,75 @@
|
||||
[
|
||||
{
|
||||
"test_name": "Sound-alike error (wurld -> world)",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "ReplaceWord",
|
||||
"word_id": null,
|
||||
"word_ids": [
|
||||
"w1"
|
||||
],
|
||||
"replacement_text": "world",
|
||||
"confidence": 0.75,
|
||||
"reason": "Sound-alike error. Reference suggests: 'world'. The transcription 'wurld' is a phonetic misspelling or sound-alike of the word 'world' which appears in both reference sources (Genius and LRCLIB).",
|
||||
"gap_category": "GapCategory.SOUND_ALIKE",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "Punctuation difference",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "NoAction",
|
||||
"word_id": null,
|
||||
"word_ids": [
|
||||
"w2"
|
||||
],
|
||||
"replacement_text": null,
|
||||
"confidence": 0.99,
|
||||
"reason": "Transcription matches at least one reference source. The gap text 'Hello' followed by the context 'is it me you're looking for' matches the Genius reference lyrics exactly, excluding punctuation. According to the guidelines, if any reference source matches the transcription text (ignoring punctuation), it should be classified as NO_ERROR.",
|
||||
"gap_category": "GapCategory.NO_ERROR",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "Extra filler word",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "DeleteWord",
|
||||
"word_id": "w3",
|
||||
"word_ids": null,
|
||||
"replacement_text": null,
|
||||
"confidence": 0.8,
|
||||
"reason": "Extra filler word 'And' at sentence start not in reference. The transcription includes the filler word 'And' at the beginning of the phrase, which is not present in the reference lyrics. This matches the definition of EXTRA_WORDS.",
|
||||
"gap_category": "GapCategory.EXTRA_WORDS",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "Complex multi-word error",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "NoAction",
|
||||
"word_id": null,
|
||||
"word_ids": [
|
||||
"w7",
|
||||
"w8",
|
||||
"w9",
|
||||
"w10"
|
||||
],
|
||||
"replacement_text": null,
|
||||
"confidence": 0.99,
|
||||
"reason": "Transcription matches at least one reference source. The transcription text 'gonna give you up' matches the Genius reference lyrics exactly when ignoring punctuation and capitalization. In context, the full phrase 'never gonna give you up' is correctly transcribed and aligns perfectly with the known lyrics and title of the song.",
|
||||
"gap_category": "GapCategory.NO_ERROR",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -19,8 +19,9 @@ class ProviderConfig:
|
||||
cache_dir: str
|
||||
|
||||
# GCP/Vertex AI settings
|
||||
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
|
||||
gcp_project_id: Optional[str] = None
|
||||
gcp_location: str = "us-central1"
|
||||
gcp_location: str = "global"
|
||||
|
||||
request_timeout_seconds: float = 30.0
|
||||
max_retries: int = 2
|
||||
@@ -51,7 +52,7 @@ class ProviderConfig:
|
||||
privacy_mode=os.getenv("PRIVACY_MODE", "false").lower() in {"1", "true", "yes"},
|
||||
cache_dir=cache_dir,
|
||||
gcp_project_id=os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID"),
|
||||
gcp_location=os.getenv("GCP_LOCATION", "us-central1"),
|
||||
gcp_location=os.getenv("GCP_LOCATION", "global"),
|
||||
request_timeout_seconds=float(os.getenv("AGENTIC_TIMEOUT_SECONDS", "30.0")),
|
||||
max_retries=int(os.getenv("AGENTIC_MAX_RETRIES", "2")),
|
||||
retry_backoff_base_seconds=float(os.getenv("AGENTIC_BACKOFF_BASE_SECONDS", "0.2")),
|
||||
|
||||
+16
-1
@@ -208,5 +208,20 @@ class LangChainBridge(BaseAIProvider):
|
||||
logger.debug(f"🤖 [LangChain] Invoking with session_id: {self._session_id}")
|
||||
|
||||
response = self._chat_model.invoke([HumanMessage(content=prompt)], config=config)
|
||||
return response.content
|
||||
content = response.content
|
||||
|
||||
# Handle multimodal response format from Gemini 3+ models
|
||||
# Response can be a list of content parts: [{'type': 'text', 'text': '...'}]
|
||||
if isinstance(content, list):
|
||||
# Extract text from the first text content part
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get('type') == 'text':
|
||||
return part.get('text', '')
|
||||
# Fallback: concatenate all text parts
|
||||
return ''.join(
|
||||
part.get('text', '') if isinstance(part, dict) else str(part)
|
||||
for part in content
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
@@ -5,10 +5,9 @@ from typing import Dict, Any
|
||||
|
||||
from .providers.config import ProviderConfig
|
||||
|
||||
# Default model for cloud deployments - Gemini 2.0 Flash via Vertex AI
|
||||
# Note: gemini-3-flash-preview requires 'global' location, not us-central1
|
||||
# Using gemini-2.0-flash for broader regional availability
|
||||
DEFAULT_CLOUD_MODEL = "vertexai/gemini-2.0-flash"
|
||||
# Default model for cloud deployments - Gemini 3 Flash via Vertex AI
|
||||
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
|
||||
DEFAULT_CLOUD_MODEL = "vertexai/gemini-3-flash-preview"
|
||||
|
||||
|
||||
class ModelRouter:
|
||||
@@ -21,7 +20,7 @@ class ModelRouter:
|
||||
"""Choose appropriate model based on gap characteristics.
|
||||
|
||||
Returns model identifier in format "provider/model" for LangChain:
|
||||
- "vertexai/gemini-2.0-flash" for Gemini via Vertex AI (default)
|
||||
- "vertexai/gemini-3-flash-preview" for Gemini via Vertex AI (default)
|
||||
- "ollama/llama3.2:latest" for local Ollama models
|
||||
- "openai/gpt-4" for OpenAI models
|
||||
- "anthropic/claude-3-sonnet-20240229" for Anthropic models
|
||||
@@ -35,7 +34,7 @@ class ModelRouter:
|
||||
if self._config.privacy_mode:
|
||||
return "ollama/llama3.2:latest"
|
||||
|
||||
# Default to Gemini 2.0 Flash for all cases (fast, cost-effective)
|
||||
# Default to Gemini 3 Flash for all cases (fast, cost-effective, latest capabilities)
|
||||
return DEFAULT_CLOUD_MODEL
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.81.4"
|
||||
version = "0.81.5"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Executable
+266
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for the agentic correction workflow.
|
||||
|
||||
This script allows testing the agentic correction locally without deploying
|
||||
to Cloud Run or running a full karaoke job.
|
||||
|
||||
Usage:
|
||||
# Test with default Vertex AI (requires gcloud auth):
|
||||
python scripts/test_agentic_correction.py
|
||||
|
||||
# Test with a specific model:
|
||||
AGENTIC_AI_MODEL=openai/gpt-4o python scripts/test_agentic_correction.py
|
||||
AGENTIC_AI_MODEL=anthropic/claude-3-5-sonnet-20241022 python scripts/test_agentic_correction.py
|
||||
|
||||
# Test with local Ollama:
|
||||
PRIVACY_MODE=1 python scripts/test_agentic_correction.py
|
||||
|
||||
Environment variables:
|
||||
GOOGLE_CLOUD_PROJECT: GCP project ID (required for Vertex AI)
|
||||
GCP_LOCATION: Vertex AI location (default: global)
|
||||
AGENTIC_AI_MODEL: Override the model (e.g., openai/gpt-4o)
|
||||
PRIVACY_MODE: Set to 1 to use local Ollama
|
||||
LANGFUSE_PUBLIC_KEY: Langfuse public key for tracing
|
||||
LANGFUSE_SECRET_KEY: Langfuse secret key for tracing
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Dict, Any
|
||||
|
||||
# Add the project to the path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
datefmt="%H:%M:%S"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sample test cases representing different gap categories
|
||||
TEST_CASES = [
|
||||
{
|
||||
"name": "Sound-alike error (wurld -> world)",
|
||||
"gap_id": "test_gap_1",
|
||||
"gap_words": [
|
||||
{"id": "w1", "text": "wurld", "start_time": 0.5, "end_time": 1.0}
|
||||
],
|
||||
"preceding_words": "hello",
|
||||
"following_words": "this is a test",
|
||||
"reference_contexts": {
|
||||
"genius": "hello world this is a test",
|
||||
"lrclib": "Hello world, this is a test"
|
||||
},
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
},
|
||||
{
|
||||
"name": "Punctuation difference",
|
||||
"gap_id": "test_gap_2",
|
||||
"gap_words": [
|
||||
{"id": "w2", "text": "Hello", "start_time": 0.0, "end_time": 0.5}
|
||||
],
|
||||
"preceding_words": "",
|
||||
"following_words": "is it me you're looking for",
|
||||
"reference_contexts": {
|
||||
"genius": "Hello, is it me you're looking for?"
|
||||
},
|
||||
"artist": "Lionel Richie",
|
||||
"title": "Hello"
|
||||
},
|
||||
{
|
||||
"name": "Extra filler word",
|
||||
"gap_id": "test_gap_3",
|
||||
"gap_words": [
|
||||
{"id": "w3", "text": "And", "start_time": 2.0, "end_time": 2.2},
|
||||
{"id": "w4", "text": "I", "start_time": 2.2, "end_time": 2.4},
|
||||
{"id": "w5", "text": "love", "start_time": 2.4, "end_time": 2.6},
|
||||
{"id": "w6", "text": "you", "start_time": 2.6, "end_time": 2.8}
|
||||
],
|
||||
"preceding_words": "baby",
|
||||
"following_words": "so much",
|
||||
"reference_contexts": {
|
||||
"genius": "baby I love you so much"
|
||||
},
|
||||
"artist": "Test Artist",
|
||||
"title": "Love Song"
|
||||
},
|
||||
{
|
||||
"name": "Complex multi-word error",
|
||||
"gap_id": "test_gap_4",
|
||||
"gap_words": [
|
||||
{"id": "w7", "text": "gonna", "start_time": 3.0, "end_time": 3.2},
|
||||
{"id": "w8", "text": "give", "start_time": 3.2, "end_time": 3.4},
|
||||
{"id": "w9", "text": "you", "start_time": 3.4, "end_time": 3.6},
|
||||
{"id": "w10", "text": "up", "start_time": 3.6, "end_time": 3.8}
|
||||
],
|
||||
"preceding_words": "never",
|
||||
"following_words": "never gonna let you down",
|
||||
"reference_contexts": {
|
||||
"genius": "Never gonna give you up, never gonna let you down"
|
||||
},
|
||||
"artist": "Rick Astley",
|
||||
"title": "Never Gonna Give You Up"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def run_single_test(test_case: Dict[str, Any], agent) -> Dict[str, Any]:
|
||||
"""Run a single test case and return results."""
|
||||
logger.info(f"\n{'='*60}")
|
||||
logger.info(f"Test: {test_case['name']}")
|
||||
logger.info(f"Gap words: {[w['text'] for w in test_case['gap_words']]}")
|
||||
logger.info(f"Reference: {list(test_case['reference_contexts'].values())[0][:50]}...")
|
||||
logger.info(f"{'='*60}")
|
||||
|
||||
try:
|
||||
proposals = agent.propose_for_gap(
|
||||
gap_id=test_case["gap_id"],
|
||||
gap_words=test_case["gap_words"],
|
||||
preceding_words=test_case["preceding_words"],
|
||||
following_words=test_case["following_words"],
|
||||
reference_contexts=test_case["reference_contexts"],
|
||||
artist=test_case.get("artist"),
|
||||
title=test_case.get("title")
|
||||
)
|
||||
|
||||
result = {
|
||||
"test_name": test_case["name"],
|
||||
"success": True,
|
||||
"proposals": []
|
||||
}
|
||||
|
||||
if not proposals:
|
||||
logger.info("No proposals returned (gap may be correct)")
|
||||
result["proposals"] = []
|
||||
else:
|
||||
for p in proposals:
|
||||
proposal_dict = {
|
||||
"action": p.action,
|
||||
"word_id": p.word_id,
|
||||
"word_ids": p.word_ids,
|
||||
"replacement_text": p.replacement_text,
|
||||
"confidence": p.confidence,
|
||||
"reason": p.reason,
|
||||
"gap_category": str(p.gap_category) if p.gap_category else None,
|
||||
"requires_human_review": p.requires_human_review
|
||||
}
|
||||
result["proposals"].append(proposal_dict)
|
||||
|
||||
logger.info(f"\nProposal:")
|
||||
logger.info(f" Action: {p.action}")
|
||||
logger.info(f" Word ID(s): {p.word_id or p.word_ids}")
|
||||
if p.replacement_text:
|
||||
logger.info(f" Replacement: {p.replacement_text}")
|
||||
logger.info(f" Confidence: {p.confidence:.2f}")
|
||||
logger.info(f" Category: {p.gap_category}")
|
||||
logger.info(f" Reason: {p.reason}")
|
||||
if p.requires_human_review:
|
||||
logger.info(f" ⚠️ Requires human review")
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return {
|
||||
"test_name": test_case["name"],
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
# Check environment
|
||||
project_id = os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID")
|
||||
location = os.getenv("GCP_LOCATION", "global")
|
||||
model_override = os.getenv("AGENTIC_AI_MODEL")
|
||||
privacy_mode = os.getenv("PRIVACY_MODE", "").lower() in ("1", "true", "yes")
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("AGENTIC CORRECTION TEST SCRIPT")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"GCP Project: {project_id or 'not set'}")
|
||||
logger.info(f"GCP Location: {location}")
|
||||
logger.info(f"Model Override: {model_override or 'none (using default)'}")
|
||||
logger.info(f"Privacy Mode: {privacy_mode}")
|
||||
logger.info(f"Langfuse: {'enabled' if os.getenv('LANGFUSE_PUBLIC_KEY') else 'disabled'}")
|
||||
|
||||
# Import after path setup
|
||||
try:
|
||||
from lyrics_transcriber.correction.agentic.agent import AgenticCorrector
|
||||
from lyrics_transcriber.correction.agentic.router import ModelRouter
|
||||
from lyrics_transcriber.correction.agentic.providers.config import ProviderConfig
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import agentic correction modules: {e}")
|
||||
logger.error("Make sure you're running from the karaoke-gen directory")
|
||||
sys.exit(1)
|
||||
|
||||
# Determine which model to use
|
||||
config = ProviderConfig.from_env()
|
||||
router = ModelRouter(config)
|
||||
model = router.choose_model("unknown", 0.5)
|
||||
logger.info(f"Selected model: {model}")
|
||||
|
||||
# Validate environment for Vertex AI
|
||||
if model.startswith("vertexai/") and not project_id:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT environment variable is required for Vertex AI")
|
||||
logger.error("Set it with: export GOOGLE_CLOUD_PROJECT=your-project-id")
|
||||
logger.error("Or use a different model: AGENTIC_AI_MODEL=openai/gpt-4o")
|
||||
sys.exit(1)
|
||||
|
||||
# Create the agent
|
||||
logger.info(f"\nCreating AgenticCorrector with model: {model}")
|
||||
try:
|
||||
agent = AgenticCorrector.from_model(
|
||||
model=model,
|
||||
session_id="test_session_local",
|
||||
cache_dir=os.path.expanduser("~/lyrics-transcriber-cache")
|
||||
)
|
||||
logger.info("Agent created successfully!")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create agent: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
# Run test cases
|
||||
results = []
|
||||
for test_case in TEST_CASES:
|
||||
result = run_single_test(test_case, agent)
|
||||
results.append(result)
|
||||
|
||||
# Summary
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("TEST SUMMARY")
|
||||
logger.info("=" * 60)
|
||||
|
||||
passed = sum(1 for r in results if r["success"])
|
||||
failed = len(results) - passed
|
||||
|
||||
for r in results:
|
||||
status = "✅ PASS" if r["success"] else "❌ FAIL"
|
||||
logger.info(f"{status}: {r['test_name']}")
|
||||
if not r["success"]:
|
||||
logger.info(f" Error: {r.get('error', 'unknown')}")
|
||||
|
||||
logger.info(f"\nResults: {passed}/{len(results)} passed")
|
||||
|
||||
# Output JSON results
|
||||
output_file = "agentic_test_results.json"
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
logger.info(f"\nDetailed results saved to: {output_file}")
|
||||
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user