diff --git a/agentic_test_results.json b/agentic_test_results.json new file mode 100644 index 00000000..89f1a457 --- /dev/null +++ b/agentic_test_results.json @@ -0,0 +1,75 @@ +[ + { + "test_name": "Sound-alike error (wurld -> world)", + "success": true, + "proposals": [ + { + "action": "ReplaceWord", + "word_id": null, + "word_ids": [ + "w1" + ], + "replacement_text": "world", + "confidence": 0.75, + "reason": "Sound-alike error. Reference suggests: 'world'. The transcription 'wurld' is a phonetic misspelling or sound-alike of the word 'world' which appears in both reference sources (Genius and LRCLIB).", + "gap_category": "GapCategory.SOUND_ALIKE", + "requires_human_review": false + } + ] + }, + { + "test_name": "Punctuation difference", + "success": true, + "proposals": [ + { + "action": "NoAction", + "word_id": null, + "word_ids": [ + "w2" + ], + "replacement_text": null, + "confidence": 0.99, + "reason": "Transcription matches at least one reference source. The gap text 'Hello' followed by the context 'is it me you're looking for' matches the Genius reference lyrics exactly, excluding punctuation. According to the guidelines, if any reference source matches the transcription text (ignoring punctuation), it should be classified as NO_ERROR.", + "gap_category": "GapCategory.NO_ERROR", + "requires_human_review": false + } + ] + }, + { + "test_name": "Extra filler word", + "success": true, + "proposals": [ + { + "action": "DeleteWord", + "word_id": "w3", + "word_ids": null, + "replacement_text": null, + "confidence": 0.8, + "reason": "Extra filler word 'And' at sentence start not in reference. The transcription includes the filler word 'And' at the beginning of the phrase, which is not present in the reference lyrics. This matches the definition of EXTRA_WORDS.", + "gap_category": "GapCategory.EXTRA_WORDS", + "requires_human_review": false + } + ] + }, + { + "test_name": "Complex multi-word error", + "success": true, + "proposals": [ + { + "action": "NoAction", + "word_id": null, + "word_ids": [ + "w7", + "w8", + "w9", + "w10" + ], + "replacement_text": null, + "confidence": 0.99, + "reason": "Transcription matches at least one reference source. The transcription text 'gonna give you up' matches the Genius reference lyrics exactly when ignoring punctuation and capitalization. In context, the full phrase 'never gonna give you up' is correctly transcribed and aligns perfectly with the known lyrics and title of the song.", + "gap_category": "GapCategory.NO_ERROR", + "requires_human_review": false + } + ] + } +] \ No newline at end of file diff --git a/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/config.py b/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/config.py index ce160ab6..ff275733 100644 --- a/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/config.py +++ b/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/config.py @@ -19,8 +19,9 @@ class ProviderConfig: cache_dir: str # GCP/Vertex AI settings + # Note: Gemini 3 models require 'global' location (not regional like us-central1) gcp_project_id: Optional[str] = None - gcp_location: str = "us-central1" + gcp_location: str = "global" request_timeout_seconds: float = 30.0 max_retries: int = 2 @@ -51,7 +52,7 @@ class ProviderConfig: privacy_mode=os.getenv("PRIVACY_MODE", "false").lower() in {"1", "true", "yes"}, cache_dir=cache_dir, gcp_project_id=os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID"), - gcp_location=os.getenv("GCP_LOCATION", "us-central1"), + gcp_location=os.getenv("GCP_LOCATION", "global"), request_timeout_seconds=float(os.getenv("AGENTIC_TIMEOUT_SECONDS", "30.0")), max_retries=int(os.getenv("AGENTIC_MAX_RETRIES", "2")), retry_backoff_base_seconds=float(os.getenv("AGENTIC_BACKOFF_BASE_SECONDS", "0.2")), diff --git a/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/langchain_bridge.py b/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/langchain_bridge.py index d7ca9881..348275cc 100644 --- a/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/langchain_bridge.py +++ b/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/langchain_bridge.py @@ -187,26 +187,41 @@ class LangChainBridge(BaseAIProvider): def _invoke_model(self, prompt: str) -> str: """Invoke the chat model with a prompt. - + This is a simple wrapper that can be passed to the retry executor. - + Args: prompt: The prompt to send - + Returns: Response content as string - + Raises: Exception: Any error from the model invocation """ from langchain_core.messages import HumanMessage - + # Prepare config with session_id in metadata (Langfuse format) config = {} if hasattr(self, '_session_id') and self._session_id: config["metadata"] = {"langfuse_session_id": self._session_id} logger.debug(f"🤖 [LangChain] Invoking with session_id: {self._session_id}") - + response = self._chat_model.invoke([HumanMessage(content=prompt)], config=config) - return response.content + content = response.content + + # Handle multimodal response format from Gemini 3+ models + # Response can be a list of content parts: [{'type': 'text', 'text': '...'}] + if isinstance(content, list): + # Extract text from the first text content part + for part in content: + if isinstance(part, dict) and part.get('type') == 'text': + return part.get('text', '') + # Fallback: concatenate all text parts + return ''.join( + part.get('text', '') if isinstance(part, dict) else str(part) + for part in content + ) + + return content diff --git a/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/router.py b/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/router.py index 4eca4e13..2e7087d9 100644 --- a/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/router.py +++ b/lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/router.py @@ -5,10 +5,9 @@ from typing import Dict, Any from .providers.config import ProviderConfig -# Default model for cloud deployments - Gemini 2.0 Flash via Vertex AI -# Note: gemini-3-flash-preview requires 'global' location, not us-central1 -# Using gemini-2.0-flash for broader regional availability -DEFAULT_CLOUD_MODEL = "vertexai/gemini-2.0-flash" +# Default model for cloud deployments - Gemini 3 Flash via Vertex AI +# Note: Gemini 3 models require 'global' location (not regional like us-central1) +DEFAULT_CLOUD_MODEL = "vertexai/gemini-3-flash-preview" class ModelRouter: @@ -21,7 +20,7 @@ class ModelRouter: """Choose appropriate model based on gap characteristics. Returns model identifier in format "provider/model" for LangChain: - - "vertexai/gemini-2.0-flash" for Gemini via Vertex AI (default) + - "vertexai/gemini-3-flash-preview" for Gemini via Vertex AI (default) - "ollama/llama3.2:latest" for local Ollama models - "openai/gpt-4" for OpenAI models - "anthropic/claude-3-sonnet-20240229" for Anthropic models @@ -35,7 +34,7 @@ class ModelRouter: if self._config.privacy_mode: return "ollama/llama3.2:latest" - # Default to Gemini 2.0 Flash for all cases (fast, cost-effective) + # Default to Gemini 3 Flash for all cases (fast, cost-effective, latest capabilities) return DEFAULT_CLOUD_MODEL diff --git a/pyproject.toml b/pyproject.toml index fb5e27c3..e0a6fc1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.81.4" +version = "0.81.5" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/test_agentic_correction.py b/scripts/test_agentic_correction.py new file mode 100755 index 00000000..9a1368f0 --- /dev/null +++ b/scripts/test_agentic_correction.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +""" +Test script for the agentic correction workflow. + +This script allows testing the agentic correction locally without deploying +to Cloud Run or running a full karaoke job. + +Usage: + # Test with default Vertex AI (requires gcloud auth): + python scripts/test_agentic_correction.py + + # Test with a specific model: + AGENTIC_AI_MODEL=openai/gpt-4o python scripts/test_agentic_correction.py + AGENTIC_AI_MODEL=anthropic/claude-3-5-sonnet-20241022 python scripts/test_agentic_correction.py + + # Test with local Ollama: + PRIVACY_MODE=1 python scripts/test_agentic_correction.py + +Environment variables: + GOOGLE_CLOUD_PROJECT: GCP project ID (required for Vertex AI) + GCP_LOCATION: Vertex AI location (default: global) + AGENTIC_AI_MODEL: Override the model (e.g., openai/gpt-4o) + PRIVACY_MODE: Set to 1 to use local Ollama + LANGFUSE_PUBLIC_KEY: Langfuse public key for tracing + LANGFUSE_SECRET_KEY: Langfuse secret key for tracing +""" + +import os +import sys +import json +import logging +from typing import List, Dict, Any + +# Add the project to the path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + datefmt="%H:%M:%S" +) +logger = logging.getLogger(__name__) + +# Sample test cases representing different gap categories +TEST_CASES = [ + { + "name": "Sound-alike error (wurld -> world)", + "gap_id": "test_gap_1", + "gap_words": [ + {"id": "w1", "text": "wurld", "start_time": 0.5, "end_time": 1.0} + ], + "preceding_words": "hello", + "following_words": "this is a test", + "reference_contexts": { + "genius": "hello world this is a test", + "lrclib": "Hello world, this is a test" + }, + "artist": "Test Artist", + "title": "Test Song" + }, + { + "name": "Punctuation difference", + "gap_id": "test_gap_2", + "gap_words": [ + {"id": "w2", "text": "Hello", "start_time": 0.0, "end_time": 0.5} + ], + "preceding_words": "", + "following_words": "is it me you're looking for", + "reference_contexts": { + "genius": "Hello, is it me you're looking for?" + }, + "artist": "Lionel Richie", + "title": "Hello" + }, + { + "name": "Extra filler word", + "gap_id": "test_gap_3", + "gap_words": [ + {"id": "w3", "text": "And", "start_time": 2.0, "end_time": 2.2}, + {"id": "w4", "text": "I", "start_time": 2.2, "end_time": 2.4}, + {"id": "w5", "text": "love", "start_time": 2.4, "end_time": 2.6}, + {"id": "w6", "text": "you", "start_time": 2.6, "end_time": 2.8} + ], + "preceding_words": "baby", + "following_words": "so much", + "reference_contexts": { + "genius": "baby I love you so much" + }, + "artist": "Test Artist", + "title": "Love Song" + }, + { + "name": "Complex multi-word error", + "gap_id": "test_gap_4", + "gap_words": [ + {"id": "w7", "text": "gonna", "start_time": 3.0, "end_time": 3.2}, + {"id": "w8", "text": "give", "start_time": 3.2, "end_time": 3.4}, + {"id": "w9", "text": "you", "start_time": 3.4, "end_time": 3.6}, + {"id": "w10", "text": "up", "start_time": 3.6, "end_time": 3.8} + ], + "preceding_words": "never", + "following_words": "never gonna let you down", + "reference_contexts": { + "genius": "Never gonna give you up, never gonna let you down" + }, + "artist": "Rick Astley", + "title": "Never Gonna Give You Up" + } +] + + +def run_single_test(test_case: Dict[str, Any], agent) -> Dict[str, Any]: + """Run a single test case and return results.""" + logger.info(f"\n{'='*60}") + logger.info(f"Test: {test_case['name']}") + logger.info(f"Gap words: {[w['text'] for w in test_case['gap_words']]}") + logger.info(f"Reference: {list(test_case['reference_contexts'].values())[0][:50]}...") + logger.info(f"{'='*60}") + + try: + proposals = agent.propose_for_gap( + gap_id=test_case["gap_id"], + gap_words=test_case["gap_words"], + preceding_words=test_case["preceding_words"], + following_words=test_case["following_words"], + reference_contexts=test_case["reference_contexts"], + artist=test_case.get("artist"), + title=test_case.get("title") + ) + + result = { + "test_name": test_case["name"], + "success": True, + "proposals": [] + } + + if not proposals: + logger.info("No proposals returned (gap may be correct)") + result["proposals"] = [] + else: + for p in proposals: + proposal_dict = { + "action": p.action, + "word_id": p.word_id, + "word_ids": p.word_ids, + "replacement_text": p.replacement_text, + "confidence": p.confidence, + "reason": p.reason, + "gap_category": str(p.gap_category) if p.gap_category else None, + "requires_human_review": p.requires_human_review + } + result["proposals"].append(proposal_dict) + + logger.info(f"\nProposal:") + logger.info(f" Action: {p.action}") + logger.info(f" Word ID(s): {p.word_id or p.word_ids}") + if p.replacement_text: + logger.info(f" Replacement: {p.replacement_text}") + logger.info(f" Confidence: {p.confidence:.2f}") + logger.info(f" Category: {p.gap_category}") + logger.info(f" Reason: {p.reason}") + if p.requires_human_review: + logger.info(f" ⚠️ Requires human review") + + return result + + except Exception as e: + logger.error(f"Test failed with error: {e}") + import traceback + traceback.print_exc() + return { + "test_name": test_case["name"], + "success": False, + "error": str(e) + } + + +def main(): + """Main entry point.""" + # Check environment + project_id = os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID") + location = os.getenv("GCP_LOCATION", "global") + model_override = os.getenv("AGENTIC_AI_MODEL") + privacy_mode = os.getenv("PRIVACY_MODE", "").lower() in ("1", "true", "yes") + + logger.info("=" * 60) + logger.info("AGENTIC CORRECTION TEST SCRIPT") + logger.info("=" * 60) + logger.info(f"GCP Project: {project_id or 'not set'}") + logger.info(f"GCP Location: {location}") + logger.info(f"Model Override: {model_override or 'none (using default)'}") + logger.info(f"Privacy Mode: {privacy_mode}") + logger.info(f"Langfuse: {'enabled' if os.getenv('LANGFUSE_PUBLIC_KEY') else 'disabled'}") + + # Import after path setup + try: + from lyrics_transcriber.correction.agentic.agent import AgenticCorrector + from lyrics_transcriber.correction.agentic.router import ModelRouter + from lyrics_transcriber.correction.agentic.providers.config import ProviderConfig + except ImportError as e: + logger.error(f"Failed to import agentic correction modules: {e}") + logger.error("Make sure you're running from the karaoke-gen directory") + sys.exit(1) + + # Determine which model to use + config = ProviderConfig.from_env() + router = ModelRouter(config) + model = router.choose_model("unknown", 0.5) + logger.info(f"Selected model: {model}") + + # Validate environment for Vertex AI + if model.startswith("vertexai/") and not project_id: + logger.error("GOOGLE_CLOUD_PROJECT environment variable is required for Vertex AI") + logger.error("Set it with: export GOOGLE_CLOUD_PROJECT=your-project-id") + logger.error("Or use a different model: AGENTIC_AI_MODEL=openai/gpt-4o") + sys.exit(1) + + # Create the agent + logger.info(f"\nCreating AgenticCorrector with model: {model}") + try: + agent = AgenticCorrector.from_model( + model=model, + session_id="test_session_local", + cache_dir=os.path.expanduser("~/lyrics-transcriber-cache") + ) + logger.info("Agent created successfully!") + except Exception as e: + logger.error(f"Failed to create agent: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + # Run test cases + results = [] + for test_case in TEST_CASES: + result = run_single_test(test_case, agent) + results.append(result) + + # Summary + logger.info("\n" + "=" * 60) + logger.info("TEST SUMMARY") + logger.info("=" * 60) + + passed = sum(1 for r in results if r["success"]) + failed = len(results) - passed + + for r in results: + status = "✅ PASS" if r["success"] else "❌ FAIL" + logger.info(f"{status}: {r['test_name']}") + if not r["success"]: + logger.info(f" Error: {r.get('error', 'unknown')}") + + logger.info(f"\nResults: {passed}/{len(results)} passed") + + # Output JSON results + output_file = "agentic_test_results.json" + with open(output_file, "w") as f: + json.dump(results, f, indent=2) + logger.info(f"\nDetailed results saved to: {output_file}") + + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main())