mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39d34ae316 | ||
|
|
41da1130e5 | ||
|
|
d566f62cd3 | ||
|
|
4cb76dc9fe | ||
|
|
a4a9d46c83 | ||
|
|
b5f0b587fc | ||
|
|
47aab175f7 | ||
|
|
3b1984b8d4 | ||
|
|
e0d20f1e43 | ||
|
|
46d7600f54 | ||
|
|
73e933d3b5 |
No files matched your search
@@ -0,0 +1,86 @@
|
||||
---
|
||||
description: Maintain git worktrees - identify merged PRs for cleanup and rename generic branches
|
||||
allowed-tools: Read, Glob, Grep, Bash, mcp__github__list_pull_requests, mcp__github__pull_request_read, mcp__github__get_me, mcp__github__search_pull_requests, AskUserQuestion
|
||||
---
|
||||
|
||||
# Git Worktree Maintenance
|
||||
|
||||
You are performing maintenance on the project's git worktrees.
|
||||
|
||||
## Process
|
||||
|
||||
### 1. List All Worktrees
|
||||
Run `git worktree list` to see all current worktrees.
|
||||
|
||||
### 2. Get PR Status for Each Branch
|
||||
For each worktree (excluding main), check the PR status:
|
||||
- Use `gh pr list --state all --limit 50 --json number,title,headRefName,state,mergedAt` to get PR info
|
||||
- Match each worktree branch to its PR
|
||||
- Identify: MERGED (can cleanup), OPEN (keep), NO PR (investigate)
|
||||
|
||||
### 3. Categorize Worktrees
|
||||
|
||||
**Can be cleaned up (PRs merged):**
|
||||
- List worktrees where the associated PR has been merged
|
||||
- These are safe to remove with `git worktree remove <path>`
|
||||
|
||||
**Outstanding/WIP (keep):**
|
||||
- List worktrees with open PRs
|
||||
- Check if the worktree name is descriptive or generic (e.g., "work-20251230-151202", "session-YYYY-MM-DD-HHMM")
|
||||
- For generic names, read the PR title/description to suggest a better name
|
||||
|
||||
**No PR found:**
|
||||
- Flag these for user attention - may be abandoned work or local-only branches
|
||||
|
||||
### 4. Present Findings
|
||||
|
||||
Show a summary table:
|
||||
```
|
||||
## Can be cleaned up (merged):
|
||||
| Worktree | Branch | PR # |
|
||||
|----------|--------|------|
|
||||
|
||||
## Outstanding PRs (keep):
|
||||
| Worktree | Branch | PR # | Suggested Rename |
|
||||
|----------|--------|------|------------------|
|
||||
|
||||
## No PR found:
|
||||
| Worktree | Branch | Notes |
|
||||
|----------|--------|-------|
|
||||
```
|
||||
|
||||
### 5. Ask User for Action
|
||||
|
||||
Use AskUserQuestion to ask what to do:
|
||||
- Clean up all merged worktrees?
|
||||
- Rename generic worktree names to descriptive ones?
|
||||
- Investigate worktrees with no PR?
|
||||
|
||||
### 6. Execute Cleanup
|
||||
|
||||
If user approves:
|
||||
|
||||
**Removing merged worktrees:**
|
||||
```bash
|
||||
git worktree remove <path>
|
||||
```
|
||||
If there are uncommitted changes, check if they're already in main before force-removing.
|
||||
|
||||
**Renaming worktrees:**
|
||||
```bash
|
||||
mv <old-path> <new-path>
|
||||
git worktree repair <new-path>
|
||||
git worktree prune
|
||||
```
|
||||
|
||||
### 7. Final Verification
|
||||
|
||||
Run `git worktree list` again to confirm the final state.
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Never remove the main worktree
|
||||
- Always check for uncommitted changes before removing
|
||||
- If a worktree has changes not in main, offer to create a PR first
|
||||
- Generic names to watch for: `work-YYYYMMDD-*`, `session-YYYY-MM-DD-*`, `feature/work-*`
|
||||
- Good descriptive names should reflect the PR content (e.g., `log-timing-analyzer`, `e2e-completion`)
|
||||
@@ -0,0 +1,75 @@
|
||||
[
|
||||
{
|
||||
"test_name": "Sound-alike error (wurld -> world)",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "ReplaceWord",
|
||||
"word_id": null,
|
||||
"word_ids": [
|
||||
"w1"
|
||||
],
|
||||
"replacement_text": "world",
|
||||
"confidence": 0.75,
|
||||
"reason": "Sound-alike error. Reference suggests: 'world'. The transcription 'wurld' is a phonetic misspelling or sound-alike of the word 'world' which appears in both reference sources (Genius and LRCLIB).",
|
||||
"gap_category": "GapCategory.SOUND_ALIKE",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "Punctuation difference",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "NoAction",
|
||||
"word_id": null,
|
||||
"word_ids": [
|
||||
"w2"
|
||||
],
|
||||
"replacement_text": null,
|
||||
"confidence": 0.99,
|
||||
"reason": "Transcription matches at least one reference source. The gap text 'Hello' followed by the context 'is it me you're looking for' matches the Genius reference lyrics exactly, excluding punctuation. According to the guidelines, if any reference source matches the transcription text (ignoring punctuation), it should be classified as NO_ERROR.",
|
||||
"gap_category": "GapCategory.NO_ERROR",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "Extra filler word",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "DeleteWord",
|
||||
"word_id": "w3",
|
||||
"word_ids": null,
|
||||
"replacement_text": null,
|
||||
"confidence": 0.8,
|
||||
"reason": "Extra filler word 'And' at sentence start not in reference. The transcription includes the filler word 'And' at the beginning of the phrase, which is not present in the reference lyrics. This matches the definition of EXTRA_WORDS.",
|
||||
"gap_category": "GapCategory.EXTRA_WORDS",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "Complex multi-word error",
|
||||
"success": true,
|
||||
"proposals": [
|
||||
{
|
||||
"action": "NoAction",
|
||||
"word_id": null,
|
||||
"word_ids": [
|
||||
"w7",
|
||||
"w8",
|
||||
"w9",
|
||||
"w10"
|
||||
],
|
||||
"replacement_text": null,
|
||||
"confidence": 0.99,
|
||||
"reason": "Transcription matches at least one reference source. The transcription text 'gonna give you up' matches the Genius reference lyrics exactly when ignoring punctuation and capitalization. In context, the full phrase 'never gonna give you up' is correctly transcribed and aligns perfectly with the known lyrics and title of the song.",
|
||||
"gap_category": "GapCategory.NO_ERROR",
|
||||
"requires_human_review": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
+68
-1
@@ -329,10 +329,33 @@ gcloud run deploy service-name \
|
||||
For Gemini via Vertex AI, use Application Default Credentials (ADC) rather than API keys:
|
||||
- **Cloud Run**: Service account automatically authenticated - just grant `roles/aiplatform.user`
|
||||
- **Local dev**: Run `gcloud auth application-default login`
|
||||
- **Config**: Set `GOOGLE_CLOUD_PROJECT` and optionally `GCP_LOCATION` (defaults to `us-central1`)
|
||||
- **Config**: Set `GOOGLE_CLOUD_PROJECT` and optionally `GCP_LOCATION` (defaults to `global`)
|
||||
|
||||
This approach uses GCP's free credits and existing IAM rather than separate API key management.
|
||||
|
||||
### Gemini 3 Models Require Global Location
|
||||
|
||||
**Problem**: Gemini 3 Flash (`gemini-3-flash-preview`) returns 404 errors when using regional Vertex AI endpoints.
|
||||
|
||||
```
|
||||
404 Publisher Model `projects/PROJECT/locations/us-central1/publishers/google/models/gemini-3-flash-preview` was not found
|
||||
```
|
||||
|
||||
**Solution**: Use `global` location instead of regional endpoints like `us-central1`:
|
||||
```python
|
||||
gcp_location: str = "global" # Required for Gemini 3 models
|
||||
```
|
||||
|
||||
**Additional gotcha**: Gemini 3 returns multimodal response format:
|
||||
```python
|
||||
# Gemini 2: response.content = "text"
|
||||
# Gemini 3: response.content = [{'type': 'text', 'text': '...'}]
|
||||
```
|
||||
|
||||
Code must handle both formats when invoking LangChain ChatVertexAI.
|
||||
|
||||
See `docs/archive/2025-12-30-gemini3-agentic-correction-fix.md` for full details.
|
||||
|
||||
### CI/CD: GitHub Actions
|
||||
|
||||
Migrated from Cloud Build to GitHub Actions for:
|
||||
@@ -441,6 +464,50 @@ export function getToken() {
|
||||
|
||||
**Lesson**: For values that can change (auth tokens, user preferences), read fresh from storage on each access rather than caching in module scope.
|
||||
|
||||
### GCP Organization Policies Block Service Account Keys
|
||||
|
||||
**Problem**: GCP organization policies can block service account key creation even when you have Owner access to the project.
|
||||
|
||||
```
|
||||
ERROR: Key creation is not allowed on this service account.
|
||||
violations: constraints/iam.disableServiceAccountKeyCreation
|
||||
```
|
||||
|
||||
**Solution**: If you have Organization Policy Administrator role, temporarily disable the constraint:
|
||||
|
||||
```bash
|
||||
# Disable at organization level (find org ID with: gcloud projects describe PROJECT --format="value(parent)")
|
||||
cat > /tmp/policy.yaml << 'EOF'
|
||||
name: organizations/ORG_ID/policies/iam.disableServiceAccountKeyCreation
|
||||
spec:
|
||||
rules:
|
||||
- enforce: false
|
||||
EOF
|
||||
gcloud org-policies set-policy /tmp/policy.yaml
|
||||
|
||||
# Wait ~30 seconds for propagation
|
||||
sleep 30
|
||||
|
||||
# Create key
|
||||
gcloud iam service-accounts keys create key.json --iam-account=SA_EMAIL
|
||||
|
||||
# Re-enable immediately
|
||||
cat > /tmp/policy.yaml << 'EOF'
|
||||
name: organizations/ORG_ID/policies/iam.disableServiceAccountKeyCreation
|
||||
spec:
|
||||
rules:
|
||||
- enforce: true
|
||||
EOF
|
||||
gcloud org-policies set-policy /tmp/policy.yaml
|
||||
|
||||
# Clean up
|
||||
rm /tmp/policy.yaml key.json # After copying key contents
|
||||
```
|
||||
|
||||
**Note**: Also check for the newer managed constraint `iam.managed.disableServiceAccountKeyCreation` and disable both if needed.
|
||||
|
||||
**Lesson**: Service account keys are a security risk and should be avoided when possible. Use Workload Identity Federation instead. But for third-party services that require JSON keys (like LangFuse), this temporary policy override is the safest approach.
|
||||
|
||||
## What We'd Do Differently
|
||||
|
||||
1. **Add Pydantic model fields test first** - Would have caught the silent field issue immediately
|
||||
|
||||
@@ -34,6 +34,8 @@
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- **LangFuse Prompt Management** (2025-12-30): Replaced hardcoded LLM prompts with LangFuse-managed prompts for dynamic iteration without redeploying. Added migration script, LangFuse Vertex AI service account for GCP credits. See [archive/2025-12-30-langfuse-prompt-management.md](archive/2025-12-30-langfuse-prompt-management.md)
|
||||
- **Gemini 3 Flash Agentic Correction Fix** (2025-12-30): Fixed agentic correction hanging due to Gemini 3 requiring `global` location instead of regional endpoints, and multimodal response format handling. Added local test script for faster iteration. See [archive/2025-12-30-gemini3-agentic-correction-fix.md](archive/2025-12-30-gemini3-agentic-correction-fix.md)
|
||||
- **Cold Start UX** (2025-12-30): Frontend now shows friendly "Warming up the generator..." messages when backend is slow to respond (cold start after deployment or inactivity). Progressive messages appear after 5s and 15s to reassure users during 30s+ cold starts.
|
||||
- **Cloud Output Structure Fix** (2025-12-30): Fixed remote CLI (`karaoke-gen-remote`) to produce identical output as local CLI: YouTube uploads now work, Dropbox includes stems/ and lyrics/ subfolders, instrumental filenames use actual model names. See [archive/2025-12-30-cloud-output-structure-fix.md](archive/2025-12-30-cloud-output-structure-fix.md)
|
||||
- **Auth Session Persistence Fix** (2025-12-30): Fixed two bugs causing 401 errors despite valid sessions: timezone-aware datetime comparisons in backend, and stale token caching in frontend. Firestore indexes now tracked in Pulumi. See [archive/2025-12-30-auth-session-persistence-fix.md](archive/2025-12-30-auth-session-persistence-fix.md)
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# Gemini 3 Flash Agentic Correction Fix
|
||||
|
||||
**Date**: 2025-12-30
|
||||
**PR**: #137
|
||||
|
||||
## Problem
|
||||
|
||||
Karaoke jobs were hanging at the agentic correction stage with no visible errors in Langfuse traces. Cloud Run logs showed:
|
||||
|
||||
```
|
||||
404 Publisher Model `projects/nomadkaraoke/locations/us-central1/publishers/google/models/gemini-3-flash-preview` was not found
|
||||
```
|
||||
|
||||
## Root Cause
|
||||
|
||||
**Gemini 3 models require the `global` location**, not regional endpoints like `us-central1`. This is different from Gemini 2 models which work with regional endpoints.
|
||||
|
||||
Additionally, Gemini 3 returns a **multimodal response format** instead of plain strings:
|
||||
```python
|
||||
# Gemini 2 response.content:
|
||||
"some text response"
|
||||
|
||||
# Gemini 3 response.content:
|
||||
[{'type': 'text', 'text': 'some text response'}]
|
||||
```
|
||||
|
||||
## Solution
|
||||
|
||||
1. Changed default `gcp_location` from `us-central1` to `global` in `config.py`
|
||||
2. Added multimodal response parsing in `langchain_bridge.py`:
|
||||
```python
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get('type') == 'text':
|
||||
return part.get('text', '')
|
||||
```
|
||||
|
||||
## Files Changed
|
||||
|
||||
- `lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/config.py` - Default location
|
||||
- `lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/providers/langchain_bridge.py` - Response parsing
|
||||
- `lyrics_transcriber_temp/lyrics_transcriber/correction/agentic/router.py` - Model name comment
|
||||
|
||||
## Testing
|
||||
|
||||
Created `scripts/test_agentic_correction.py` for local testing without Cloud Run deployment:
|
||||
```bash
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke python scripts/test_agentic_correction.py
|
||||
```
|
||||
|
||||
Four test cases covering sound-alike errors, punctuation, extra words, and no-error scenarios all pass.
|
||||
|
||||
## Key Learnings
|
||||
|
||||
1. **Gemini 3 requires global location** - Not documented prominently; discovered via error message
|
||||
2. **Model response formats differ between versions** - Always handle both string and list content types
|
||||
3. **Local test scripts are essential** - Enabled rapid iteration without Cloud Run deploy cycles
|
||||
4. **Clear LLM cache when debugging** - Cached broken responses can mask fixes: `rm -rf ~/lyrics-transcriber-cache/*.json`
|
||||
@@ -0,0 +1,91 @@
|
||||
# LangFuse Prompt Management Integration
|
||||
|
||||
**Date**: 2025-12-30
|
||||
**PR**: https://github.com/nomadkaraoke/karaoke-gen/pull/140
|
||||
|
||||
## Summary
|
||||
|
||||
Replaced hardcoded LLM prompts in the agentic correction workflow with LangFuse-managed prompts, enabling prompt iteration without redeploying the backend.
|
||||
|
||||
## What Changed
|
||||
|
||||
### New Components
|
||||
|
||||
1. **LangFusePromptService** (`lyrics_transcriber/.../prompts/langfuse_prompts.py`)
|
||||
- Fetches prompts from LangFuse with hardcoded fallback for local development
|
||||
- Fetches few-shot examples from LangFuse datasets
|
||||
- Singleton pattern with fail-fast when LangFuse is configured
|
||||
|
||||
2. **Migration Script** (`scripts/migrate-prompts-to-langfuse.py`)
|
||||
- Uploads `gap-classifier` prompt template to LangFuse
|
||||
- Uploads 9 few-shot examples as `gap-classifier-examples` dataset
|
||||
- Supports `--dry-run`, `--force`, `--verify-only` flags
|
||||
|
||||
3. **LangFuse Vertex AI Service Account** (Pulumi)
|
||||
- `langfuse-vertexai@nomadkaraoke.iam.gserviceaccount.com`
|
||||
- Has `roles/aiplatform.user` for calling Gemini models
|
||||
- JSON key generated for LangFuse to use GCP credits
|
||||
|
||||
### Modified Components
|
||||
|
||||
- `langfuse_integration.py` - Added `fetch_prompt()` and `fetch_dataset()` with label fallback
|
||||
- `classifier.py` - Now uses LangFusePromptService, hardcoded function renamed to `build_classification_prompt_hardcoded`
|
||||
- Default LangFuse host changed to US region (`https://us.cloud.langfuse.com`)
|
||||
|
||||
### Deleted Components
|
||||
|
||||
- `handlers/llm.py` - Legacy LLM handler not being used
|
||||
- `handlers/llm_providers.py` - Legacy provider implementations
|
||||
|
||||
## Configuration
|
||||
|
||||
### LangFuse Credentials (GCP Secret Manager)
|
||||
|
||||
- `langfuse-public-key` - Project: nomadkaraoke
|
||||
- `langfuse-secret-key` - Project: nomadkaraoke
|
||||
|
||||
### Environment Variables
|
||||
|
||||
```bash
|
||||
LANGFUSE_PUBLIC_KEY=pk-lf-...
|
||||
LANGFUSE_SECRET_KEY=sk-lf-...
|
||||
LANGFUSE_HOST=https://us.cloud.langfuse.com # Optional, this is the default
|
||||
```
|
||||
|
||||
### LangFuse Resources
|
||||
|
||||
- **Prompt**: `gap-classifier` (promote to "production" label for default fetch)
|
||||
- **Dataset**: `gap-classifier-examples` (9 few-shot examples)
|
||||
- **Project**: nomadkaraoke at https://us.cloud.langfuse.com
|
||||
|
||||
## How Prompt Fetching Works
|
||||
|
||||
1. Try to fetch prompt with "production" label (default)
|
||||
2. If not found, fall back to version 1 (for newly created prompts)
|
||||
3. If LangFuse not configured, use hardcoded fallback
|
||||
|
||||
This allows:
|
||||
- New prompts to work immediately after upload
|
||||
- Production prompts to be explicitly promoted
|
||||
- Local development without LangFuse credentials
|
||||
|
||||
## GCP Organization Policy Issue
|
||||
|
||||
Creating service account keys was blocked by organization policy `iam.disableServiceAccountKeyCreation`. Solution:
|
||||
|
||||
1. Temporarily disable policy at organization level
|
||||
2. Create key via `gcloud iam service-accounts keys create`
|
||||
3. Re-enable policy immediately
|
||||
4. Delete local key file after pasting into LangFuse
|
||||
|
||||
This is a one-time operation. The key is now stored in LangFuse, not in any repo or GCP secret.
|
||||
|
||||
## Testing
|
||||
|
||||
All 676 unit tests pass. The prompt service has dedicated tests in `test_langfuse_prompts.py` that mock the LangFuse client.
|
||||
|
||||
## Future Work
|
||||
|
||||
- Add more prompts for other agentic workflows
|
||||
- Set up LangFuse evaluations for prompt quality
|
||||
- Consider A/B testing different prompt versions
|
||||
@@ -150,7 +150,7 @@ export function JobSubmission({ onJobCreated }: JobSubmissionProps) {
|
||||
<TabsList className="grid w-full grid-cols-3 h-auto rounded-lg p-1" style={{ backgroundColor: 'var(--secondary)' }}>
|
||||
<TabsTrigger
|
||||
value="search"
|
||||
className="gap-1.5 sm:gap-2 py-3 min-h-[44px] text-xs sm:text-sm rounded-md data-[state=active]:bg-amber-500 data-[state=active]:text-white data-[state=active]:shadow-md data-[state=inactive]:hover:bg-amber-500/10"
|
||||
className="gap-1.5 sm:gap-2 py-3 min-h-[44px] text-xs sm:text-sm rounded-md data-[state=active]:!bg-amber-600 data-[state=active]:!text-white data-[state=active]:shadow-md data-[state=inactive]:hover:bg-amber-600/10"
|
||||
style={{ color: activeTab === 'search' ? 'white' : 'var(--text-muted)' }}
|
||||
>
|
||||
<Music className="w-4 h-4 shrink-0" />
|
||||
@@ -158,7 +158,7 @@ export function JobSubmission({ onJobCreated }: JobSubmissionProps) {
|
||||
</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="upload"
|
||||
className="gap-1.5 sm:gap-2 py-3 min-h-[44px] text-xs sm:text-sm rounded-md data-[state=active]:bg-amber-500 data-[state=active]:text-white data-[state=active]:shadow-md data-[state=inactive]:hover:bg-amber-500/10"
|
||||
className="gap-1.5 sm:gap-2 py-3 min-h-[44px] text-xs sm:text-sm rounded-md data-[state=active]:!bg-amber-600 data-[state=active]:!text-white data-[state=active]:shadow-md data-[state=inactive]:hover:bg-amber-600/10"
|
||||
style={{ color: activeTab === 'upload' ? 'white' : 'var(--text-muted)' }}
|
||||
>
|
||||
<Upload className="w-4 h-4 shrink-0" />
|
||||
@@ -166,7 +166,7 @@ export function JobSubmission({ onJobCreated }: JobSubmissionProps) {
|
||||
</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="url"
|
||||
className="gap-1.5 sm:gap-2 py-3 min-h-[44px] text-xs sm:text-sm rounded-md data-[state=active]:bg-amber-500 data-[state=active]:text-white data-[state=active]:shadow-md data-[state=inactive]:hover:bg-amber-500/10"
|
||||
className="gap-1.5 sm:gap-2 py-3 min-h-[44px] text-xs sm:text-sm rounded-md data-[state=active]:!bg-amber-600 data-[state=active]:!text-white data-[state=active]:shadow-md data-[state=inactive]:hover:bg-amber-600/10"
|
||||
style={{ color: activeTab === 'url' ? 'white' : 'var(--text-muted)' }}
|
||||
>
|
||||
<Youtube className="w-4 h-4 shrink-0" />
|
||||
@@ -232,6 +232,9 @@ export function JobSubmission({ onJobCreated }: JobSubmissionProps) {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-xs" style={{ color: 'var(--text-muted)' }}>
|
||||
<span className="text-amber-500 font-medium">Note:</span> Format these exactly as you want them on the title card and video filename.
|
||||
</p>
|
||||
|
||||
{/* Theme Selection */}
|
||||
<div className="space-y-4 pt-4 border-t" style={{ borderColor: 'var(--card-border)' }}>
|
||||
@@ -311,6 +314,9 @@ export function JobSubmission({ onJobCreated }: JobSubmissionProps) {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-xs" style={{ color: 'var(--text-muted)' }}>
|
||||
<span className="text-amber-500 font-medium">Note:</span> Override auto-detected values if needed. These appear on the title card and video filename.
|
||||
</p>
|
||||
|
||||
{/* Theme Selection */}
|
||||
<div className="space-y-4 pt-4 border-t" style={{ borderColor: 'var(--card-border)' }}>
|
||||
@@ -373,6 +379,9 @@ export function JobSubmission({ onJobCreated }: JobSubmissionProps) {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-xs" style={{ color: 'var(--text-muted)' }}>
|
||||
<span className="text-amber-500 font-medium">Note:</span> Format these exactly as you want them on the title card and video filename.
|
||||
</p>
|
||||
|
||||
{/* Theme Selection */}
|
||||
<div className="space-y-4 pt-4 border-t" style={{ borderColor: 'var(--card-border)' }}>
|
||||
|
||||
+178
-5
@@ -1,28 +1,115 @@
|
||||
from typing import Optional, Dict, Any
|
||||
"""LangFuse integration for agentic correction observability and prompt management.
|
||||
|
||||
This module provides:
|
||||
- Client initialization with fail-fast behavior when configured
|
||||
- Metrics recording for observability
|
||||
- Prompt fetching for dynamic prompt management
|
||||
- Dataset fetching for few-shot examples
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any, List
|
||||
import os
|
||||
import threading
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Module-level client singleton
|
||||
_langfuse_client: Optional[Any] = None
|
||||
_client_initialized: bool = False
|
||||
|
||||
|
||||
def setup_langfuse(client_name: str = "agentic-corrector") -> Optional[object]:
|
||||
class LangFuseConfigError(Exception):
|
||||
"""Raised when LangFuse is configured but initialization fails."""
|
||||
pass
|
||||
|
||||
|
||||
def is_langfuse_configured() -> bool:
|
||||
"""Check if LangFuse credentials are configured in environment."""
|
||||
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
secret_key = os.getenv("LANGFUSE_SECRET_KEY")
|
||||
return bool(public_key and secret_key)
|
||||
|
||||
|
||||
def setup_langfuse() -> Optional[object]:
|
||||
"""Initialize Langfuse client if keys are present; return client or None.
|
||||
|
||||
This avoids hard dependency at import time; caller can check for None and
|
||||
no-op if observability is not configured.
|
||||
|
||||
Note: This function does NOT fail fast - use get_langfuse_client() for
|
||||
fail-fast behavior when LangFuse is required.
|
||||
"""
|
||||
secret = os.getenv("LANGFUSE_SECRET_KEY")
|
||||
public = os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
host = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
|
||||
host = os.getenv("LANGFUSE_HOST", "https://us.cloud.langfuse.com")
|
||||
if not (secret and public):
|
||||
return None
|
||||
try:
|
||||
from langfuse import Langfuse # type: ignore
|
||||
|
||||
client = Langfuse(secret_key=secret, public_key=public, host=host, sdk_integration=client_name)
|
||||
client = Langfuse(secret_key=secret, public_key=public, host=host)
|
||||
return client
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_langfuse_client() -> Optional[Any]:
|
||||
"""Get or create the LangFuse client singleton.
|
||||
|
||||
Unlike setup_langfuse(), this function implements fail-fast behavior:
|
||||
if LangFuse keys are configured but initialization fails, it raises
|
||||
an exception rather than returning None.
|
||||
|
||||
Returns:
|
||||
Langfuse client instance, or None if not configured
|
||||
|
||||
Raises:
|
||||
LangFuseConfigError: If keys are set but initialization fails
|
||||
"""
|
||||
global _langfuse_client, _client_initialized
|
||||
|
||||
if _client_initialized:
|
||||
return _langfuse_client
|
||||
|
||||
secret = os.getenv("LANGFUSE_SECRET_KEY")
|
||||
public = os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
host = os.getenv("LANGFUSE_HOST", "https://us.cloud.langfuse.com")
|
||||
|
||||
if not (secret and public):
|
||||
logger.debug("LangFuse keys not configured, client disabled")
|
||||
_client_initialized = True
|
||||
return None
|
||||
|
||||
try:
|
||||
from langfuse import Langfuse
|
||||
|
||||
_langfuse_client = Langfuse(
|
||||
secret_key=secret,
|
||||
public_key=public,
|
||||
host=host,
|
||||
)
|
||||
_client_initialized = True
|
||||
logger.info(f"LangFuse client initialized (host: {host})")
|
||||
return _langfuse_client
|
||||
|
||||
except Exception as e:
|
||||
# Fail fast - if keys are set, we expect LangFuse to work
|
||||
raise LangFuseConfigError(
|
||||
f"LangFuse keys are set but initialization failed: {e}\n"
|
||||
f"Check:\n"
|
||||
f" - LANGFUSE_PUBLIC_KEY: {public[:10] if public else 'not set'}...\n"
|
||||
f" - LANGFUSE_SECRET_KEY: {'set' if secret else 'not set'}\n"
|
||||
f" - LANGFUSE_HOST: {host}"
|
||||
) from e
|
||||
|
||||
|
||||
def reset_langfuse_client() -> None:
|
||||
"""Reset the global LangFuse client (for testing)."""
|
||||
global _langfuse_client, _client_initialized
|
||||
_langfuse_client = None
|
||||
_client_initialized = False
|
||||
|
||||
|
||||
def record_metrics(client: Optional[object], name: str, metrics: Dict[str, Any]) -> None:
|
||||
"""Record custom metrics to Langfuse if initialized."""
|
||||
if client is None:
|
||||
@@ -33,3 +120,89 @@ def record_metrics(client: Optional[object], name: str, metrics: Dict[str, Any])
|
||||
except Exception:
|
||||
# Swallow observability errors to never impact core flow
|
||||
pass
|
||||
|
||||
|
||||
def fetch_prompt(name: str, client: Optional[Any] = None, label: Optional[str] = "production") -> Any:
|
||||
"""Fetch a prompt template from LangFuse.
|
||||
|
||||
Args:
|
||||
name: The prompt name in LangFuse
|
||||
client: Optional pre-initialized client. If None, uses get_langfuse_client()
|
||||
label: Prompt label to fetch (default: "production"). If the labeled version
|
||||
is not found, falls back to version 1.
|
||||
|
||||
Returns:
|
||||
LangFuse prompt object
|
||||
|
||||
Raises:
|
||||
LangFuseConfigError: If LangFuse is not configured
|
||||
RuntimeError: If prompt fetch fails
|
||||
"""
|
||||
if client is None:
|
||||
client = get_langfuse_client()
|
||||
|
||||
if client is None:
|
||||
raise LangFuseConfigError(
|
||||
f"Cannot fetch prompt '{name}': LangFuse is not configured. "
|
||||
f"Set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY."
|
||||
)
|
||||
|
||||
try:
|
||||
# Try to fetch with the specified label (default: production)
|
||||
prompt = client.get_prompt(name, label=label)
|
||||
logger.debug(f"Fetched prompt '{name}' (label={label}) from LangFuse")
|
||||
return prompt
|
||||
except Exception as label_error:
|
||||
# If labeled version not found, try fetching version 1 as fallback
|
||||
# This handles newly created prompts that haven't been promoted yet
|
||||
try:
|
||||
prompt = client.get_prompt(name, version=1)
|
||||
logger.warning(
|
||||
f"Prompt '{name}' label '{label}' not found, using version 1. "
|
||||
f"Consider promoting this prompt in LangFuse UI."
|
||||
)
|
||||
return prompt
|
||||
except Exception as version_error:
|
||||
raise RuntimeError(
|
||||
f"Failed to fetch prompt '{name}' from LangFuse: "
|
||||
f"Label '{label}' error: {label_error}, "
|
||||
f"Version 1 fallback error: {version_error}"
|
||||
) from version_error
|
||||
|
||||
|
||||
def fetch_dataset(name: str, client: Optional[Any] = None) -> List[Dict[str, Any]]:
|
||||
"""Fetch a dataset from LangFuse and return its items.
|
||||
|
||||
Args:
|
||||
name: The dataset name in LangFuse
|
||||
client: Optional pre-initialized client. If None, uses get_langfuse_client()
|
||||
|
||||
Returns:
|
||||
List of dataset item inputs (the actual example data)
|
||||
|
||||
Raises:
|
||||
LangFuseConfigError: If LangFuse is not configured
|
||||
RuntimeError: If dataset fetch fails
|
||||
"""
|
||||
if client is None:
|
||||
client = get_langfuse_client()
|
||||
|
||||
if client is None:
|
||||
raise LangFuseConfigError(
|
||||
f"Cannot fetch dataset '{name}': LangFuse is not configured. "
|
||||
f"Set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY."
|
||||
)
|
||||
|
||||
try:
|
||||
dataset = client.get_dataset(name)
|
||||
items = []
|
||||
for item in dataset.items:
|
||||
if hasattr(item, 'input') and item.input:
|
||||
items.append(item.input)
|
||||
|
||||
logger.debug(f"Fetched {len(items)} items from dataset '{name}'")
|
||||
return items
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"Failed to fetch dataset '{name}' from LangFuse: {e}"
|
||||
) from e
|
||||
@@ -1,2 +1,25 @@
|
||||
"""Prompt templates for agentic correction."""
|
||||
|
||||
from .classifier import (
|
||||
build_classification_prompt,
|
||||
build_classification_prompt_hardcoded,
|
||||
get_hardcoded_examples,
|
||||
)
|
||||
from .langfuse_prompts import (
|
||||
LangFusePromptService,
|
||||
LangFusePromptError,
|
||||
LangFuseDatasetError,
|
||||
get_prompt_service,
|
||||
reset_prompt_service,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"build_classification_prompt",
|
||||
"build_classification_prompt_hardcoded",
|
||||
"get_hardcoded_examples",
|
||||
"LangFusePromptService",
|
||||
"LangFusePromptError",
|
||||
"LangFuseDatasetError",
|
||||
"get_prompt_service",
|
||||
"reset_prompt_service",
|
||||
]
|
||||
+66
-6
@@ -1,23 +1,35 @@
|
||||
"""Gap classification prompt builder for agentic correction."""
|
||||
"""Gap classification prompt builder for agentic correction.
|
||||
|
||||
This module provides two modes of operation:
|
||||
1. LangFuse mode: Prompts and examples fetched from LangFuse for dynamic iteration
|
||||
2. Hardcoded mode: Fallback for local development when LangFuse is not configured
|
||||
|
||||
The main entry point is `build_classification_prompt()` which automatically
|
||||
selects the appropriate mode based on LangFuse configuration.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
import yaml
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_few_shot_examples() -> Dict[str, List[Dict]]:
|
||||
"""Load few-shot examples from examples.yaml if it exists."""
|
||||
examples_path = Path(__file__).parent / "examples.yaml"
|
||||
|
||||
|
||||
if not examples_path.exists():
|
||||
return get_hardcoded_examples()
|
||||
|
||||
|
||||
try:
|
||||
with open(examples_path, 'r') as f:
|
||||
data = yaml.safe_load(f)
|
||||
return data.get('examples_by_category', {})
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load examples.yaml, using hardcoded examples: {e}")
|
||||
return get_hardcoded_examples()
|
||||
|
||||
|
||||
@@ -122,7 +134,12 @@ def build_classification_prompt(
|
||||
gap_id: Optional[str] = None
|
||||
) -> str:
|
||||
"""Build a prompt for classifying a gap in the transcription.
|
||||
|
||||
|
||||
This function automatically selects between LangFuse and hardcoded prompts:
|
||||
- If LangFuse is configured (LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY set),
|
||||
fetches the prompt template and examples from LangFuse.
|
||||
- Otherwise, uses hardcoded prompts for local development.
|
||||
|
||||
Args:
|
||||
gap_text: The text of the gap that needs classification
|
||||
preceding_words: Text immediately before the gap
|
||||
@@ -131,7 +148,50 @@ def build_classification_prompt(
|
||||
artist: Song artist name for context
|
||||
title: Song title for context
|
||||
gap_id: Identifier for the gap
|
||||
|
||||
|
||||
Returns:
|
||||
Formatted prompt string for the LLM
|
||||
|
||||
Raises:
|
||||
LangFusePromptError: If LangFuse is configured but prompt fetch fails
|
||||
"""
|
||||
from .langfuse_prompts import get_prompt_service
|
||||
|
||||
service = get_prompt_service()
|
||||
return service.get_classification_prompt(
|
||||
gap_text=gap_text,
|
||||
preceding_words=preceding_words,
|
||||
following_words=following_words,
|
||||
reference_contexts=reference_contexts,
|
||||
artist=artist,
|
||||
title=title,
|
||||
gap_id=gap_id
|
||||
)
|
||||
|
||||
|
||||
def build_classification_prompt_hardcoded(
|
||||
gap_text: str,
|
||||
preceding_words: str,
|
||||
following_words: str,
|
||||
reference_contexts: Dict[str, str],
|
||||
artist: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
gap_id: Optional[str] = None
|
||||
) -> str:
|
||||
"""Build a prompt for classifying a gap using hardcoded templates.
|
||||
|
||||
This is the fallback implementation used when LangFuse is not configured.
|
||||
It is also used as the source of truth for migrating prompts to LangFuse.
|
||||
|
||||
Args:
|
||||
gap_text: The text of the gap that needs classification
|
||||
preceding_words: Text immediately before the gap
|
||||
following_words: Text immediately after the gap
|
||||
reference_contexts: Dictionary of reference lyrics from each source
|
||||
artist: Song artist name for context
|
||||
title: Song title for context
|
||||
gap_id: Identifier for the gap
|
||||
|
||||
Returns:
|
||||
Formatted prompt string for the LLM
|
||||
"""
|
||||
|
||||
+298
@@ -0,0 +1,298 @@
|
||||
"""LangFuse prompt management for agentic correction.
|
||||
|
||||
This module provides prompt fetching from LangFuse, enabling dynamic prompt
|
||||
iteration without code redeployment.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Any
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LangFusePromptError(Exception):
|
||||
"""Raised when LangFuse prompt fetching fails."""
|
||||
pass
|
||||
|
||||
|
||||
class LangFuseDatasetError(Exception):
|
||||
"""Raised when LangFuse dataset fetching fails."""
|
||||
pass
|
||||
|
||||
|
||||
class LangFusePromptService:
|
||||
"""Fetches prompts and datasets from LangFuse for agentic correction.
|
||||
|
||||
This service handles:
|
||||
- Fetching prompt templates from LangFuse
|
||||
- Fetching few-shot examples from LangFuse datasets
|
||||
- Compiling prompts with dynamic variables
|
||||
- Fail-fast behavior when LangFuse is configured but unavailable
|
||||
|
||||
When LangFuse keys are not configured, falls back to hardcoded prompts
|
||||
for local development.
|
||||
"""
|
||||
|
||||
# Prompt and dataset names in LangFuse
|
||||
CLASSIFIER_PROMPT_NAME = "gap-classifier"
|
||||
EXAMPLES_DATASET_NAME = "gap-classifier-examples"
|
||||
|
||||
def __init__(self, client: Optional[Any] = None):
|
||||
"""Initialize the prompt service.
|
||||
|
||||
Args:
|
||||
client: Optional pre-initialized Langfuse client (for testing).
|
||||
If None, will initialize from environment variables.
|
||||
"""
|
||||
self._client = client
|
||||
self._initialized = False
|
||||
self._use_langfuse = self._should_use_langfuse()
|
||||
|
||||
if self._use_langfuse and client is None:
|
||||
self._init_client()
|
||||
|
||||
def _should_use_langfuse(self) -> bool:
|
||||
"""Check if LangFuse credentials are configured."""
|
||||
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
secret_key = os.getenv("LANGFUSE_SECRET_KEY")
|
||||
return bool(public_key and secret_key)
|
||||
|
||||
def _init_client(self) -> None:
|
||||
"""Initialize the Langfuse client using the shared singleton."""
|
||||
from ..observability.langfuse_integration import get_langfuse_client, LangFuseConfigError
|
||||
|
||||
try:
|
||||
self._client = get_langfuse_client()
|
||||
if self._client:
|
||||
self._initialized = True
|
||||
logger.info("LangFuse prompt service initialized")
|
||||
else:
|
||||
logger.debug("LangFuse keys not configured, will use hardcoded prompts")
|
||||
except LangFuseConfigError as e:
|
||||
# Re-raise as RuntimeError for consistent error handling
|
||||
raise RuntimeError(str(e)) from e
|
||||
|
||||
def get_classification_prompt(
|
||||
self,
|
||||
gap_text: str,
|
||||
preceding_words: str,
|
||||
following_words: str,
|
||||
reference_contexts: Dict[str, str],
|
||||
artist: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
gap_id: Optional[str] = None
|
||||
) -> str:
|
||||
"""Fetch and compile the gap classification prompt.
|
||||
|
||||
If LangFuse is configured, fetches the prompt template and examples
|
||||
from LangFuse. Otherwise, falls back to hardcoded prompts.
|
||||
|
||||
Args:
|
||||
gap_text: The text of the gap that needs classification
|
||||
preceding_words: Text immediately before the gap
|
||||
following_words: Text immediately after the gap
|
||||
reference_contexts: Dictionary of reference lyrics from each source
|
||||
artist: Song artist name for context
|
||||
title: Song title for context
|
||||
gap_id: Identifier for the gap
|
||||
|
||||
Returns:
|
||||
Compiled prompt string ready for LLM
|
||||
|
||||
Raises:
|
||||
LangFusePromptError: If LangFuse is configured but prompt fetch fails
|
||||
"""
|
||||
if not self._use_langfuse:
|
||||
# Fall back to hardcoded prompt for development
|
||||
from .classifier import build_classification_prompt_hardcoded
|
||||
return build_classification_prompt_hardcoded(
|
||||
gap_text=gap_text,
|
||||
preceding_words=preceding_words,
|
||||
following_words=following_words,
|
||||
reference_contexts=reference_contexts,
|
||||
artist=artist,
|
||||
title=title,
|
||||
gap_id=gap_id
|
||||
)
|
||||
|
||||
# Fetch from LangFuse
|
||||
try:
|
||||
prompt_template = self._fetch_prompt(self.CLASSIFIER_PROMPT_NAME)
|
||||
examples = self._fetch_examples()
|
||||
|
||||
# Build component strings
|
||||
song_context = self._build_song_context(artist, title)
|
||||
examples_text = self._format_examples(examples)
|
||||
references_text = self._format_references(reference_contexts)
|
||||
|
||||
# Compile the prompt with variables
|
||||
compiled = prompt_template.compile(
|
||||
song_context=song_context,
|
||||
examples_text=examples_text,
|
||||
gap_id=gap_id or "unknown",
|
||||
preceding_words=preceding_words,
|
||||
gap_text=gap_text,
|
||||
following_words=following_words,
|
||||
references_text=references_text
|
||||
)
|
||||
|
||||
logger.debug(f"Compiled LangFuse prompt for gap {gap_id}")
|
||||
return compiled
|
||||
|
||||
except Exception as e:
|
||||
raise LangFusePromptError(
|
||||
f"Failed to fetch/compile prompt from LangFuse: {e}"
|
||||
) from e
|
||||
|
||||
def _fetch_prompt(self, name: str, label: str = "production") -> Any:
|
||||
"""Fetch a prompt template from LangFuse.
|
||||
|
||||
Args:
|
||||
name: The prompt name in LangFuse
|
||||
label: Prompt label to fetch (default: "production"). Falls back to
|
||||
version 1 if labeled version not found.
|
||||
|
||||
Returns:
|
||||
LangFuse prompt object
|
||||
|
||||
Raises:
|
||||
LangFusePromptError: If fetch fails
|
||||
"""
|
||||
if not self._client:
|
||||
raise LangFusePromptError("LangFuse client not initialized")
|
||||
|
||||
try:
|
||||
# Try to fetch with the specified label (default: production)
|
||||
prompt = self._client.get_prompt(name, label=label)
|
||||
logger.debug(f"Fetched prompt '{name}' (label={label}) from LangFuse")
|
||||
return prompt
|
||||
except Exception as label_error:
|
||||
# If labeled version not found, try fetching version 1 as fallback
|
||||
# This handles newly created prompts that haven't been promoted yet
|
||||
try:
|
||||
prompt = self._client.get_prompt(name, version=1)
|
||||
logger.warning(
|
||||
f"Prompt '{name}' label '{label}' not found, using version 1. "
|
||||
f"Consider promoting this prompt in LangFuse UI."
|
||||
)
|
||||
return prompt
|
||||
except Exception as version_error:
|
||||
raise LangFusePromptError(
|
||||
f"Failed to fetch prompt '{name}' from LangFuse: "
|
||||
f"Label '{label}' error: {label_error}, "
|
||||
f"Version 1 fallback error: {version_error}"
|
||||
) from version_error
|
||||
|
||||
def _fetch_examples(self) -> List[Dict[str, Any]]:
|
||||
"""Fetch few-shot examples from LangFuse dataset.
|
||||
|
||||
Returns:
|
||||
List of example dictionaries
|
||||
|
||||
Raises:
|
||||
LangFuseDatasetError: If fetch fails
|
||||
"""
|
||||
if not self._client:
|
||||
raise LangFuseDatasetError("LangFuse client not initialized")
|
||||
|
||||
try:
|
||||
dataset = self._client.get_dataset(self.EXAMPLES_DATASET_NAME)
|
||||
examples = []
|
||||
for item in dataset.items:
|
||||
# Dataset items have 'input' field with the example data
|
||||
if hasattr(item, 'input') and item.input:
|
||||
examples.append(item.input)
|
||||
|
||||
logger.debug(f"Fetched {len(examples)} examples from LangFuse dataset")
|
||||
return examples
|
||||
except Exception as e:
|
||||
raise LangFuseDatasetError(
|
||||
f"Failed to fetch dataset '{self.EXAMPLES_DATASET_NAME}' from LangFuse: {e}"
|
||||
) from e
|
||||
|
||||
def _build_song_context(self, artist: Optional[str], title: Optional[str]) -> str:
|
||||
"""Build song context section for the prompt."""
|
||||
if artist and title:
|
||||
return (
|
||||
f"\n## Song Context\n\n"
|
||||
f"**Artist:** {artist}\n"
|
||||
f"**Title:** {title}\n\n"
|
||||
f"Note: The song title and artist name may help identify proper nouns "
|
||||
f"or unusual words that could be mis-heard.\n"
|
||||
)
|
||||
return ""
|
||||
|
||||
def _format_examples(self, examples: List[Dict[str, Any]]) -> str:
|
||||
"""Format few-shot examples for inclusion in prompt.
|
||||
|
||||
Args:
|
||||
examples: List of example dictionaries from LangFuse dataset
|
||||
|
||||
Returns:
|
||||
Formatted examples string
|
||||
"""
|
||||
if not examples:
|
||||
return ""
|
||||
|
||||
# Group examples by category
|
||||
examples_by_category: Dict[str, List[Dict]] = {}
|
||||
for ex in examples:
|
||||
category = ex.get("category", "unknown")
|
||||
if category not in examples_by_category:
|
||||
examples_by_category[category] = []
|
||||
examples_by_category[category].append(ex)
|
||||
|
||||
# Build formatted text
|
||||
text = "## Example Classifications\n\n"
|
||||
for category, category_examples in examples_by_category.items():
|
||||
text += f"### {category.upper().replace('_', ' ')}\n\n"
|
||||
for ex in category_examples[:2]: # Limit to 2 examples per category
|
||||
text += f"**Gap:** {ex.get('gap_text', '')}\n"
|
||||
text += f"**Context:** ...{ex.get('preceding', '')}... [GAP] ...{ex.get('following', '')}...\n"
|
||||
if 'reference' in ex:
|
||||
text += f"**Reference:** {ex['reference']}\n"
|
||||
text += f"**Reasoning:** {ex.get('reasoning', '')}\n"
|
||||
text += f"**Action:** {ex.get('action', '')}\n\n"
|
||||
|
||||
return text
|
||||
|
||||
def _format_references(self, reference_contexts: Dict[str, str]) -> str:
|
||||
"""Format reference lyrics for inclusion in prompt.
|
||||
|
||||
Args:
|
||||
reference_contexts: Dictionary of reference lyrics from each source
|
||||
|
||||
Returns:
|
||||
Formatted references string
|
||||
"""
|
||||
if not reference_contexts:
|
||||
return ""
|
||||
|
||||
text = "## Available Reference Lyrics\n\n"
|
||||
for source, context in reference_contexts.items():
|
||||
text += f"**{source.upper()}:** {context}\n\n"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
# Module-level singleton for convenience
|
||||
_prompt_service: Optional[LangFusePromptService] = None
|
||||
|
||||
|
||||
def get_prompt_service() -> LangFusePromptService:
|
||||
"""Get or create the global prompt service instance.
|
||||
|
||||
Returns:
|
||||
LangFusePromptService singleton instance
|
||||
"""
|
||||
global _prompt_service
|
||||
if _prompt_service is None:
|
||||
_prompt_service = LangFusePromptService()
|
||||
return _prompt_service
|
||||
|
||||
|
||||
def reset_prompt_service() -> None:
|
||||
"""Reset the global prompt service instance (for testing)."""
|
||||
global _prompt_service
|
||||
_prompt_service = None
|
||||
@@ -19,8 +19,9 @@ class ProviderConfig:
|
||||
cache_dir: str
|
||||
|
||||
# GCP/Vertex AI settings
|
||||
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
|
||||
gcp_project_id: Optional[str] = None
|
||||
gcp_location: str = "us-central1"
|
||||
gcp_location: str = "global"
|
||||
|
||||
request_timeout_seconds: float = 30.0
|
||||
max_retries: int = 2
|
||||
@@ -51,7 +52,7 @@ class ProviderConfig:
|
||||
privacy_mode=os.getenv("PRIVACY_MODE", "false").lower() in {"1", "true", "yes"},
|
||||
cache_dir=cache_dir,
|
||||
gcp_project_id=os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID"),
|
||||
gcp_location=os.getenv("GCP_LOCATION", "us-central1"),
|
||||
gcp_location=os.getenv("GCP_LOCATION", "global"),
|
||||
request_timeout_seconds=float(os.getenv("AGENTIC_TIMEOUT_SECONDS", "30.0")),
|
||||
max_retries=int(os.getenv("AGENTIC_MAX_RETRIES", "2")),
|
||||
retry_backoff_base_seconds=float(os.getenv("AGENTIC_BACKOFF_BASE_SECONDS", "0.2")),
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ RESPONSE_LOG_LENGTH = 500 # Characters to log from responses
|
||||
MODEL_SPEC_FORMAT = "provider/model" # Expected format for model identifiers
|
||||
|
||||
# Default Langfuse host
|
||||
DEFAULT_LANGFUSE_HOST = "https://cloud.langfuse.com"
|
||||
DEFAULT_LANGFUSE_HOST = "https://us.cloud.langfuse.com"
|
||||
|
||||
# Raw response indicator
|
||||
RAW_RESPONSE_KEY = "raw" # Key used to wrap unparsed responses
|
||||
|
||||
+22
-7
@@ -187,26 +187,41 @@ class LangChainBridge(BaseAIProvider):
|
||||
|
||||
def _invoke_model(self, prompt: str) -> str:
|
||||
"""Invoke the chat model with a prompt.
|
||||
|
||||
|
||||
This is a simple wrapper that can be passed to the retry executor.
|
||||
|
||||
|
||||
Args:
|
||||
prompt: The prompt to send
|
||||
|
||||
|
||||
Returns:
|
||||
Response content as string
|
||||
|
||||
|
||||
Raises:
|
||||
Exception: Any error from the model invocation
|
||||
"""
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
|
||||
# Prepare config with session_id in metadata (Langfuse format)
|
||||
config = {}
|
||||
if hasattr(self, '_session_id') and self._session_id:
|
||||
config["metadata"] = {"langfuse_session_id": self._session_id}
|
||||
logger.debug(f"🤖 [LangChain] Invoking with session_id: {self._session_id}")
|
||||
|
||||
|
||||
response = self._chat_model.invoke([HumanMessage(content=prompt)], config=config)
|
||||
return response.content
|
||||
content = response.content
|
||||
|
||||
# Handle multimodal response format from Gemini 3+ models
|
||||
# Response can be a list of content parts: [{'type': 'text', 'text': '...'}]
|
||||
if isinstance(content, list):
|
||||
# Extract text from the first text content part
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get('type') == 'text':
|
||||
return part.get('text', '')
|
||||
# Fallback: concatenate all text parts
|
||||
return ''.join(
|
||||
part.get('text', '') if isinstance(part, dict) else str(part)
|
||||
for part in content
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
+3
-12
@@ -100,19 +100,10 @@ class ModelFactory:
|
||||
return
|
||||
|
||||
try:
|
||||
from langfuse import Langfuse
|
||||
from langfuse.langchain import CallbackHandler
|
||||
|
||||
# Initialize Langfuse client first (this is required!)
|
||||
langfuse_client = Langfuse(
|
||||
public_key=public_key,
|
||||
secret_key=secret_key,
|
||||
host=os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com"),
|
||||
)
|
||||
|
||||
# Then create callback handler with the same public_key
|
||||
# The handler will use the initialized client
|
||||
self._langfuse_handler = CallbackHandler(public_key=public_key)
|
||||
|
||||
# CallbackHandler auto-discovers credentials from environment variables
|
||||
self._langfuse_handler = CallbackHandler()
|
||||
logger.info(f"🤖 Langfuse callback handler initialized for {model_spec}")
|
||||
except Exception as e:
|
||||
# If Langfuse keys are set, we MUST fail fast
|
||||
|
||||
@@ -5,10 +5,9 @@ from typing import Dict, Any
|
||||
|
||||
from .providers.config import ProviderConfig
|
||||
|
||||
# Default model for cloud deployments - Gemini 2.0 Flash via Vertex AI
|
||||
# Note: gemini-3-flash-preview requires 'global' location, not us-central1
|
||||
# Using gemini-2.0-flash for broader regional availability
|
||||
DEFAULT_CLOUD_MODEL = "vertexai/gemini-2.0-flash"
|
||||
# Default model for cloud deployments - Gemini 3 Flash via Vertex AI
|
||||
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
|
||||
DEFAULT_CLOUD_MODEL = "vertexai/gemini-3-flash-preview"
|
||||
|
||||
|
||||
class ModelRouter:
|
||||
@@ -21,7 +20,7 @@ class ModelRouter:
|
||||
"""Choose appropriate model based on gap characteristics.
|
||||
|
||||
Returns model identifier in format "provider/model" for LangChain:
|
||||
- "vertexai/gemini-2.0-flash" for Gemini via Vertex AI (default)
|
||||
- "vertexai/gemini-3-flash-preview" for Gemini via Vertex AI (default)
|
||||
- "ollama/llama3.2:latest" for local Ollama models
|
||||
- "openai/gpt-4" for OpenAI models
|
||||
- "anthropic/claude-3-sonnet-20240229" for Anthropic models
|
||||
@@ -35,7 +34,7 @@ class ModelRouter:
|
||||
if self._config.privacy_mode:
|
||||
return "ollama/llama3.2:latest"
|
||||
|
||||
# Default to Gemini 2.0 Flash for all cases (fast, cost-effective)
|
||||
# Default to Gemini 3 Flash for all cases (fast, cost-effective, latest capabilities)
|
||||
return DEFAULT_CLOUD_MODEL
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import os
|
||||
import shortuuid
|
||||
|
||||
from lyrics_transcriber.correction.handlers.levenshtein import LevenshteinHandler
|
||||
from lyrics_transcriber.correction.handlers.llm import LLMHandler
|
||||
from lyrics_transcriber.correction.handlers.no_space_punct_match import NoSpacePunctuationMatchHandler
|
||||
from lyrics_transcriber.correction.handlers.relaxed_word_count_match import RelaxedWordCountMatchHandler
|
||||
from lyrics_transcriber.correction.handlers.repeat import RepeatCorrectionHandler
|
||||
@@ -27,7 +26,6 @@ from lyrics_transcriber.correction.anchor_sequence import AnchorSequenceFinder
|
||||
from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
|
||||
from lyrics_transcriber.correction.handlers.extend_anchor import ExtendAnchorHandler
|
||||
from lyrics_transcriber.utils.word_utils import WordUtils
|
||||
from lyrics_transcriber.correction.handlers.llm_providers import OllamaProvider, OpenAIProvider
|
||||
|
||||
|
||||
class LyricsCorrector:
|
||||
@@ -57,60 +55,18 @@ class LyricsCorrector:
|
||||
]
|
||||
|
||||
# Create all handlers but respect enabled_handlers if provided
|
||||
# Note: Legacy LLMHandler removed - use AgenticCorrector via USE_AGENTIC_AI=1 instead
|
||||
all_handlers = [
|
||||
("ExtendAnchorHandler", ExtendAnchorHandler(logger=self.logger)),
|
||||
("WordCountMatchHandler", WordCountMatchHandler(logger=self.logger)),
|
||||
("SyllablesMatchHandler", SyllablesMatchHandler(logger=self.logger)),
|
||||
("RelaxedWordCountMatchHandler", RelaxedWordCountMatchHandler(logger=self.logger)),
|
||||
("NoSpacePunctuationMatchHandler", NoSpacePunctuationMatchHandler(logger=self.logger)),
|
||||
(
|
||||
"LLMHandler_Ollama_R17B",
|
||||
LLMHandler(
|
||||
provider=OllamaProvider(model="deepseek-r1:7b", logger=self.logger),
|
||||
name="LLMHandler_Ollama_R17B",
|
||||
logger=self.logger,
|
||||
cache_dir=self._cache_dir,
|
||||
),
|
||||
),
|
||||
("RepeatCorrectionHandler", RepeatCorrectionHandler(logger=self.logger)),
|
||||
("SoundAlikeHandler", SoundAlikeHandler(logger=self.logger)),
|
||||
("LevenshteinHandler", LevenshteinHandler(logger=self.logger)),
|
||||
]
|
||||
|
||||
# Add OpenRouter handlers only if API key is available
|
||||
if os.getenv("OPENROUTER_API_KEY"):
|
||||
openrouter_handlers = [
|
||||
(
|
||||
"LLMHandler_OpenRouter_Sonnet",
|
||||
LLMHandler(
|
||||
provider=OpenAIProvider(
|
||||
model="anthropic/claude-3-sonnet",
|
||||
api_key=os.getenv("OPENROUTER_API_KEY"),
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
logger=self.logger,
|
||||
),
|
||||
name="LLMHandler_OpenRouter_Sonnet",
|
||||
logger=self.logger,
|
||||
cache_dir=self._cache_dir,
|
||||
),
|
||||
),
|
||||
(
|
||||
"LLMHandler_OpenRouter_R1",
|
||||
LLMHandler(
|
||||
provider=OpenAIProvider(
|
||||
model="deepseek/deepseek-r1",
|
||||
api_key=os.getenv("OPENROUTER_API_KEY"),
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
logger=self.logger,
|
||||
),
|
||||
name="LLMHandler_OpenRouter_R1",
|
||||
logger=self.logger,
|
||||
cache_dir=self._cache_dir,
|
||||
),
|
||||
),
|
||||
]
|
||||
all_handlers.extend(openrouter_handlers)
|
||||
|
||||
# Store all handler information
|
||||
self.all_handlers = [
|
||||
{
|
||||
|
||||
@@ -1,293 +0,0 @@
|
||||
from typing import List, Optional, Tuple, Dict, Any, Union
|
||||
import logging
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from lyrics_transcriber.types import GapSequence, WordCorrection
|
||||
from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
|
||||
from lyrics_transcriber.correction.handlers.word_operations import WordOperations
|
||||
from lyrics_transcriber.correction.handlers.llm_providers import LLMProvider
|
||||
|
||||
|
||||
class LLMHandler(GapCorrectionHandler):
|
||||
"""Uses an LLM to analyze and correct gaps by comparing with reference lyrics."""
|
||||
|
||||
def __init__(
|
||||
self, provider: LLMProvider, name: str, logger: Optional[logging.Logger] = None, cache_dir: Optional[Union[str, Path]] = None
|
||||
):
|
||||
super().__init__(logger)
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
self.provider = provider
|
||||
self.name = name
|
||||
self.cache_dir = Path(cache_dir) if cache_dir else None
|
||||
|
||||
def _format_prompt(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""Format the prompt for the LLM with context about the gap and reference lyrics."""
|
||||
word_map = data.get("word_map", {})
|
||||
metadata = data.get("metadata", {}) if data else {}
|
||||
|
||||
if not word_map:
|
||||
self.logger.error("No word_map provided in data")
|
||||
return ""
|
||||
|
||||
# Format transcribed words with their IDs
|
||||
transcribed_words = [{"id": word_id, "text": word_map[word_id].text} for word_id in gap.transcribed_word_ids if word_id in word_map]
|
||||
|
||||
prompt = (
|
||||
"You are a lyrics correction expert. You will be given transcribed lyrics that may contain errors "
|
||||
"and reference lyrics from multiple sources. Your task is to analyze each word in the transcribed text "
|
||||
"and suggest specific corrections based on the reference lyrics.\n\n"
|
||||
"Each word has a unique ID. When suggesting corrections, you must specify the ID of the word being corrected. "
|
||||
"This ensures accuracy in applying your corrections.\n\n"
|
||||
"For each correction, specify:\n"
|
||||
"1. The word ID being corrected\n"
|
||||
"2. The correction type ('replace', 'split', 'combine', or 'delete')\n"
|
||||
"3. The corrected text\n"
|
||||
"4. Your confidence level\n"
|
||||
"5. The reason for the correction\n\n"
|
||||
)
|
||||
|
||||
# Add song context if available
|
||||
if metadata and metadata.get("artist") and metadata.get("title"):
|
||||
prompt += f"Song: {metadata['title']}\nArtist: {metadata['artist']}\n\n"
|
||||
|
||||
# Format transcribed words with IDs
|
||||
prompt += "Transcribed words:\n"
|
||||
for word in transcribed_words:
|
||||
prompt += f"- ID: {word['id']}, Text: '{word['text']}'\n"
|
||||
|
||||
prompt += "\nReference lyrics from different sources:\n"
|
||||
|
||||
# Add each reference source with words and their IDs
|
||||
for source, word_ids in gap.reference_word_ids.items():
|
||||
reference_words = [{"id": word_id, "text": word_map[word_id].text} for word_id in word_ids if word_id in word_map]
|
||||
prompt += f"\n{source} immediate context:\n"
|
||||
for word in reference_words:
|
||||
prompt += f"- ID: {word['id']}, Text: '{word['text']}'\n"
|
||||
|
||||
# Add full lyrics if available
|
||||
if metadata and metadata.get("full_reference_texts", {}).get(source):
|
||||
prompt += f"\nFull {source} lyrics:\n{metadata['full_reference_texts'][source]}\n"
|
||||
|
||||
# Add context about surrounding anchors if available
|
||||
if gap.preceding_anchor_id:
|
||||
preceding_anchor = next((a.anchor for a in data.get("anchor_sequences", []) if a.anchor.id == gap.preceding_anchor_id), None)
|
||||
if preceding_anchor:
|
||||
anchor_words = [
|
||||
{"id": word_id, "text": word_map[word_id].text}
|
||||
for word_id in preceding_anchor.transcribed_word_ids
|
||||
if word_id in word_map
|
||||
]
|
||||
prompt += "\nPreceding correct words:\n"
|
||||
for word in anchor_words:
|
||||
prompt += f"- ID: {word['id']}, Text: '{word['text']}'\n"
|
||||
|
||||
prompt += (
|
||||
"\nProvide corrections in the following JSON format:\n"
|
||||
"{\n"
|
||||
' "corrections": [\n'
|
||||
" {\n"
|
||||
' "word_id": "id_of_word_to_correct",\n'
|
||||
' "type": "replace|split|combine|delete",\n'
|
||||
' "corrected_text": "new text",\n'
|
||||
' "reference_word_id": "id_from_reference_lyrics", // Optional, use when matching a specific reference word\n'
|
||||
' "confidence": 0.9,\n'
|
||||
' "reason": "explanation of correction"\n'
|
||||
" }\n"
|
||||
" ]\n"
|
||||
"}\n\n"
|
||||
"Important rules:\n"
|
||||
"1. Always include the word_id for each correction\n"
|
||||
"2. For 'split' type, corrected_text should contain the space-separated words\n"
|
||||
"3. For 'combine' type, word_id should be the first word to combine\n"
|
||||
"4. Include reference_word_id when the correction matches a specific reference word\n"
|
||||
"5. Only suggest corrections when you're confident they improve the lyrics\n"
|
||||
"6. Preserve any existing words that match the reference lyrics\n"
|
||||
"7. Respond ONLY with the JSON object, no other text"
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
def can_handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Dict[str, Any]]:
|
||||
"""LLM handler can attempt to handle any gap with reference words."""
|
||||
if not gap.reference_word_ids:
|
||||
self.logger.debug("No reference words available")
|
||||
return False, {}
|
||||
|
||||
return True, {}
|
||||
|
||||
def _write_debug_info(self, prompt: str, response: str, gap_index: int, audio_file_hash: Optional[str] = None) -> None:
|
||||
"""Write prompt and response to debug files."""
|
||||
if not self.cache_dir:
|
||||
self.logger.warning("No cache directory provided, skipping LLM debug output")
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
debug_dir = self.cache_dir / "llm_debug"
|
||||
debug_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
hash_prefix = f"{audio_file_hash}_" if audio_file_hash else ""
|
||||
filename = debug_dir / f"llm_debug_{hash_prefix}{gap_index}_{timestamp}.txt"
|
||||
|
||||
debug_content = "=== LLM PROMPT ===\n" f"{prompt}\n\n" "=== LLM RESPONSE ===\n" f"{response}\n"
|
||||
|
||||
try:
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
f.write(debug_content)
|
||||
except IOError as e:
|
||||
self.logger.error(f"Failed to write LLM debug file: {e}")
|
||||
|
||||
def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
|
||||
"""Process the gap using the LLM and create corrections based on its response."""
|
||||
if not data or "word_map" not in data:
|
||||
self.logger.error("No word_map provided in data")
|
||||
return []
|
||||
|
||||
word_map = data["word_map"]
|
||||
transcribed_words = [word_map[word_id].text for word_id in gap.transcribed_word_ids if word_id in word_map]
|
||||
|
||||
# Calculate reference positions using the centralized method
|
||||
reference_positions = (
|
||||
WordOperations.calculate_reference_positions(gap, anchor_sequences=data.get("anchor_sequences", [])) or {}
|
||||
) # Ensure empty dict if None
|
||||
|
||||
prompt = self._format_prompt(gap, data)
|
||||
if not prompt:
|
||||
return []
|
||||
|
||||
# Get a unique index for this gap based on its position
|
||||
gap_index = gap.transcription_position
|
||||
|
||||
try:
|
||||
self.logger.debug(f"Processing gap words: {transcribed_words}")
|
||||
self.logger.debug(f"Reference word IDs: {gap.reference_word_ids}")
|
||||
|
||||
response = self.provider.generate_response(prompt)
|
||||
|
||||
# Write debug info to files
|
||||
self._write_debug_info(prompt, response, gap_index, audio_file_hash=data.get("audio_file_hash"))
|
||||
|
||||
try:
|
||||
corrections_data = json.loads(response)
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.error(f"Failed to parse LLM response as JSON: {e}")
|
||||
self.logger.error(f"Raw response content: {response}")
|
||||
return []
|
||||
|
||||
# Check if corrections exist and are non-empty
|
||||
if not corrections_data.get("corrections"):
|
||||
self.logger.debug("No corrections suggested by LLM")
|
||||
return []
|
||||
|
||||
corrections = []
|
||||
for correction in corrections_data["corrections"]:
|
||||
# Validate word_id exists in gap
|
||||
if correction["word_id"] not in gap.transcribed_word_ids:
|
||||
self.logger.error(f"LLM suggested correction for word_id {correction['word_id']} which is not in the gap")
|
||||
continue
|
||||
|
||||
# Get original word from word map
|
||||
original_word = word_map[correction["word_id"]]
|
||||
position = gap.transcription_position + gap.transcribed_word_ids.index(correction["word_id"])
|
||||
|
||||
self.logger.debug(f"Processing correction: {correction}")
|
||||
|
||||
if correction["type"] == "replace":
|
||||
self.logger.debug(
|
||||
f"Creating replacement: '{original_word.text}' -> '{correction['corrected_text']}' " f"at position {position}"
|
||||
)
|
||||
corrections.append(
|
||||
WordOperations.create_word_replacement_correction(
|
||||
original_word=original_word.text,
|
||||
corrected_word=correction["corrected_text"],
|
||||
original_position=position,
|
||||
source="LLM",
|
||||
confidence=correction["confidence"],
|
||||
reason=correction["reason"],
|
||||
handler=self.name,
|
||||
reference_positions=reference_positions,
|
||||
original_word_id=correction["word_id"],
|
||||
corrected_word_id=correction.get("reference_word_id"),
|
||||
)
|
||||
)
|
||||
elif correction["type"] == "split":
|
||||
split_words = correction["corrected_text"].split()
|
||||
self.logger.debug(f"Creating split: '{original_word.text}' -> {split_words} " f"at position {position}")
|
||||
|
||||
# Get reference word IDs if provided
|
||||
reference_word_ids = correction.get("reference_word_ids", [None] * len(split_words))
|
||||
|
||||
corrections.extend(
|
||||
WordOperations.create_word_split_corrections(
|
||||
original_word=original_word.text,
|
||||
reference_words=split_words,
|
||||
original_position=position,
|
||||
source="LLM",
|
||||
confidence=correction["confidence"],
|
||||
reason=correction["reason"],
|
||||
handler=self.name,
|
||||
reference_positions=reference_positions,
|
||||
original_word_id=correction["word_id"],
|
||||
corrected_word_ids=reference_word_ids,
|
||||
)
|
||||
)
|
||||
elif correction["type"] == "combine":
|
||||
# Get all word IDs to combine
|
||||
word_ids_to_combine = []
|
||||
current_idx = gap.transcribed_word_ids.index(correction["word_id"])
|
||||
words_needed = len(correction["corrected_text"].split())
|
||||
|
||||
if current_idx + words_needed <= len(gap.transcribed_word_ids):
|
||||
word_ids_to_combine = gap.transcribed_word_ids[current_idx : current_idx + words_needed]
|
||||
else:
|
||||
self.logger.error(f"Not enough words available to combine at position {position}")
|
||||
continue
|
||||
|
||||
words_to_combine = [word_map[word_id].text for word_id in word_ids_to_combine]
|
||||
|
||||
self.logger.debug(
|
||||
f"Creating combine: {words_to_combine} -> '{correction['corrected_text']}' " f"at position {position}"
|
||||
)
|
||||
|
||||
corrections.extend(
|
||||
WordOperations.create_word_combine_corrections(
|
||||
original_words=words_to_combine,
|
||||
reference_word=correction["corrected_text"],
|
||||
original_position=position,
|
||||
source="LLM",
|
||||
confidence=correction["confidence"],
|
||||
combine_reason=correction["reason"],
|
||||
delete_reason=f"Part of combining words: {correction['reason']}",
|
||||
handler=self.name,
|
||||
reference_positions=reference_positions,
|
||||
original_word_ids=word_ids_to_combine,
|
||||
corrected_word_id=correction.get("reference_word_id"),
|
||||
)
|
||||
)
|
||||
elif correction["type"] == "delete":
|
||||
self.logger.debug(f"Creating deletion: '{original_word.text}' at position {position}")
|
||||
corrections.append(
|
||||
WordCorrection(
|
||||
original_word=original_word.text,
|
||||
corrected_word="",
|
||||
segment_index=0,
|
||||
original_position=position,
|
||||
confidence=correction["confidence"],
|
||||
source="LLM",
|
||||
reason=correction["reason"],
|
||||
alternatives={},
|
||||
is_deletion=True,
|
||||
handler=self.name,
|
||||
reference_positions=reference_positions,
|
||||
word_id=correction["word_id"],
|
||||
corrected_word_id=None,
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.debug(f"Created {len(corrections)} corrections: {[f'{c.original_word}->{c.corrected_word}' for c in corrections]}")
|
||||
return corrections
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Unexpected error in LLM handler: {e}")
|
||||
return []
|
||||
@@ -1,60 +0,0 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional
|
||||
import logging
|
||||
from ollama import chat as ollama_chat
|
||||
import openai
|
||||
|
||||
|
||||
class LLMProvider(ABC):
|
||||
"""Abstract base class for LLM providers."""
|
||||
|
||||
def __init__(self, logger: Optional[logging.Logger] = None):
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
|
||||
@abstractmethod
|
||||
def generate_response(self, prompt: str, **kwargs) -> str:
|
||||
"""Generate a response from the LLM.
|
||||
|
||||
Args:
|
||||
prompt: The prompt to send to the LLM
|
||||
**kwargs: Additional provider-specific parameters
|
||||
|
||||
Returns:
|
||||
str: The LLM's response
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class OllamaProvider(LLMProvider):
|
||||
"""Provider for local Ollama models."""
|
||||
|
||||
def __init__(self, model: str, logger: Optional[logging.Logger] = None):
|
||||
super().__init__(logger)
|
||||
self.model = model
|
||||
|
||||
def generate_response(self, prompt: str, **kwargs) -> str:
|
||||
try:
|
||||
response = ollama_chat(model=self.model, messages=[{"role": "user", "content": prompt}], format="json")
|
||||
return response.message.content
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error generating Ollama response: {e}")
|
||||
raise
|
||||
|
||||
|
||||
class OpenAIProvider(LLMProvider):
|
||||
"""Provider for OpenAI-compatible APIs (including OpenRouter)."""
|
||||
|
||||
def __init__(self, model: str, api_key: str, base_url: Optional[str] = None, logger: Optional[logging.Logger] = None):
|
||||
super().__init__(logger)
|
||||
self.model = model
|
||||
self.client = openai.OpenAI(api_key=api_key, base_url=base_url)
|
||||
|
||||
def generate_response(self, prompt: str, **kwargs) -> str:
|
||||
try:
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model, messages=[{"role": "user", "content": prompt}], response_format={"type": "json_object"}, **kwargs
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error generating OpenAI response: {e}")
|
||||
raise
|
||||
@@ -62,6 +62,25 @@ def test_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def _reset_langfuse_state():
|
||||
"""Helper to reset LangFuse singletons."""
|
||||
try:
|
||||
from lyrics_transcriber.correction.agentic.prompts.langfuse_prompts import reset_prompt_service
|
||||
from lyrics_transcriber.correction.agentic.observability.langfuse_integration import reset_langfuse_client
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
except ImportError:
|
||||
pass # Module may not be available in all test contexts
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_langfuse_singletons():
|
||||
"""Reset LangFuse singletons before and after each test to avoid state leakage."""
|
||||
_reset_langfuse_state()
|
||||
yield # Run the test
|
||||
_reset_langfuse_state()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def cleanup_test_directories():
|
||||
"""Clean up test directories after all tests complete."""
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Tests for LangFuse prompt management integration."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
import os
|
||||
|
||||
from lyrics_transcriber.correction.agentic.prompts.langfuse_prompts import (
|
||||
LangFusePromptService,
|
||||
LangFusePromptError,
|
||||
LangFuseDatasetError,
|
||||
get_prompt_service,
|
||||
reset_prompt_service,
|
||||
)
|
||||
from lyrics_transcriber.correction.agentic.observability.langfuse_integration import (
|
||||
get_langfuse_client,
|
||||
reset_langfuse_client,
|
||||
fetch_prompt,
|
||||
fetch_dataset,
|
||||
is_langfuse_configured,
|
||||
LangFuseConfigError,
|
||||
)
|
||||
|
||||
|
||||
class TestLangFusePromptService:
|
||||
"""Tests for the LangFusePromptService class."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset singletons before each test."""
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
|
||||
def test_service_uses_hardcoded_when_langfuse_not_configured(self):
|
||||
"""Service falls back to hardcoded prompts when LangFuse keys not set."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
# Remove LangFuse keys
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
service = LangFusePromptService()
|
||||
|
||||
# Should not use LangFuse
|
||||
assert not service._use_langfuse
|
||||
assert service._client is None
|
||||
|
||||
def test_service_uses_langfuse_when_configured(self):
|
||||
"""Service uses LangFuse when keys are set."""
|
||||
with patch.dict(os.environ, {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-test",
|
||||
"LANGFUSE_SECRET_KEY": "sk-test",
|
||||
}):
|
||||
with patch("langfuse.Langfuse") as mock_langfuse:
|
||||
mock_client = Mock()
|
||||
mock_langfuse.return_value = mock_client
|
||||
|
||||
service = LangFusePromptService()
|
||||
|
||||
assert service._use_langfuse
|
||||
assert service._client == mock_client
|
||||
|
||||
def test_service_fails_fast_on_init_error(self):
|
||||
"""Service raises error if LangFuse keys set but init fails."""
|
||||
with patch.dict(os.environ, {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-test",
|
||||
"LANGFUSE_SECRET_KEY": "sk-test",
|
||||
}):
|
||||
with patch("langfuse.Langfuse") as mock_langfuse:
|
||||
mock_langfuse.side_effect = Exception("Connection failed")
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
LangFusePromptService()
|
||||
|
||||
assert "initialization failed" in str(excinfo.value)
|
||||
|
||||
def test_get_classification_prompt_hardcoded_fallback(self):
|
||||
"""get_classification_prompt returns hardcoded prompt when LangFuse not configured."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
service = LangFusePromptService()
|
||||
prompt = service.get_classification_prompt(
|
||||
gap_text="out, I'm starting",
|
||||
preceding_words="Oh no, was it",
|
||||
following_words="gonna sleep",
|
||||
reference_contexts={"genius": "now, I'm starting"},
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
gap_id="gap_1",
|
||||
)
|
||||
|
||||
# Should contain key elements of the hardcoded prompt
|
||||
assert "transcription errors" in prompt
|
||||
assert "SOUND_ALIKE" in prompt
|
||||
assert "PUNCTUATION_ONLY" in prompt
|
||||
assert "gap_1" in prompt
|
||||
assert "Test Artist" in prompt
|
||||
|
||||
def test_get_classification_prompt_from_langfuse(self):
|
||||
"""get_classification_prompt fetches from LangFuse when configured."""
|
||||
with patch.dict(os.environ, {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-test",
|
||||
"LANGFUSE_SECRET_KEY": "sk-test",
|
||||
}):
|
||||
# Create mock client
|
||||
mock_client = Mock()
|
||||
|
||||
# Mock prompt
|
||||
mock_prompt = Mock()
|
||||
mock_prompt.compile.return_value = "Compiled prompt with {{gap_text}}"
|
||||
mock_client.get_prompt.return_value = mock_prompt
|
||||
|
||||
# Mock dataset
|
||||
mock_dataset = Mock()
|
||||
mock_item = Mock()
|
||||
mock_item.input = {"category": "sound_alike", "gap_text": "example"}
|
||||
mock_dataset.items = [mock_item]
|
||||
mock_client.get_dataset.return_value = mock_dataset
|
||||
|
||||
with patch("langfuse.Langfuse", return_value=mock_client):
|
||||
service = LangFusePromptService()
|
||||
prompt = service.get_classification_prompt(
|
||||
gap_text="test gap",
|
||||
preceding_words="before",
|
||||
following_words="after",
|
||||
reference_contexts={"source": "reference"},
|
||||
gap_id="gap_1",
|
||||
)
|
||||
|
||||
# Verify prompt was returned from compile
|
||||
assert prompt == "Compiled prompt with {{gap_text}}"
|
||||
# Verify LangFuse was called with label parameter
|
||||
mock_client.get_prompt.assert_called_once_with("gap-classifier", label="production")
|
||||
mock_client.get_dataset.assert_called_once_with("gap-classifier-examples")
|
||||
mock_prompt.compile.assert_called_once()
|
||||
|
||||
def test_service_raises_on_prompt_fetch_error(self):
|
||||
"""Service raises LangFusePromptError on fetch failure."""
|
||||
with patch.dict(os.environ, {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-test",
|
||||
"LANGFUSE_SECRET_KEY": "sk-test",
|
||||
}):
|
||||
mock_client = Mock()
|
||||
mock_client.get_prompt.side_effect = Exception("Network error")
|
||||
|
||||
with patch("langfuse.Langfuse", return_value=mock_client):
|
||||
service = LangFusePromptService()
|
||||
|
||||
with pytest.raises(LangFusePromptError) as excinfo:
|
||||
service.get_classification_prompt(
|
||||
gap_text="test",
|
||||
preceding_words="before",
|
||||
following_words="after",
|
||||
reference_contexts={},
|
||||
)
|
||||
|
||||
assert "Failed to fetch" in str(excinfo.value)
|
||||
|
||||
|
||||
class TestLangFuseIntegration:
|
||||
"""Tests for the langfuse_integration module functions."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset singletons before each test."""
|
||||
reset_langfuse_client()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
reset_langfuse_client()
|
||||
|
||||
def test_is_langfuse_configured_false_when_no_keys(self):
|
||||
"""is_langfuse_configured returns False when keys not set."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
assert not is_langfuse_configured()
|
||||
|
||||
def test_is_langfuse_configured_true_when_keys_set(self):
|
||||
"""is_langfuse_configured returns True when both keys set."""
|
||||
with patch.dict(os.environ, {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-test",
|
||||
"LANGFUSE_SECRET_KEY": "sk-test",
|
||||
}):
|
||||
assert is_langfuse_configured()
|
||||
|
||||
def test_get_langfuse_client_returns_none_when_not_configured(self):
|
||||
"""get_langfuse_client returns None when keys not set."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
client = get_langfuse_client()
|
||||
assert client is None
|
||||
|
||||
def test_get_langfuse_client_fails_fast_on_error(self):
|
||||
"""get_langfuse_client raises LangFuseConfigError on init failure."""
|
||||
with patch.dict(os.environ, {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-test",
|
||||
"LANGFUSE_SECRET_KEY": "sk-test",
|
||||
}):
|
||||
with patch("langfuse.Langfuse") as mock_langfuse:
|
||||
mock_langfuse.side_effect = Exception("Auth failed")
|
||||
|
||||
with pytest.raises(LangFuseConfigError) as excinfo:
|
||||
get_langfuse_client()
|
||||
|
||||
assert "initialization failed" in str(excinfo.value)
|
||||
|
||||
def test_fetch_prompt_raises_when_not_configured(self):
|
||||
"""fetch_prompt raises LangFuseConfigError when not configured."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
with pytest.raises(LangFuseConfigError):
|
||||
fetch_prompt("test-prompt")
|
||||
|
||||
def test_fetch_dataset_raises_when_not_configured(self):
|
||||
"""fetch_dataset raises LangFuseConfigError when not configured."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
with pytest.raises(LangFuseConfigError):
|
||||
fetch_dataset("test-dataset")
|
||||
|
||||
|
||||
class TestPromptServiceSingleton:
|
||||
"""Tests for the prompt service singleton pattern."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset singletons before each test."""
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
|
||||
def test_get_prompt_service_returns_same_instance(self):
|
||||
"""get_prompt_service returns the same instance on subsequent calls."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
service1 = get_prompt_service()
|
||||
service2 = get_prompt_service()
|
||||
|
||||
assert service1 is service2
|
||||
|
||||
def test_reset_prompt_service_clears_singleton(self):
|
||||
"""reset_prompt_service clears the singleton instance."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
service1 = get_prompt_service()
|
||||
reset_prompt_service()
|
||||
service2 = get_prompt_service()
|
||||
|
||||
assert service1 is not service2
|
||||
|
||||
|
||||
class TestBuildClassificationPrompt:
|
||||
"""Tests for the build_classification_prompt function."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset singletons before each test."""
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
reset_prompt_service()
|
||||
reset_langfuse_client()
|
||||
|
||||
def test_build_classification_prompt_works_without_langfuse(self):
|
||||
"""build_classification_prompt works when LangFuse not configured."""
|
||||
from lyrics_transcriber.correction.agentic.prompts.classifier import build_classification_prompt
|
||||
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("LANGFUSE_PUBLIC_KEY", None)
|
||||
os.environ.pop("LANGFUSE_SECRET_KEY", None)
|
||||
|
||||
prompt = build_classification_prompt(
|
||||
gap_text="test gap text",
|
||||
preceding_words="words before",
|
||||
following_words="words after",
|
||||
reference_contexts={"genius": "reference text"},
|
||||
artist="Artist Name",
|
||||
title="Song Title",
|
||||
gap_id="gap_123",
|
||||
)
|
||||
|
||||
# Verify prompt contains expected content
|
||||
assert "test gap text" in prompt
|
||||
assert "words before" in prompt
|
||||
assert "words after" in prompt
|
||||
assert "gap_123" in prompt
|
||||
assert "Artist Name" in prompt
|
||||
assert "Song Title" in prompt
|
||||
assert "GENIUS" in prompt # Source name should be uppercase
|
||||
assert "SOUND_ALIKE" in prompt # Categories should be present
|
||||
@@ -284,9 +284,13 @@ class TestRepeatedSectionHandler:
|
||||
|
||||
class TestClassificationPrompt:
|
||||
"""Test classification prompt generation."""
|
||||
|
||||
def test_prompt_includes_all_context(self):
|
||||
|
||||
def test_prompt_includes_all_context(self, monkeypatch):
|
||||
"""Test that prompt includes all required context."""
|
||||
# Clear LangFuse env vars to force hardcoded fallback
|
||||
monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False)
|
||||
monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False)
|
||||
|
||||
prompt = build_classification_prompt(
|
||||
gap_text="out I'm starting over",
|
||||
preceding_words="Oh no, was it worth it? Starting",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.81.4"
|
||||
version = "0.82.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
+181
-13
@@ -1,18 +1,32 @@
|
||||
#!/bin/bash
|
||||
# Debug a specific karaoke job
|
||||
# Usage: ./scripts/debug-job.sh <job_id>
|
||||
#
|
||||
# Requires KARAOKE_GEN_AUTH_TOKEN env var to be set to a valid admin token.
|
||||
# Get your token from the GCP Secret Manager or .env file.
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: $0 <job_id>"
|
||||
echo "Example: $0 e168cb20"
|
||||
echo ""
|
||||
echo "Requires KARAOKE_GEN_AUTH_TOKEN environment variable (admin token)."
|
||||
echo "Set it with: export KARAOKE_GEN_AUTH_TOKEN=your-admin-token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
JOB_ID="$1"
|
||||
BACKEND_URL="${BACKEND_URL:-https://karaoke-backend-ipzqd2k4yq-uc.a.run.app}"
|
||||
AUTH_TOKEN="${AUTH_TOKEN:-$(gcloud auth print-identity-token)}"
|
||||
BACKEND_URL="${BACKEND_URL:-https://api.nomadkaraoke.com}"
|
||||
AUTH_TOKEN="${KARAOKE_GEN_AUTH_TOKEN}"
|
||||
|
||||
if [ -z "$AUTH_TOKEN" ]; then
|
||||
echo "Error: KARAOKE_GEN_AUTH_TOKEN environment variable is not set."
|
||||
echo ""
|
||||
echo "Set it with: export KARAOKE_GEN_AUTH_TOKEN=your-admin-token"
|
||||
echo "You can find admin tokens in GCP Secret Manager or your .env file."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
@@ -29,14 +43,63 @@ echo ""
|
||||
# 1. Get job status
|
||||
echo -e "${BLUE}1. Job Status:${NC}"
|
||||
echo "---"
|
||||
JOB_STATUS=$(curl -s -H "Authorization: Bearer $AUTH_TOKEN" "$BACKEND_URL/api/jobs/$JOB_ID")
|
||||
echo "$JOB_STATUS" | jq '{status, progress, error_message, error_details, timeline: .timeline | length}'
|
||||
|
||||
# Capture both response body and HTTP status code
|
||||
HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: Bearer $AUTH_TOKEN" "$BACKEND_URL/api/jobs/$JOB_ID")
|
||||
HTTP_STATUS=$(echo "$HTTP_RESPONSE" | tail -n 1)
|
||||
JOB_STATUS=$(echo "$HTTP_RESPONSE" | sed '$d')
|
||||
|
||||
# Check HTTP status
|
||||
if [ "$HTTP_STATUS" != "200" ]; then
|
||||
echo -e "${RED}API Error (HTTP $HTTP_STATUS):${NC}"
|
||||
echo "$JOB_STATUS" | jq -r '.detail // .' 2>/dev/null || echo "$JOB_STATUS"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Possible causes:${NC}"
|
||||
case "$HTTP_STATUS" in
|
||||
401)
|
||||
echo " - Invalid or expired AUTH_TOKEN"
|
||||
echo " - Token doesn't have access to this job"
|
||||
;;
|
||||
403)
|
||||
echo " - Token lacks permission to view this job"
|
||||
echo " - Job belongs to a different user (need admin token)"
|
||||
;;
|
||||
404)
|
||||
echo " - Job ID '$JOB_ID' not found"
|
||||
echo " - Job may have been deleted"
|
||||
;;
|
||||
*)
|
||||
echo " - Server error or network issue"
|
||||
;;
|
||||
esac
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$JOB_STATUS" | jq '{status, progress, error_message, error_details, timeline: (.timeline | length)}'
|
||||
echo ""
|
||||
|
||||
# 2. Get full timeline
|
||||
echo -e "${BLUE}2. Timeline (last 10 events):${NC}"
|
||||
echo "---"
|
||||
echo "$JOB_STATUS" | jq -r '.timeline[-10:][] | "\(.timestamp) [\(.status)] \(.message)"'
|
||||
TIMELINE_COUNT=$(echo "$JOB_STATUS" | jq '.timeline | length')
|
||||
if [ "$TIMELINE_COUNT" = "0" ] || [ "$TIMELINE_COUNT" = "null" ]; then
|
||||
echo "(no timeline events)"
|
||||
else
|
||||
echo "$JOB_STATUS" | jq -r '.timeline[-10:][] | "\(.timestamp) [\(.status)] \(.message // "(no message)")"'
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 2b. Get worker logs (more accurate real-time status)
|
||||
echo -e "${BLUE}2b. Worker Logs (last 10):${NC}"
|
||||
echo "---"
|
||||
LOGS_RESPONSE=$(curl -s -H "Authorization: Bearer $AUTH_TOKEN" "$BACKEND_URL/api/jobs/$JOB_ID/logs?since_index=0")
|
||||
LOGS_STATUS=$(echo "$LOGS_RESPONSE" | jq -r '.total_logs // 0')
|
||||
if [ "$LOGS_STATUS" = "0" ] || [ "$LOGS_STATUS" = "null" ]; then
|
||||
echo "(no worker logs)"
|
||||
else
|
||||
echo "Total logs: $LOGS_STATUS"
|
||||
echo "$LOGS_RESPONSE" | jq -r '.logs[-10:][] | "\(.timestamp) [\(.worker)] \(.level): \(.message)"' 2>/dev/null || echo "(error parsing logs)"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 3. Check if files exist in GCS
|
||||
@@ -77,30 +140,135 @@ gcloud logging read "resource.type=cloud_run_revision AND resource.labels.servic
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 5. Check Firestore document
|
||||
echo -e "${BLUE}5. Firestore Document Fields:${NC}"
|
||||
# 5. Job Details
|
||||
echo -e "${BLUE}5. Job Details:${NC}"
|
||||
echo "---"
|
||||
echo "$JOB_STATUS" | jq 'keys | sort'
|
||||
echo "$JOB_STATUS" | jq '{
|
||||
artist,
|
||||
title,
|
||||
user_email,
|
||||
created_at,
|
||||
updated_at,
|
||||
prep_only,
|
||||
finalise_only
|
||||
}'
|
||||
echo ""
|
||||
|
||||
# 6. Summary
|
||||
# 6. File URLs
|
||||
echo -e "${BLUE}6. File URLs:${NC}"
|
||||
echo "---"
|
||||
echo "$JOB_STATUS" | jq '.file_urls // {}' | head -50
|
||||
echo ""
|
||||
|
||||
# 7. Summary
|
||||
echo "========================================"
|
||||
echo -e "${BLUE}Summary:${NC}"
|
||||
ARTIST=$(echo "$JOB_STATUS" | jq -r '.artist // "Unknown"')
|
||||
TITLE=$(echo "$JOB_STATUS" | jq -r '.title // "Unknown"')
|
||||
STATUS=$(echo "$JOB_STATUS" | jq -r '.status')
|
||||
ERROR=$(echo "$JOB_STATUS" | jq -r '.error_message // "None"')
|
||||
PROGRESS=$(echo "$JOB_STATUS" | jq -r '.progress // 0')
|
||||
|
||||
echo "Track: $ARTIST - $TITLE"
|
||||
|
||||
# Compute enhanced status from state_data (like frontend does)
|
||||
# The backend status can be stale during parallel processing
|
||||
AUDIO_STAGE=$(echo "$JOB_STATUS" | jq -r '.state_data.audio_progress.stage // empty')
|
||||
LYRICS_STAGE=$(echo "$JOB_STATUS" | jq -r '.state_data.lyrics_progress.stage // empty')
|
||||
AUDIO_COMPLETE=$(echo "$JOB_STATUS" | jq -r '.state_data.audio_complete // false')
|
||||
LYRICS_COMPLETE=$(echo "$JOB_STATUS" | jq -r '.state_data.lyrics_complete // false')
|
||||
|
||||
# Determine effective status (mirrors frontend job-status.ts logic)
|
||||
EFFECTIVE_STATUS="$STATUS"
|
||||
STEP_INFO=""
|
||||
|
||||
if [ "$STATUS" = "downloading" ]; then
|
||||
# Check if parallel workers are active
|
||||
if [ -n "$AUDIO_STAGE" ] || [ -n "$LYRICS_STAGE" ]; then
|
||||
EFFECTIVE_STATUS="processing"
|
||||
# Build status string
|
||||
PARTS=""
|
||||
if [ "$AUDIO_COMPLETE" != "true" ] && [ -n "$AUDIO_STAGE" ]; then
|
||||
case "$AUDIO_STAGE" in
|
||||
"separating_stage1") PARTS="Audio 1/2" ;;
|
||||
"separating_stage2") PARTS="Audio 2/2" ;;
|
||||
"audio_complete") PARTS="Audio done" ;;
|
||||
*) PARTS="Audio" ;;
|
||||
esac
|
||||
fi
|
||||
if [ "$LYRICS_COMPLETE" != "true" ] && [ -n "$LYRICS_STAGE" ]; then
|
||||
case "$LYRICS_STAGE" in
|
||||
"transcribing") LYRICS_PART="Transcribing" ;;
|
||||
"correcting") LYRICS_PART="Correcting" ;;
|
||||
"lyrics_complete") LYRICS_PART="Lyrics done" ;;
|
||||
*) LYRICS_PART="Lyrics" ;;
|
||||
esac
|
||||
if [ -n "$PARTS" ]; then
|
||||
PARTS="$PARTS + $LYRICS_PART"
|
||||
else
|
||||
PARTS="$LYRICS_PART"
|
||||
fi
|
||||
fi
|
||||
if [ -n "$PARTS" ]; then
|
||||
STEP_INFO="[4/10] $PARTS"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Map status to step (simplified version of frontend logic)
|
||||
if [ -z "$STEP_INFO" ]; then
|
||||
case "$STATUS" in
|
||||
"pending") STEP_INFO="[1/10] Setting up" ;;
|
||||
"searching_audio") STEP_INFO="[2/10] Searching for audio" ;;
|
||||
"awaiting_audio_selection") STEP_INFO="[2/10] Select audio source" ;;
|
||||
"downloading_audio"|"downloading") STEP_INFO="[3/10] Downloading" ;;
|
||||
"separating_stage1") STEP_INFO="[4/10] Separating audio (1/2)" ;;
|
||||
"separating_stage2") STEP_INFO="[4/10] Separating audio (2/2)" ;;
|
||||
"transcribing") STEP_INFO="[4/10] Transcribing lyrics" ;;
|
||||
"correcting") STEP_INFO="[4/10] Correcting lyrics" ;;
|
||||
"generating_screens") STEP_INFO="[5/10] Generating screens" ;;
|
||||
"awaiting_review"|"in_review") STEP_INFO="[6/10] Review lyrics" ;;
|
||||
"rendering_video") STEP_INFO="[7/10] Rendering video" ;;
|
||||
"awaiting_instrumental_selection") STEP_INFO="[8/10] Select instrumental" ;;
|
||||
"generating_video"|"encoding"|"packaging") STEP_INFO="[9/10] Encoding" ;;
|
||||
"complete"|"prep_complete") STEP_INFO="[10/10] Complete" ;;
|
||||
*) STEP_INFO="$STATUS" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
echo "Status: $STEP_INFO"
|
||||
if [ "$STATUS" != "$EFFECTIVE_STATUS" ] && [ "$EFFECTIVE_STATUS" = "processing" ]; then
|
||||
echo -e "${YELLOW}(Backend status '$STATUS' is stale - workers are actively processing)${NC}"
|
||||
fi
|
||||
|
||||
case "$STATUS" in
|
||||
"completed")
|
||||
"complete")
|
||||
echo -e "${GREEN}✓ Job completed successfully${NC}"
|
||||
;;
|
||||
"prep_complete")
|
||||
echo -e "${GREEN}✓ Prep completed (ready for finalization)${NC}"
|
||||
;;
|
||||
"failed")
|
||||
echo -e "${RED}✗ Job failed: $ERROR${NC}"
|
||||
# Show error details if available
|
||||
ERROR_DETAILS=$(echo "$JOB_STATUS" | jq -r '.error_details // empty')
|
||||
if [ -n "$ERROR_DETAILS" ] && [ "$ERROR_DETAILS" != "null" ]; then
|
||||
echo -e "${RED}Error details:${NC}"
|
||||
echo "$JOB_STATUS" | jq '.error_details'
|
||||
fi
|
||||
;;
|
||||
"pending"|"processing"*)
|
||||
echo -e "${YELLOW}⏳ Job in progress: $STATUS${NC}"
|
||||
"awaiting_review"|"awaiting_instrumental_selection"|"awaiting_audio_selection")
|
||||
echo -e "${YELLOW}⏸️ Waiting for user action${NC}"
|
||||
;;
|
||||
"cancelled")
|
||||
echo -e "${YELLOW}🚫 Job cancelled${NC}"
|
||||
;;
|
||||
*)
|
||||
echo -e "Status: $STATUS"
|
||||
if [ "$EFFECTIVE_STATUS" = "processing" ]; then
|
||||
echo -e "${BLUE}⏳ Workers actively processing${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⏳ Job in progress${NC}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
echo "========================================"
|
||||
|
||||
Executable
+353
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migrate hardcoded prompts and examples to LangFuse.
|
||||
|
||||
This script uploads:
|
||||
1. The gap-classifier prompt template
|
||||
2. The few-shot examples as a dataset
|
||||
|
||||
Usage:
|
||||
# Dry run (preview what would be uploaded)
|
||||
python scripts/migrate-prompts-to-langfuse.py --dry-run
|
||||
|
||||
# Actual migration
|
||||
python scripts/migrate-prompts-to-langfuse.py
|
||||
|
||||
# Force overwrite existing prompts/datasets
|
||||
python scripts/migrate-prompts-to-langfuse.py --force
|
||||
|
||||
Environment Variables Required:
|
||||
LANGFUSE_PUBLIC_KEY
|
||||
LANGFUSE_SECRET_KEY
|
||||
LANGFUSE_HOST (optional, defaults to https://us.cloud.langfuse.com)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add the project root to the path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root / "lyrics_transcriber_temp"))
|
||||
|
||||
|
||||
def get_langfuse_client():
|
||||
"""Initialize and return the Langfuse client."""
|
||||
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
secret_key = os.getenv("LANGFUSE_SECRET_KEY")
|
||||
host = os.getenv("LANGFUSE_HOST", "https://us.cloud.langfuse.com")
|
||||
|
||||
if not (public_key and secret_key):
|
||||
print("Error: LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY must be set")
|
||||
sys.exit(1)
|
||||
|
||||
from langfuse import Langfuse
|
||||
|
||||
return Langfuse(
|
||||
public_key=public_key,
|
||||
secret_key=secret_key,
|
||||
host=host,
|
||||
)
|
||||
|
||||
|
||||
def get_prompt_template() -> str:
|
||||
"""Get the prompt template with LangFuse variable placeholders.
|
||||
|
||||
This template uses {{variable}} syntax for LangFuse variable substitution.
|
||||
"""
|
||||
return """You are an expert at analyzing transcription errors in song lyrics. Your task is to classify gaps (mismatches between transcription and reference lyrics) into categories to determine the best correction approach.
|
||||
|
||||
{{song_context}}
|
||||
|
||||
## Categories
|
||||
|
||||
Use these EXACT category names in your response:
|
||||
|
||||
1. **PUNCTUATION_ONLY**: Only difference is punctuation, capitalization, or symbols (hyphens, quotes). No text changes needed.
|
||||
|
||||
2. **SOUND_ALIKE**: Transcription mis-heard words that sound similar (e.g., "out" vs "now", "said to watch" vs "set the watch"). Common for homophones or similar-sounding phrases.
|
||||
|
||||
3. **BACKGROUND_VOCALS**: Transcription includes backing vocals (usually in parentheses) that aren't in the main reference lyrics. Should typically be removed for karaoke.
|
||||
|
||||
4. **EXTRA_WORDS**: Transcription adds common filler words like "And", "But", "Well" at sentence starts that aren't in reference lyrics.
|
||||
|
||||
5. **REPEATED_SECTION**: Transcription shows repeated chorus/lyrics that may or may not appear in condensed reference lyrics. Often needs human verification via audio.
|
||||
|
||||
6. **COMPLEX_MULTI_ERROR**: Large gaps (many words) with multiple different error types. Too complex for automatic correction.
|
||||
|
||||
7. **NO_ERROR**: At least one reference source matches the transcription exactly, indicating the transcription is correct and other references are incomplete/wrong.
|
||||
|
||||
8. **AMBIGUOUS**: Cannot determine correct action without listening to audio. Similar to repeated sections but less clear.
|
||||
|
||||
{{examples_text}}
|
||||
|
||||
## Gap to Classify
|
||||
|
||||
**Gap ID:** {{gap_id}}
|
||||
|
||||
**Preceding Context:** {{preceding_words}}
|
||||
|
||||
**Gap Text:** {{gap_text}}
|
||||
|
||||
**Following Context:** {{following_words}}
|
||||
|
||||
{{references_text}}
|
||||
|
||||
## Important Guidelines
|
||||
|
||||
- If ANY reference source matches the gap text exactly (ignoring punctuation), classify as **NO_ERROR**
|
||||
- Consider whether the song title/artist contains words that might appear in the gap
|
||||
- Parentheses in transcription usually indicate background vocals
|
||||
- Sound-alike errors are very common in song transcription
|
||||
- Flag for human review when uncertain
|
||||
|
||||
## Your Task
|
||||
|
||||
Analyze this gap and respond with a JSON object matching this schema:
|
||||
|
||||
{
|
||||
"gap_id": "{{gap_id}}",
|
||||
"category": "<one of the 8 categories above>",
|
||||
"confidence": <float between 0 and 1>,
|
||||
"reasoning": "<detailed explanation for your classification>",
|
||||
"suggested_handler": "<name of handler or null>"
|
||||
}
|
||||
|
||||
Provide ONLY the JSON response, no other text."""
|
||||
|
||||
|
||||
def get_examples():
|
||||
"""Get the few-shot examples from the hardcoded source."""
|
||||
from lyrics_transcriber.correction.agentic.prompts.classifier import get_hardcoded_examples
|
||||
|
||||
examples = get_hardcoded_examples()
|
||||
|
||||
# Flatten the examples by category into a list
|
||||
flattened = []
|
||||
for category, category_examples in examples.items():
|
||||
for ex in category_examples:
|
||||
flattened.append({
|
||||
"category": category,
|
||||
**ex
|
||||
})
|
||||
|
||||
return flattened
|
||||
|
||||
|
||||
def upload_prompt(client, dry_run: bool = False, force: bool = False) -> bool:
|
||||
"""Upload the prompt template to LangFuse.
|
||||
|
||||
Args:
|
||||
client: Langfuse client
|
||||
dry_run: If True, only print what would be done
|
||||
force: If True, overwrite existing prompt
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
prompt_name = "gap-classifier"
|
||||
template = get_prompt_template()
|
||||
|
||||
print(f"\n{'[DRY RUN] ' if dry_run else ''}Uploading prompt: {prompt_name}")
|
||||
print(f" Template length: {len(template)} characters")
|
||||
|
||||
if dry_run:
|
||||
print(" Preview (first 500 chars):")
|
||||
print(f" {template[:500]}...")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Check if prompt exists
|
||||
try:
|
||||
existing = client.get_prompt(prompt_name)
|
||||
if existing and not force:
|
||||
print(f" Prompt '{prompt_name}' already exists. Use --force to overwrite.")
|
||||
return True
|
||||
except Exception:
|
||||
pass # Prompt doesn't exist, which is fine
|
||||
|
||||
# Create the prompt
|
||||
client.create_prompt(
|
||||
name=prompt_name,
|
||||
prompt=template,
|
||||
)
|
||||
print(f" Successfully created prompt '{prompt_name}'")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error uploading prompt: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def upload_examples(client, dry_run: bool = False, force: bool = False) -> bool:
|
||||
"""Upload the few-shot examples as a LangFuse dataset.
|
||||
|
||||
Args:
|
||||
client: Langfuse client
|
||||
dry_run: If True, only print what would be done
|
||||
force: If True, overwrite existing dataset
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
dataset_name = "gap-classifier-examples"
|
||||
examples = get_examples()
|
||||
|
||||
print(f"\n{'[DRY RUN] ' if dry_run else ''}Uploading dataset: {dataset_name}")
|
||||
print(f" Total examples: {len(examples)}")
|
||||
|
||||
# Count by category
|
||||
categories = {}
|
||||
for ex in examples:
|
||||
cat = ex.get("category", "unknown")
|
||||
categories[cat] = categories.get(cat, 0) + 1
|
||||
print(" Examples by category:")
|
||||
for cat, count in sorted(categories.items()):
|
||||
print(f" {cat}: {count}")
|
||||
|
||||
if dry_run:
|
||||
print("\n Sample examples:")
|
||||
for ex in examples[:3]:
|
||||
print(f" - {ex.get('category')}: {ex.get('gap_text', '')[:50]}...")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Try to get existing dataset
|
||||
dataset = None
|
||||
try:
|
||||
dataset = client.get_dataset(dataset_name)
|
||||
if dataset and not force:
|
||||
print(f" Dataset '{dataset_name}' already exists. Use --force to overwrite.")
|
||||
return True
|
||||
except Exception:
|
||||
pass # Dataset doesn't exist
|
||||
|
||||
# Create dataset if it doesn't exist
|
||||
if dataset is None:
|
||||
dataset = client.create_dataset(
|
||||
name=dataset_name,
|
||||
description="Few-shot examples for gap classification in lyrics correction",
|
||||
)
|
||||
print(f" Created dataset '{dataset_name}'")
|
||||
|
||||
# Upload each example as a dataset item
|
||||
for i, ex in enumerate(examples):
|
||||
client.create_dataset_item(
|
||||
dataset_name=dataset_name,
|
||||
input=ex,
|
||||
metadata={
|
||||
"category": ex.get("category"),
|
||||
"index": i,
|
||||
},
|
||||
)
|
||||
|
||||
print(f" Successfully uploaded {len(examples)} examples to '{dataset_name}'")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error uploading dataset: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def verify_roundtrip(client) -> bool:
|
||||
"""Verify that prompts and datasets can be fetched back.
|
||||
|
||||
Args:
|
||||
client: Langfuse client
|
||||
|
||||
Returns:
|
||||
True if verification passes, False otherwise
|
||||
"""
|
||||
print("\nVerifying round-trip...")
|
||||
|
||||
try:
|
||||
# Verify prompt
|
||||
prompt = client.get_prompt("gap-classifier")
|
||||
if not prompt:
|
||||
print(" Error: Could not fetch prompt 'gap-classifier'")
|
||||
return False
|
||||
print(f" Prompt 'gap-classifier' verified (length: {len(prompt.prompt)} chars)")
|
||||
|
||||
# Verify dataset
|
||||
dataset = client.get_dataset("gap-classifier-examples")
|
||||
if not dataset:
|
||||
print(" Error: Could not fetch dataset 'gap-classifier-examples'")
|
||||
return False
|
||||
|
||||
item_count = len(list(dataset.items))
|
||||
print(f" Dataset 'gap-classifier-examples' verified ({item_count} items)")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" Verification failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Migrate hardcoded prompts to LangFuse"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Preview what would be uploaded without making changes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Overwrite existing prompts and datasets",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verify-only",
|
||||
action="store_true",
|
||||
help="Only verify existing prompts/datasets can be fetched",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print("LangFuse Prompt Migration")
|
||||
print("=" * 50)
|
||||
|
||||
if args.dry_run:
|
||||
print("Mode: DRY RUN (no changes will be made)")
|
||||
elif args.force:
|
||||
print("Mode: FORCE (will overwrite existing)")
|
||||
else:
|
||||
print("Mode: Normal (will skip existing)")
|
||||
|
||||
# Get client
|
||||
client = get_langfuse_client()
|
||||
print(f"Connected to LangFuse at {os.getenv('LANGFUSE_HOST', 'https://us.cloud.langfuse.com')}")
|
||||
|
||||
if args.verify_only:
|
||||
success = verify_roundtrip(client)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
# Upload prompt
|
||||
prompt_ok = upload_prompt(client, dry_run=args.dry_run, force=args.force)
|
||||
|
||||
# Upload examples
|
||||
examples_ok = upload_examples(client, dry_run=args.dry_run, force=args.force)
|
||||
|
||||
# Verify if not dry run
|
||||
if not args.dry_run and prompt_ok and examples_ok:
|
||||
verify_ok = verify_roundtrip(client)
|
||||
else:
|
||||
verify_ok = True
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 50)
|
||||
print("Summary:")
|
||||
print(f" Prompt upload: {'OK' if prompt_ok else 'FAILED'}")
|
||||
print(f" Examples upload: {'OK' if examples_ok else 'FAILED'}")
|
||||
if not args.dry_run:
|
||||
print(f" Verification: {'OK' if verify_ok else 'FAILED'}")
|
||||
|
||||
success = prompt_ok and examples_ok and (args.dry_run or verify_ok)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+266
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for the agentic correction workflow.
|
||||
|
||||
This script allows testing the agentic correction locally without deploying
|
||||
to Cloud Run or running a full karaoke job.
|
||||
|
||||
Usage:
|
||||
# Test with default Vertex AI (requires gcloud auth):
|
||||
python scripts/test_agentic_correction.py
|
||||
|
||||
# Test with a specific model:
|
||||
AGENTIC_AI_MODEL=openai/gpt-4o python scripts/test_agentic_correction.py
|
||||
AGENTIC_AI_MODEL=anthropic/claude-3-5-sonnet-20241022 python scripts/test_agentic_correction.py
|
||||
|
||||
# Test with local Ollama:
|
||||
PRIVACY_MODE=1 python scripts/test_agentic_correction.py
|
||||
|
||||
Environment variables:
|
||||
GOOGLE_CLOUD_PROJECT: GCP project ID (required for Vertex AI)
|
||||
GCP_LOCATION: Vertex AI location (default: global)
|
||||
AGENTIC_AI_MODEL: Override the model (e.g., openai/gpt-4o)
|
||||
PRIVACY_MODE: Set to 1 to use local Ollama
|
||||
LANGFUSE_PUBLIC_KEY: Langfuse public key for tracing
|
||||
LANGFUSE_SECRET_KEY: Langfuse secret key for tracing
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Dict, Any
|
||||
|
||||
# Add the project to the path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
datefmt="%H:%M:%S"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sample test cases representing different gap categories
|
||||
TEST_CASES = [
|
||||
{
|
||||
"name": "Sound-alike error (wurld -> world)",
|
||||
"gap_id": "test_gap_1",
|
||||
"gap_words": [
|
||||
{"id": "w1", "text": "wurld", "start_time": 0.5, "end_time": 1.0}
|
||||
],
|
||||
"preceding_words": "hello",
|
||||
"following_words": "this is a test",
|
||||
"reference_contexts": {
|
||||
"genius": "hello world this is a test",
|
||||
"lrclib": "Hello world, this is a test"
|
||||
},
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
},
|
||||
{
|
||||
"name": "Punctuation difference",
|
||||
"gap_id": "test_gap_2",
|
||||
"gap_words": [
|
||||
{"id": "w2", "text": "Hello", "start_time": 0.0, "end_time": 0.5}
|
||||
],
|
||||
"preceding_words": "",
|
||||
"following_words": "is it me you're looking for",
|
||||
"reference_contexts": {
|
||||
"genius": "Hello, is it me you're looking for?"
|
||||
},
|
||||
"artist": "Lionel Richie",
|
||||
"title": "Hello"
|
||||
},
|
||||
{
|
||||
"name": "Extra filler word",
|
||||
"gap_id": "test_gap_3",
|
||||
"gap_words": [
|
||||
{"id": "w3", "text": "And", "start_time": 2.0, "end_time": 2.2},
|
||||
{"id": "w4", "text": "I", "start_time": 2.2, "end_time": 2.4},
|
||||
{"id": "w5", "text": "love", "start_time": 2.4, "end_time": 2.6},
|
||||
{"id": "w6", "text": "you", "start_time": 2.6, "end_time": 2.8}
|
||||
],
|
||||
"preceding_words": "baby",
|
||||
"following_words": "so much",
|
||||
"reference_contexts": {
|
||||
"genius": "baby I love you so much"
|
||||
},
|
||||
"artist": "Test Artist",
|
||||
"title": "Love Song"
|
||||
},
|
||||
{
|
||||
"name": "Complex multi-word error",
|
||||
"gap_id": "test_gap_4",
|
||||
"gap_words": [
|
||||
{"id": "w7", "text": "gonna", "start_time": 3.0, "end_time": 3.2},
|
||||
{"id": "w8", "text": "give", "start_time": 3.2, "end_time": 3.4},
|
||||
{"id": "w9", "text": "you", "start_time": 3.4, "end_time": 3.6},
|
||||
{"id": "w10", "text": "up", "start_time": 3.6, "end_time": 3.8}
|
||||
],
|
||||
"preceding_words": "never",
|
||||
"following_words": "never gonna let you down",
|
||||
"reference_contexts": {
|
||||
"genius": "Never gonna give you up, never gonna let you down"
|
||||
},
|
||||
"artist": "Rick Astley",
|
||||
"title": "Never Gonna Give You Up"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def run_single_test(test_case: Dict[str, Any], agent) -> Dict[str, Any]:
|
||||
"""Run a single test case and return results."""
|
||||
logger.info(f"\n{'='*60}")
|
||||
logger.info(f"Test: {test_case['name']}")
|
||||
logger.info(f"Gap words: {[w['text'] for w in test_case['gap_words']]}")
|
||||
logger.info(f"Reference: {list(test_case['reference_contexts'].values())[0][:50]}...")
|
||||
logger.info(f"{'='*60}")
|
||||
|
||||
try:
|
||||
proposals = agent.propose_for_gap(
|
||||
gap_id=test_case["gap_id"],
|
||||
gap_words=test_case["gap_words"],
|
||||
preceding_words=test_case["preceding_words"],
|
||||
following_words=test_case["following_words"],
|
||||
reference_contexts=test_case["reference_contexts"],
|
||||
artist=test_case.get("artist"),
|
||||
title=test_case.get("title")
|
||||
)
|
||||
|
||||
result = {
|
||||
"test_name": test_case["name"],
|
||||
"success": True,
|
||||
"proposals": []
|
||||
}
|
||||
|
||||
if not proposals:
|
||||
logger.info("No proposals returned (gap may be correct)")
|
||||
result["proposals"] = []
|
||||
else:
|
||||
for p in proposals:
|
||||
proposal_dict = {
|
||||
"action": p.action,
|
||||
"word_id": p.word_id,
|
||||
"word_ids": p.word_ids,
|
||||
"replacement_text": p.replacement_text,
|
||||
"confidence": p.confidence,
|
||||
"reason": p.reason,
|
||||
"gap_category": str(p.gap_category) if p.gap_category else None,
|
||||
"requires_human_review": p.requires_human_review
|
||||
}
|
||||
result["proposals"].append(proposal_dict)
|
||||
|
||||
logger.info(f"\nProposal:")
|
||||
logger.info(f" Action: {p.action}")
|
||||
logger.info(f" Word ID(s): {p.word_id or p.word_ids}")
|
||||
if p.replacement_text:
|
||||
logger.info(f" Replacement: {p.replacement_text}")
|
||||
logger.info(f" Confidence: {p.confidence:.2f}")
|
||||
logger.info(f" Category: {p.gap_category}")
|
||||
logger.info(f" Reason: {p.reason}")
|
||||
if p.requires_human_review:
|
||||
logger.info(f" ⚠️ Requires human review")
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return {
|
||||
"test_name": test_case["name"],
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
# Check environment
|
||||
project_id = os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT_ID")
|
||||
location = os.getenv("GCP_LOCATION", "global")
|
||||
model_override = os.getenv("AGENTIC_AI_MODEL")
|
||||
privacy_mode = os.getenv("PRIVACY_MODE", "").lower() in ("1", "true", "yes")
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("AGENTIC CORRECTION TEST SCRIPT")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"GCP Project: {project_id or 'not set'}")
|
||||
logger.info(f"GCP Location: {location}")
|
||||
logger.info(f"Model Override: {model_override or 'none (using default)'}")
|
||||
logger.info(f"Privacy Mode: {privacy_mode}")
|
||||
logger.info(f"Langfuse: {'enabled' if os.getenv('LANGFUSE_PUBLIC_KEY') else 'disabled'}")
|
||||
|
||||
# Import after path setup
|
||||
try:
|
||||
from lyrics_transcriber.correction.agentic.agent import AgenticCorrector
|
||||
from lyrics_transcriber.correction.agentic.router import ModelRouter
|
||||
from lyrics_transcriber.correction.agentic.providers.config import ProviderConfig
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import agentic correction modules: {e}")
|
||||
logger.error("Make sure you're running from the karaoke-gen directory")
|
||||
sys.exit(1)
|
||||
|
||||
# Determine which model to use
|
||||
config = ProviderConfig.from_env()
|
||||
router = ModelRouter(config)
|
||||
model = router.choose_model("unknown", 0.5)
|
||||
logger.info(f"Selected model: {model}")
|
||||
|
||||
# Validate environment for Vertex AI
|
||||
if model.startswith("vertexai/") and not project_id:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT environment variable is required for Vertex AI")
|
||||
logger.error("Set it with: export GOOGLE_CLOUD_PROJECT=your-project-id")
|
||||
logger.error("Or use a different model: AGENTIC_AI_MODEL=openai/gpt-4o")
|
||||
sys.exit(1)
|
||||
|
||||
# Create the agent
|
||||
logger.info(f"\nCreating AgenticCorrector with model: {model}")
|
||||
try:
|
||||
agent = AgenticCorrector.from_model(
|
||||
model=model,
|
||||
session_id="test_session_local",
|
||||
cache_dir=os.path.expanduser("~/lyrics-transcriber-cache")
|
||||
)
|
||||
logger.info("Agent created successfully!")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create agent: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
# Run test cases
|
||||
results = []
|
||||
for test_case in TEST_CASES:
|
||||
result = run_single_test(test_case, agent)
|
||||
results.append(result)
|
||||
|
||||
# Summary
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("TEST SUMMARY")
|
||||
logger.info("=" * 60)
|
||||
|
||||
passed = sum(1 for r in results if r["success"])
|
||||
failed = len(results) - passed
|
||||
|
||||
for r in results:
|
||||
status = "✅ PASS" if r["success"] else "❌ FAIL"
|
||||
logger.info(f"{status}: {r['test_name']}")
|
||||
if not r["success"]:
|
||||
logger.info(f" Error: {r.get('error', 'unknown')}")
|
||||
|
||||
logger.info(f"\nResults: {passed}/{len(results)} passed")
|
||||
|
||||
# Output JSON results
|
||||
output_file = "agentic_test_results.json"
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
logger.info(f"\nDetailed results saved to: {output_file}")
|
||||
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user