mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary
- Add relevance filtering to the correction pipeline — automatically
rejects wrong-song reference lyrics (below 30% word match threshold,
validated against 168 prod source pairs)
- New `POST /api/review/{job_id}/search-lyrics` endpoint lets users
search with alternate artist/title from the review UI, with force-add
override for edge cases
- Redesigned Add Reference Lyrics modal with Search (default) and Paste
tabs, plus inline empty state when no valid sources exist
## Changes
### Backend
- **Relevance filtering** (`correction/relevance.py`): Per-source
scoring based on anchor sequence word coverage. Integrated into
`LyricsCorrector.run()` after anchor finding — sources below threshold
removed before corrections computed
- **Search endpoint** (`routes/review.py`): Queries all providers
(Genius, Spotify, Musixmatch, LRCLIB) with user-supplied artist/title,
auto-filters results, returns added/rejected/not-found breakdown
- **Provider factory** (`operations.py`): `create_lyrics_providers()`
reads env vars, `search_lyrics_sources()` orchestrates
fetch+filter+correct
- **Force override**: `add-lyrics` and `search-lyrics` support `force`
flag to bypass threshold
### Frontend
- **Tabbed modal**: `AddLyricsModal` rewritten with Search (default) and
Paste tabs
- **SearchLyricsTab**: Artist/title inputs, provider search, rejected
source cards with force-add checkboxes
- **ReferenceEmptyState**: Inline search/paste UI when no valid
reference sources exist
- **API client**: New `searchLyrics()` method with
`SearchLyricsResponse` types
### Analysis
- Empirical threshold analysis script
(`scripts/analyze_reference_relevance.py`) + visual review tool
- 168 source pairs from 81 prod jobs: wrong-song max 3.5%, correct-song
min 26.8% → 30% threshold
## Testing
- 35 new backend tests (relevance scoring, filtering, corrector
integration, search operation, API endpoint)
- 717 existing frontend tests passing (no regressions)
- Edge case: all sources filtered out handled gracefully
## Review
- [x] Local CodeRabbit review completed (3 issues fixed: response
contract, force param coercion, null checks)
@coderabbitai ignore
---
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
370 lines
12 KiB
Python
Executable File
370 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Empirical threshold analysis: measure per-source reference relevance scores
|
|
from production jobs that have corrections.json files.
|
|
|
|
Relevance score = (reference words appearing in anchor sequences) / (total reference words)
|
|
|
|
This helps determine appropriate thresholds for filtering low-quality reference
|
|
sources before running anchor sequence detection.
|
|
|
|
Usage:
|
|
python scripts/analyze_reference_relevance.py
|
|
python scripts/analyze_reference_relevance.py --limit 50
|
|
python scripts/analyze_reference_relevance.py --output results.json
|
|
|
|
Requires:
|
|
- google-cloud-firestore
|
|
- google-cloud-storage
|
|
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
|
"""
|
|
import argparse
|
|
import io
|
|
import json
|
|
import logging
|
|
import sys
|
|
from collections import defaultdict
|
|
from typing import Optional
|
|
|
|
from google.cloud import firestore, storage
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
PROJECT = "nomadkaraoke"
|
|
GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Firestore helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def fetch_candidate_jobs(db: firestore.Client, limit: int) -> list[dict]:
|
|
"""
|
|
Pull jobs that have a corrections file URL from Firestore.
|
|
|
|
We over-fetch (up to 3x the limit) so we can bias toward jobs where at
|
|
least one source has a low match score (mixed results). We filter down
|
|
after computing scores.
|
|
"""
|
|
logger.info("Querying Firestore for completed jobs with correction data...")
|
|
|
|
# Query jobs that are complete and have lyrics corrections URL stored
|
|
query = (
|
|
db.collection("jobs")
|
|
.where("status", "==", "complete")
|
|
.order_by("created_at", direction=firestore.Query.DESCENDING)
|
|
.limit(limit * 3)
|
|
)
|
|
|
|
jobs = []
|
|
for doc in query.stream():
|
|
data = doc.to_dict()
|
|
data["_job_id"] = doc.id
|
|
|
|
# Require a corrections URL
|
|
corrections_url = _get_corrections_url(data)
|
|
if corrections_url:
|
|
data["_corrections_url"] = corrections_url
|
|
jobs.append(data)
|
|
|
|
logger.info(f"Found {len(jobs)} jobs with corrections URL")
|
|
return jobs
|
|
|
|
|
|
def _get_corrections_url(job_data: dict) -> Optional[str]:
|
|
"""Extract the GCS path for the corrections.json from a Firestore job doc."""
|
|
file_urls = job_data.get("file_urls", {})
|
|
lyrics = file_urls.get("lyrics", {})
|
|
corrections = lyrics.get("corrections")
|
|
if corrections:
|
|
return corrections
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# GCS helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def download_corrections(gcs_client: storage.Client, gcs_path: str) -> Optional[dict]:
|
|
"""
|
|
Download and parse a corrections.json from GCS.
|
|
|
|
gcs_path may be:
|
|
- A full gs:// URL: gs://karaoke-gen-outputs/jobs/.../lyrics/corrections.json
|
|
- A bare object path: jobs/.../lyrics/corrections.json
|
|
"""
|
|
if gcs_path.startswith("gs://"):
|
|
# Strip gs://<bucket>/
|
|
parts = gcs_path[5:].split("/", 1)
|
|
bucket_name = parts[0]
|
|
blob_path = parts[1] if len(parts) > 1 else ""
|
|
else:
|
|
bucket_name = GCS_BUCKET
|
|
blob_path = gcs_path
|
|
|
|
try:
|
|
bucket = gcs_client.bucket(bucket_name)
|
|
blob = bucket.blob(blob_path)
|
|
content = blob.download_as_bytes()
|
|
return json.loads(content)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to download {gcs_path}: {e}")
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Relevance computation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def compute_relevance_scores(corrections: dict) -> dict[str, dict]:
|
|
"""
|
|
Compute per-source relevance scores from a corrections.json dict.
|
|
|
|
Relevance = (# reference words that appear in at least one anchor) /
|
|
(total words in that source's reference lyrics)
|
|
|
|
Returns a dict keyed by source name, each value being:
|
|
{
|
|
"total_words": int,
|
|
"anchored_words": int,
|
|
"score": float,
|
|
}
|
|
"""
|
|
reference_lyrics: dict = corrections.get("reference_lyrics", {})
|
|
anchor_sequences: list = corrections.get("anchor_sequences", [])
|
|
|
|
# Build set of anchored word IDs per source
|
|
anchored_ids: dict[str, set] = defaultdict(set)
|
|
for anchor in anchor_sequences:
|
|
ref_word_ids: dict = anchor.get("reference_word_ids", {})
|
|
for source, word_ids in ref_word_ids.items():
|
|
anchored_ids[source].update(word_ids)
|
|
|
|
scores = {}
|
|
for source, ref_data in reference_lyrics.items():
|
|
if not ref_data:
|
|
continue
|
|
|
|
# Count total words in this source
|
|
total_words = 0
|
|
for segment in ref_data.get("segments", []):
|
|
total_words += len(segment.get("words", []))
|
|
|
|
if total_words == 0:
|
|
scores[source] = {"total_words": 0, "anchored_words": 0, "score": 0.0}
|
|
continue
|
|
|
|
anchored_count = len(anchored_ids.get(source, set()))
|
|
score = anchored_count / total_words
|
|
|
|
scores[source] = {
|
|
"total_words": total_words,
|
|
"anchored_words": anchored_count,
|
|
"score": round(score, 4),
|
|
}
|
|
|
|
return scores
|
|
|
|
|
|
def has_mixed_results(scores: dict[str, dict], low_threshold: float = 0.5) -> bool:
|
|
"""Return True if at least one source has a score below low_threshold."""
|
|
if len(scores) < 2:
|
|
return False
|
|
score_values = [v["score"] for v in scores.values()]
|
|
return min(score_values) < low_threshold
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Report generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def build_report(job_results: list[dict]) -> str:
|
|
"""Build a human-readable text report sorted by min source score."""
|
|
lines = []
|
|
lines.append("=" * 72)
|
|
lines.append("REFERENCE LYRICS RELEVANCE ANALYSIS")
|
|
lines.append(f"Jobs analyzed: {len(job_results)}")
|
|
lines.append("=" * 72)
|
|
lines.append("")
|
|
|
|
# Sort: mixed-result jobs first (most interesting), then by min score asc
|
|
def sort_key(r):
|
|
scores = [v["score"] for v in r["scores"].values()] if r["scores"] else [1.0]
|
|
return (0 if r["is_mixed"] else 1, min(scores))
|
|
|
|
sorted_results = sorted(job_results, key=sort_key)
|
|
|
|
for idx, result in enumerate(sorted_results, 1):
|
|
job_id = result["job_id"]
|
|
artist = result.get("artist", "Unknown")
|
|
title = result.get("title", "Unknown")
|
|
mixed_flag = " [MIXED]" if result["is_mixed"] else ""
|
|
|
|
lines.append(f"{idx:3d}. {artist} - {title} ({job_id}){mixed_flag}")
|
|
|
|
scores = result["scores"]
|
|
if not scores:
|
|
lines.append(" (no reference sources found)")
|
|
else:
|
|
for source, info in sorted(scores.items(), key=lambda x: x[1]["score"]):
|
|
bar_len = int(info["score"] * 20)
|
|
bar = "#" * bar_len + "-" * (20 - bar_len)
|
|
lines.append(
|
|
f" {source:<12s} [{bar}] {info['score']:.1%}"
|
|
f" ({info['anchored_words']}/{info['total_words']} words)"
|
|
)
|
|
lines.append("")
|
|
|
|
# --- Aggregate statistics ---
|
|
lines.append("=" * 72)
|
|
lines.append("AGGREGATE STATISTICS")
|
|
lines.append("=" * 72)
|
|
|
|
all_source_scores: dict[str, list[float]] = defaultdict(list)
|
|
for result in job_results:
|
|
for source, info in result["scores"].items():
|
|
all_source_scores[source].append(info["score"])
|
|
|
|
for source in sorted(all_source_scores):
|
|
vals = all_source_scores[source]
|
|
avg = sum(vals) / len(vals) if vals else 0
|
|
below_50 = sum(1 for v in vals if v < 0.5)
|
|
below_30 = sum(1 for v in vals if v < 0.3)
|
|
below_10 = sum(1 for v in vals if v < 0.1)
|
|
lines.append(
|
|
f" {source:<12s} n={len(vals):3d} avg={avg:.1%}"
|
|
f" <50%: {below_50:3d} <30%: {below_30:3d} <10%: {below_10:3d}"
|
|
)
|
|
|
|
mixed_count = sum(1 for r in job_results if r["is_mixed"])
|
|
lines.append("")
|
|
lines.append(f" Jobs with mixed results (at least 1 source <50%): {mixed_count}/{len(job_results)}")
|
|
|
|
# Score distribution across ALL sources
|
|
all_scores = [
|
|
info["score"]
|
|
for result in job_results
|
|
for info in result["scores"].values()
|
|
]
|
|
if all_scores:
|
|
thresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
|
|
lines.append("")
|
|
lines.append(" Score distribution (all sources):")
|
|
for t in thresholds:
|
|
count = sum(1 for s in all_scores if s < t)
|
|
pct = count / len(all_scores) * 100
|
|
lines.append(f" < {t:.0%}: {count:4d} / {len(all_scores)} ({pct:.1f}%)")
|
|
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Analyze reference lyrics relevance scores from production jobs"
|
|
)
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=30,
|
|
help="Target number of jobs to analyze (default: 30)",
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=str,
|
|
default=None,
|
|
help="Write raw JSON results to this file (optional)",
|
|
)
|
|
parser.add_argument(
|
|
"--no-mixed-bias",
|
|
action="store_true",
|
|
help="Disable bias toward jobs with mixed results",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
db = firestore.Client(project=PROJECT)
|
|
gcs_client = storage.Client(project=PROJECT)
|
|
|
|
# Step 1: Fetch candidate jobs from Firestore
|
|
candidate_jobs = fetch_candidate_jobs(db, limit=args.limit)
|
|
if not candidate_jobs:
|
|
logger.error("No candidate jobs found. Check Firestore access.")
|
|
sys.exit(1)
|
|
|
|
# Step 2: Download corrections and compute scores
|
|
job_results = []
|
|
skipped = 0
|
|
|
|
for job in candidate_jobs:
|
|
job_id = job["_job_id"]
|
|
corrections_url = job["_corrections_url"]
|
|
artist = job.get("artist", job.get("state_data", {}).get("artist", "Unknown"))
|
|
title = job.get("title", job.get("state_data", {}).get("title", "Unknown"))
|
|
|
|
logger.info(f"Processing {job_id}: {artist} - {title}")
|
|
|
|
corrections = download_corrections(gcs_client, corrections_url)
|
|
if not corrections:
|
|
skipped += 1
|
|
continue
|
|
|
|
scores = compute_relevance_scores(corrections)
|
|
if not scores:
|
|
skipped += 1
|
|
continue
|
|
|
|
is_mixed = has_mixed_results(scores)
|
|
|
|
job_results.append({
|
|
"job_id": job_id,
|
|
"artist": artist,
|
|
"title": title,
|
|
"corrections_url": corrections_url,
|
|
"scores": scores,
|
|
"is_mixed": is_mixed,
|
|
})
|
|
|
|
if skipped:
|
|
logger.info(f"Skipped {skipped} jobs (download failure or no reference sources)")
|
|
|
|
if not job_results:
|
|
logger.error("No results to report.")
|
|
sys.exit(1)
|
|
|
|
# Step 3: Optionally bias toward mixed-result jobs
|
|
if not args.no_mixed_bias:
|
|
mixed = [r for r in job_results if r["is_mixed"]]
|
|
non_mixed = [r for r in job_results if not r["is_mixed"]]
|
|
# Aim for at least half mixed, rest filled with non-mixed
|
|
target_mixed = min(len(mixed), max(args.limit // 2, len(mixed)))
|
|
target_non_mixed = min(len(non_mixed), args.limit - target_mixed)
|
|
job_results = mixed[:target_mixed] + non_mixed[:target_non_mixed]
|
|
logger.info(
|
|
f"After mixed-bias selection: {len(job_results)} jobs "
|
|
f"({target_mixed} mixed, {target_non_mixed} non-mixed)"
|
|
)
|
|
else:
|
|
job_results = job_results[: args.limit]
|
|
|
|
# Step 4: Print report
|
|
report = build_report(job_results)
|
|
print(report)
|
|
|
|
# Step 5: Optionally write JSON
|
|
if args.output:
|
|
with open(args.output, "w") as f:
|
|
json.dump(job_results, f, indent=2)
|
|
logger.info(f"Raw results written to {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|