Files
karaoke-gen/scripts/compare_local_vs_remote.py
T
Andrew Beveridge ☄️andClaude Opus 4.5 30f6fc6c05 perf: Improve cloud karaoke generation performance (#160)
## Summary

Infrastructure changes to address 3x performance gap between local and
cloud karaoke generation:

- **Keep Cloud Run warm** with `min-instances: 4` (eliminates 77s cold
starts)
- **Upgrade FFmpeg** from Debian 5.1.8 to static 7.0.2 (~20-40% faster
encoding)
- **Add diagnostics** to `/api/health/detailed` endpoint (FFmpeg
version, CPU info)
- **Add performance scripts** for benchmarking and comparison

## Changes

- `.github/workflows/ci.yml`: Changed `--min-instances` from 0 to 4
- `backend/Dockerfile.base`: Replace `apt-get install ffmpeg` with John
Van Sickle's static build
- `backend/api/routes/health.py`: Add `get_system_info()` and
`get_ffmpeg_info()` to detailed health endpoint
- `docs/DEVELOPMENT.md`: Add new scripts to Key Scripts table
- `docs/LESSONS-LEARNED.md`: Document Cloud Run cold start and FFmpeg
optimization findings
- `scripts/benchmark_ffmpeg.py`: Isolated FFmpeg encoding benchmark
- `scripts/compare_local_vs_remote.py`: Compare local vs cloud
karaoke-gen performance

## Expected Performance Improvement

| Metric | Before | After |
|--------|--------|-------|
| Cold start | 77s | 0s (warm instances) |
| Video encoding | 174s | ~120-140s (FFmpeg 7.x) |

## Testing

- [ ] Tests pass locally
- [x] FFmpeg static build verified in Docker container
- [ ] Manual testing after deploy

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-03 00:00:44 -05:00

361 lines
12 KiB
Python
Executable File

#!/usr/bin/env python3
"""
Performance comparison script: Local vs Remote karaoke generation.
Runs the same audio file through both local and remote pipelines in
non-interactive mode, capturing detailed timing logs for analysis.
Usage:
python scripts/compare_local_vs_remote.py <audio_file> [--local-only] [--remote-only]
# Run both tests
python scripts/compare_local_vs_remote.py input/minutemen/Search.flac
# Run only local test
python scripts/compare_local_vs_remote.py input/minutemen/Search.flac --local-only
# Run only remote test (useful if local already done)
python scripts/compare_local_vs_remote.py input/minutemen/Search.flac --remote-only
Output:
- output/perf-test-{timestamp}/local/ - Local CLI output + logs
- output/perf-test-{timestamp}/remote/ - Remote CLI output + logs
- output/perf-test-{timestamp}/comparison.txt - Side-by-side timing analysis
"""
import argparse
import os
import subprocess
import sys
import time
import json
import re
from datetime import datetime
from pathlib import Path
def run_command(cmd: list[str], log_file: Path, cwd: Path) -> tuple[int, float]:
"""
Run a command and capture output to a log file.
Returns:
Tuple of (return_code, elapsed_seconds)
"""
print(f"\n{'='*60}")
print(f"Running: {' '.join(cmd)}")
print(f"Logging to: {log_file}")
print(f"{'='*60}\n")
start_time = time.time()
with open(log_file, 'w') as f:
# Write command header
f.write(f"# Command: {' '.join(cmd)}\n")
f.write(f"# Started: {datetime.now().isoformat()}\n")
f.write(f"# CWD: {cwd}\n")
f.write("#" + "=" * 79 + "\n\n")
f.flush()
# Run with real-time output to both console and file
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=cwd,
text=True,
bufsize=1,
)
for line in process.stdout:
# Add timestamp to each line for log analysis
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
timestamped_line = f"{timestamp} - {line}"
# Write to file
f.write(timestamped_line if not line.startswith("20") else line)
f.flush()
# Echo to console (without extra timestamp)
print(line, end='')
process.wait()
elapsed = time.time() - start_time
# Append timing summary to log
with open(log_file, 'a') as f:
f.write(f"\n# Completed: {datetime.now().isoformat()}\n")
f.write(f"# Return code: {process.returncode}\n")
f.write(f"# Total elapsed: {elapsed:.2f} seconds ({elapsed/60:.1f} minutes)\n")
return process.returncode, elapsed
def extract_job_id_from_log(log_file: Path) -> str | None:
"""Extract the job ID from remote CLI log output."""
content = log_file.read_text()
# Look for job ID in various formats
patterns = [
r'Job ID: ([a-f0-9-]+)',
r'job_id["\']?\s*[:=]\s*["\']?([a-f0-9-]+)',
r'Created job: ([a-f0-9-]+)',
r'/jobs/([a-f0-9-]+)',
]
for pattern in patterns:
match = re.search(pattern, content, re.IGNORECASE)
if match:
return match.group(1)
return None
def run_local_test(audio_file: Path, output_dir: Path, artist: str, title: str) -> tuple[int, float, Path]:
"""
Run local karaoke-gen CLI test.
Returns:
Tuple of (return_code, elapsed_seconds, log_file_path)
"""
print("\n" + "=" * 70)
print("RUNNING LOCAL TEST (karaoke-gen)")
print("=" * 70)
log_file = output_dir / "local_cli.log"
cmd = [
"poetry", "run", "karaoke-gen",
str(audio_file.absolute()),
artist,
title,
"--output_dir", str(output_dir.absolute()),
"--log_level", "DEBUG",
"--skip_transcription_review", # Non-interactive: skip lyrics review
"--skip_instrumental_review", # Non-interactive: skip instrumental selection
"-y", # Auto-confirm prompts
]
# Get project root for cwd
project_root = Path(__file__).parent.parent
return_code, elapsed = run_command(cmd, log_file, project_root)
return return_code, elapsed, log_file
def run_remote_test(audio_file: Path, output_dir: Path, artist: str, title: str) -> tuple[int, float, Path, str | None]:
"""
Run remote karaoke-gen-remote CLI test.
Returns:
Tuple of (return_code, elapsed_seconds, log_file_path, job_id)
"""
print("\n" + "=" * 70)
print("RUNNING REMOTE TEST (karaoke-gen-remote)")
print("=" * 70)
log_file = output_dir / "remote_cli.log"
cmd = [
"poetry", "run", "karaoke-gen-remote",
str(audio_file.absolute()),
artist,
title,
"--output_dir", str(output_dir.absolute()),
"--log_level", "DEBUG",
"--skip_transcription_review", # Non-interactive: skip lyrics review
"--skip_instrumental_review", # Non-interactive: skip instrumental selection
"-y", # Auto-confirm prompts
]
# Get project root for cwd
project_root = Path(__file__).parent.parent
return_code, elapsed = run_command(cmd, log_file, project_root)
# Try to extract job ID for fetching cloud logs
job_id = extract_job_id_from_log(log_file)
return return_code, elapsed, log_file, job_id
def analyze_logs(local_log: Path | None, remote_log: Path | None, output_dir: Path) -> None:
"""Run timing analysis on the logs and generate comparison."""
project_root = Path(__file__).parent.parent
analyzer = project_root / "scripts" / "analyze_log_timing.py"
results = {}
if local_log and local_log.exists():
print("\n" + "=" * 70)
print("ANALYZING LOCAL LOG")
print("=" * 70)
# Run analyzer with JSON output
result = subprocess.run(
["python", str(analyzer), str(local_log), "--json"],
capture_output=True,
text=True,
)
if result.returncode == 0:
try:
results['local'] = json.loads(result.stdout)
except json.JSONDecodeError:
print(f"Warning: Could not parse local log analysis")
# Also print human-readable output
subprocess.run(
["python", str(analyzer), str(local_log)],
cwd=project_root,
)
if remote_log and remote_log.exists():
print("\n" + "=" * 70)
print("ANALYZING REMOTE LOG")
print("=" * 70)
# Run analyzer with JSON output
result = subprocess.run(
["python", str(analyzer), str(remote_log), "--json"],
capture_output=True,
text=True,
)
if result.returncode == 0:
try:
results['remote'] = json.loads(result.stdout)
except json.JSONDecodeError:
print(f"Warning: Could not parse remote log analysis")
# Also print human-readable output
subprocess.run(
["python", str(analyzer), str(remote_log)],
cwd=project_root,
)
# Generate comparison summary
if results:
comparison_file = output_dir / "comparison.json"
comparison_file.write_text(json.dumps(results, indent=2))
print(f"\nComparison data saved to: {comparison_file}")
# Print comparison summary
if 'local' in results and 'remote' in results:
print("\n" + "=" * 70)
print("PERFORMANCE COMPARISON SUMMARY")
print("=" * 70)
local_total = results['local'].get('total_wall_clock_seconds', 0)
remote_total = results['remote'].get('total_wall_clock_seconds', 0)
local_auto = results['local'].get('automated_processing_seconds', 0)
remote_auto = results['remote'].get('automated_processing_seconds', 0)
print(f"\n{'Metric':<35} {'Local':>12} {'Remote':>12} {'Diff':>12}")
print("-" * 73)
if local_total and remote_total:
diff = remote_total - local_total
print(f"{'Total wall-clock time':35} {local_total:>10.1f}s {remote_total:>10.1f}s {diff:>+10.1f}s")
if local_auto and remote_auto:
diff = remote_auto - local_auto
print(f"{'Automated processing':35} {local_auto:>10.1f}s {remote_auto:>10.1f}s {diff:>+10.1f}s")
# Compare category totals
print("\nCategory breakdown:")
local_cats = results['local'].get('category_totals', {})
remote_cats = results['remote'].get('category_totals', {})
all_cats = set(local_cats.keys()) | set(remote_cats.keys())
for cat in sorted(all_cats):
local_time = local_cats.get(cat, 0)
remote_time = remote_cats.get(cat, 0)
if local_time > 0 or remote_time > 0:
diff = remote_time - local_time
cat_label = cat.replace('_', ' ').title()
print(f" {cat_label:<33} {local_time:>10.1f}s {remote_time:>10.1f}s {diff:>+10.1f}s")
def main():
parser = argparse.ArgumentParser(
description="Compare local vs remote karaoke generation performance"
)
parser.add_argument("audio_file", type=Path, help="Path to audio file to process")
parser.add_argument("--artist", default="Minutemen", help="Artist name (default: Minutemen)")
parser.add_argument("--title", default="Search", help="Track title (default: Search)")
parser.add_argument("--local-only", action="store_true", help="Only run local test")
parser.add_argument("--remote-only", action="store_true", help="Only run remote test")
parser.add_argument("--output-dir", type=Path, help="Custom output directory")
args = parser.parse_args()
# Validate audio file
if not args.audio_file.exists():
print(f"Error: Audio file not found: {args.audio_file}")
sys.exit(1)
# Create timestamped output directory
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
if args.output_dir:
output_base = args.output_dir
else:
output_base = Path(__file__).parent.parent / "output" / f"perf-test-{timestamp}"
output_base.mkdir(parents=True, exist_ok=True)
local_output = output_base / "local"
remote_output = output_base / "remote"
print(f"\n{'='*70}")
print(f"KARAOKE GENERATION PERFORMANCE TEST")
print(f"{'='*70}")
print(f"Audio file: {args.audio_file}")
print(f"Artist: {args.artist}")
print(f"Title: {args.title}")
print(f"Output directory: {output_base}")
print(f"{'='*70}")
local_log = None
remote_log = None
# Run local test
if not args.remote_only:
local_output.mkdir(parents=True, exist_ok=True)
return_code, elapsed, local_log = run_local_test(
args.audio_file, local_output, args.artist, args.title
)
print(f"\nLocal test completed in {elapsed:.1f}s ({elapsed/60:.1f} minutes)")
print(f"Return code: {return_code}")
# Run remote test
if not args.local_only:
remote_output.mkdir(parents=True, exist_ok=True)
return_code, elapsed, remote_log, job_id = run_remote_test(
args.audio_file, remote_output, args.artist, args.title
)
print(f"\nRemote test completed in {elapsed:.1f}s ({elapsed/60:.1f} minutes)")
print(f"Return code: {return_code}")
if job_id:
print(f"Job ID: {job_id}")
print(f"\nTo fetch detailed cloud logs:")
print(f" python scripts/fetch_job_logs.py {job_id}")
# Analyze logs
analyze_logs(local_log, remote_log, output_base)
print(f"\n{'='*70}")
print(f"TEST COMPLETE")
print(f"{'='*70}")
print(f"Output directory: {output_base}")
if local_log:
print(f"Local log: {local_log}")
if remote_log:
print(f"Remote log: {remote_log}")
if __name__ == "__main__":
main()