mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Moved lyrics_transcriber from lyrics_transcriber_temp/ to libs/ - Removed obsolete standalone package files - Updated pyproject.toml paths - Archived python-lyrics-transcriber repo ## Test plan - [x] poetry install succeeds - [x] Backend tests pass (2614) - [x] python-lyrics-transcriber archived @coderabbitai ignore --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
555 lines
20 KiB
Python
555 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Encoding Performance Benchmark Script
|
|
|
|
This script benchmarks the actual encoding operations used in karaoke generation
|
|
by importing and using the real code paths from LyricsTranscriber and karaoke-gen.
|
|
|
|
Usage:
|
|
# Download test files and run benchmark locally
|
|
python scripts/benchmark_encoding.py --download --run
|
|
|
|
# Run benchmark on already-downloaded files
|
|
python scripts/benchmark_encoding.py --run
|
|
|
|
Test data source:
|
|
Job fddad04d (piri - dog) from GCS
|
|
|
|
Benchmark stages (using actual code):
|
|
1. Preview video (480x270 with ASS) - LocalPreviewEncodingService
|
|
2. Full "With Vocals" video (4K with ASS) - LyricsTranscriber VideoGenerator
|
|
3. Remux with instrumental - LocalEncodingService
|
|
4. Lossless 4K concat (title + karaoke + end) - LocalEncodingService
|
|
5. Lossy 4K (AAC audio) - LocalEncodingService
|
|
6. MKV (FLAC audio) - LocalEncodingService
|
|
7. 720p downscale - LocalEncodingService
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
# Add project root to path for imports
|
|
PROJECT_ROOT = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
# Test job ID (piri - dog)
|
|
TEST_JOB_ID = "fddad04d"
|
|
GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
|
|
GCS_JOB_PATH = f"gs://{GCS_BUCKET}/jobs/{TEST_JOB_ID}"
|
|
GCS_THEME_PATH = f"gs://{GCS_BUCKET}/themes/nomad/assets"
|
|
|
|
# Local benchmark directory
|
|
BENCHMARK_DIR = Path("benchmark_data")
|
|
|
|
# Configure logging
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class BenchmarkResult:
|
|
"""Result of a single benchmark operation."""
|
|
name: str
|
|
duration_seconds: float
|
|
success: bool
|
|
output_size_mb: float = 0.0
|
|
error: Optional[str] = None
|
|
|
|
|
|
@dataclass
|
|
class SystemInfo:
|
|
"""System information for the benchmark."""
|
|
platform: str
|
|
platform_version: str
|
|
cpu_model: str
|
|
cpu_count: int
|
|
memory_gb: float
|
|
ffmpeg_version: str
|
|
hostname: str
|
|
|
|
|
|
def get_system_info() -> SystemInfo:
|
|
"""Collect system information."""
|
|
cpu_model = "Unknown"
|
|
if platform.system() == "Darwin":
|
|
try:
|
|
result = subprocess.run(
|
|
["sysctl", "-n", "machdep.cpu.brand_string"],
|
|
capture_output=True, text=True
|
|
)
|
|
cpu_model = result.stdout.strip()
|
|
except Exception:
|
|
pass
|
|
elif platform.system() == "Linux":
|
|
try:
|
|
with open("/proc/cpuinfo") as f:
|
|
for line in f:
|
|
if line.startswith("model name"):
|
|
cpu_model = line.split(":")[1].strip()
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
memory_gb = 0.0
|
|
if platform.system() == "Darwin":
|
|
try:
|
|
result = subprocess.run(
|
|
["sysctl", "-n", "hw.memsize"],
|
|
capture_output=True, text=True
|
|
)
|
|
memory_gb = int(result.stdout.strip()) / (1024**3)
|
|
except Exception:
|
|
pass
|
|
elif platform.system() == "Linux":
|
|
try:
|
|
with open("/proc/meminfo") as f:
|
|
for line in f:
|
|
if line.startswith("MemTotal"):
|
|
mem_kb = int(line.split()[1])
|
|
memory_gb = mem_kb / (1024**2)
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
ffmpeg_version = "Unknown"
|
|
try:
|
|
result = subprocess.run(
|
|
["ffmpeg", "-version"],
|
|
capture_output=True, text=True
|
|
)
|
|
ffmpeg_version = result.stdout.split("\n")[0]
|
|
except Exception:
|
|
pass
|
|
|
|
hostname = platform.node()
|
|
|
|
return SystemInfo(
|
|
platform=platform.system(),
|
|
platform_version=platform.release(),
|
|
cpu_model=cpu_model,
|
|
cpu_count=os.cpu_count() or 0,
|
|
memory_gb=round(memory_gb, 1),
|
|
ffmpeg_version=ffmpeg_version,
|
|
hostname=hostname,
|
|
)
|
|
|
|
|
|
def download_test_files(benchmark_dir: Path) -> bool:
|
|
"""Download test files from GCS."""
|
|
print(f"\n{'='*60}")
|
|
print("Downloading test files from GCS...")
|
|
print(f"{'='*60}")
|
|
|
|
benchmark_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
files_to_download = [
|
|
# Lyrics/subtitles
|
|
(f"{GCS_JOB_PATH}/lyrics/karaoke.ass", benchmark_dir / "karaoke.ass"),
|
|
# Screens
|
|
(f"{GCS_JOB_PATH}/screens/title.mov", benchmark_dir / "title.mov"),
|
|
(f"{GCS_JOB_PATH}/screens/end.mov", benchmark_dir / "end.mov"),
|
|
# Audio
|
|
(f"{GCS_JOB_PATH}/stems/instrumental_clean.flac", benchmark_dir / "instrumental_clean.flac"),
|
|
(f"{GCS_JOB_PATH}/stems/vocals_clean.flac", benchmark_dir / "vocals.flac"),
|
|
# Theme assets
|
|
(f"{GCS_THEME_PATH}/karaoke-background-image-nomad-4k.png", benchmark_dir / "background.png"),
|
|
(f"{GCS_THEME_PATH}/AvenirNext-Bold.ttf", benchmark_dir / "font.ttf"),
|
|
]
|
|
|
|
for gcs_path, local_path in files_to_download:
|
|
if local_path.exists():
|
|
print(f" [cached] {local_path.name}")
|
|
continue
|
|
|
|
print(f" Downloading {local_path.name}...")
|
|
try:
|
|
result = subprocess.run(
|
|
["gsutil", "-q", "cp", gcs_path, str(local_path)],
|
|
capture_output=True, text=True, timeout=300
|
|
)
|
|
if result.returncode != 0:
|
|
print(f" ERROR: {result.stderr}")
|
|
return False
|
|
except subprocess.TimeoutExpired:
|
|
print(f" ERROR: Download timed out")
|
|
return False
|
|
|
|
print(" All files downloaded successfully.")
|
|
return True
|
|
|
|
|
|
def time_operation(func, name: str) -> BenchmarkResult:
|
|
"""Time a function and return a BenchmarkResult."""
|
|
print(f"\n Running: {name}")
|
|
start_time = time.perf_counter()
|
|
|
|
try:
|
|
result = func()
|
|
duration = time.perf_counter() - start_time
|
|
|
|
# Get output size if result is a path
|
|
output_size_mb = 0.0
|
|
if isinstance(result, (str, Path)):
|
|
result_path = Path(result)
|
|
if result_path.exists():
|
|
output_size_mb = result_path.stat().st_size / (1024 * 1024)
|
|
|
|
print(f" Duration: {duration:.2f}s, Output: {output_size_mb:.1f}MB")
|
|
return BenchmarkResult(
|
|
name=name,
|
|
duration_seconds=duration,
|
|
success=True,
|
|
output_size_mb=output_size_mb,
|
|
)
|
|
except Exception as e:
|
|
duration = time.perf_counter() - start_time
|
|
print(f" FAILED: {e}")
|
|
return BenchmarkResult(
|
|
name=name,
|
|
duration_seconds=duration,
|
|
success=False,
|
|
error=str(e),
|
|
)
|
|
|
|
|
|
def run_benchmarks(benchmark_dir: Path, output_dir: Path) -> List[BenchmarkResult]:
|
|
"""Run all encoding benchmarks using actual code paths."""
|
|
from backend.services.local_encoding_service import LocalEncodingService, EncodingConfig
|
|
from backend.services.local_preview_encoding_service import (
|
|
LocalPreviewEncodingService, PreviewEncodingConfig
|
|
)
|
|
from lyrics_transcriber.output.video import VideoGenerator
|
|
|
|
results = []
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
cache_dir = output_dir / "cache"
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Input files
|
|
ass_file = str(benchmark_dir / "karaoke.ass")
|
|
vocals_audio = str(benchmark_dir / "vocals.flac")
|
|
instrumental_audio = str(benchmark_dir / "instrumental_clean.flac")
|
|
title_mov = str(benchmark_dir / "title.mov")
|
|
end_mov = str(benchmark_dir / "end.mov")
|
|
background_image = str(benchmark_dir / "background.png")
|
|
font_file = str(benchmark_dir / "font.ttf")
|
|
|
|
# Verify input files exist
|
|
required_files = [ass_file, vocals_audio, instrumental_audio, title_mov, end_mov, background_image]
|
|
for f in required_files:
|
|
if not Path(f).exists():
|
|
print(f"ERROR: Missing required file: {f}")
|
|
return results
|
|
|
|
print(f"\n{'='*60}")
|
|
print("Running Encoding Benchmarks (using actual code paths)")
|
|
print(f"{'='*60}")
|
|
|
|
# ============================================================
|
|
# Stage 1: Preview video (480x270 with ASS overlay)
|
|
# Uses: LocalPreviewEncodingService
|
|
# ============================================================
|
|
preview_output = str(output_dir / "preview.mp4")
|
|
preview_service = LocalPreviewEncodingService(logger=logger)
|
|
preview_config = PreviewEncodingConfig(
|
|
ass_path=ass_file,
|
|
audio_path=vocals_audio,
|
|
output_path=preview_output,
|
|
background_image_path=background_image,
|
|
background_color="black",
|
|
font_path=font_file,
|
|
)
|
|
|
|
def run_preview():
|
|
result = preview_service.encode_preview(preview_config)
|
|
if not result.success:
|
|
raise RuntimeError(result.error)
|
|
return result.output_path
|
|
|
|
results.append(time_operation(run_preview, "Stage 1: Preview Video (480x270 ASS)"))
|
|
|
|
# ============================================================
|
|
# Stage 2: Full "With Vocals" video (4K with ASS overlay)
|
|
# Uses: LyricsTranscriber VideoGenerator._build_ffmpeg_command
|
|
# This is the HEAVIEST operation - full 4K render with libass
|
|
# ============================================================
|
|
with_vocals_output = str(output_dir / "with_vocals.mkv")
|
|
|
|
# Create VideoGenerator with realistic styles config
|
|
styles = {
|
|
"karaoke": {
|
|
"background_image": background_image,
|
|
"background_color": "black",
|
|
"font_path": font_file,
|
|
}
|
|
}
|
|
video_gen = VideoGenerator(
|
|
output_dir=str(output_dir),
|
|
cache_dir=str(cache_dir),
|
|
video_resolution=(3840, 2160), # 4K
|
|
styles=styles,
|
|
logger=logger,
|
|
)
|
|
|
|
def run_with_vocals():
|
|
# Use the actual generate_video method
|
|
return video_gen.generate_video(ass_file, vocals_audio, "benchmark")
|
|
|
|
results.append(time_operation(run_with_vocals, "Stage 2: With Vocals Video (4K ASS) - HEAVIEST"))
|
|
|
|
# Get the actual output path from VideoGenerator
|
|
with_vocals_actual = str(output_dir / "benchmark (With Vocals).mkv")
|
|
if not Path(with_vocals_actual).exists():
|
|
print(f" ERROR: With Vocals video not generated at {with_vocals_actual}")
|
|
# Try to find it
|
|
for f in output_dir.glob("*.mkv"):
|
|
print(f" Found: {f}")
|
|
with_vocals_actual = str(f)
|
|
break
|
|
|
|
# ============================================================
|
|
# Stage 3: Remux with instrumental audio
|
|
# Uses: LocalEncodingService.remux_with_instrumental
|
|
# ============================================================
|
|
karaoke_mp4 = str(output_dir / "karaoke.mp4")
|
|
encoding_service = LocalEncodingService(logger=logger)
|
|
|
|
def run_remux():
|
|
success = encoding_service.remux_with_instrumental(
|
|
with_vocals_actual,
|
|
instrumental_audio,
|
|
karaoke_mp4,
|
|
)
|
|
if not success:
|
|
raise RuntimeError("Remux failed")
|
|
return karaoke_mp4
|
|
|
|
results.append(time_operation(run_remux, "Stage 3: Remux with Instrumental"))
|
|
|
|
# ============================================================
|
|
# Stage 4: Convert title/end MOV to MP4 (needed for concat)
|
|
# Uses: LocalEncodingService.convert_mov_to_mp4
|
|
# ============================================================
|
|
title_mp4 = str(output_dir / "title.mp4")
|
|
end_mp4 = str(output_dir / "end.mp4")
|
|
|
|
def run_title_convert():
|
|
success = encoding_service.convert_mov_to_mp4(title_mov, title_mp4)
|
|
if not success:
|
|
raise RuntimeError("Title convert failed")
|
|
return title_mp4
|
|
|
|
def run_end_convert():
|
|
success = encoding_service.convert_mov_to_mp4(end_mov, end_mp4)
|
|
if not success:
|
|
raise RuntimeError("End convert failed")
|
|
return end_mp4
|
|
|
|
results.append(time_operation(run_title_convert, "Stage 4a: Convert Title MOV to MP4"))
|
|
results.append(time_operation(run_end_convert, "Stage 4b: Convert End MOV to MP4"))
|
|
|
|
# ============================================================
|
|
# Stage 5: Lossless 4K concatenation (title + karaoke + end)
|
|
# Uses: LocalEncodingService.encode_lossless_mp4
|
|
# ============================================================
|
|
lossless_4k = str(output_dir / "final_lossless_4k.mp4")
|
|
|
|
def run_lossless_concat():
|
|
success = encoding_service.encode_lossless_mp4(
|
|
title_mp4,
|
|
karaoke_mp4,
|
|
lossless_4k,
|
|
end_mp4,
|
|
)
|
|
if not success:
|
|
raise RuntimeError("Lossless concat failed")
|
|
return lossless_4k
|
|
|
|
results.append(time_operation(run_lossless_concat, "Stage 5: Lossless 4K Concat (title+karaoke+end)"))
|
|
|
|
# ============================================================
|
|
# Stage 6: Lossy 4K with AAC audio
|
|
# Uses: LocalEncodingService.encode_lossy_mp4
|
|
# ============================================================
|
|
lossy_4k = str(output_dir / "final_lossy_4k.mp4")
|
|
|
|
def run_lossy():
|
|
success = encoding_service.encode_lossy_mp4(lossless_4k, lossy_4k)
|
|
if not success:
|
|
raise RuntimeError("Lossy 4K failed")
|
|
return lossy_4k
|
|
|
|
if Path(lossless_4k).exists():
|
|
results.append(time_operation(run_lossy, "Stage 6: Lossy 4K (AAC audio)"))
|
|
|
|
# ============================================================
|
|
# Stage 7: MKV with FLAC audio
|
|
# Uses: LocalEncodingService.encode_lossless_mkv
|
|
# ============================================================
|
|
lossless_mkv = str(output_dir / "final_lossless_4k.mkv")
|
|
|
|
def run_mkv():
|
|
success = encoding_service.encode_lossless_mkv(lossless_4k, lossless_mkv)
|
|
if not success:
|
|
raise RuntimeError("MKV failed")
|
|
return lossless_mkv
|
|
|
|
if Path(lossless_4k).exists():
|
|
results.append(time_operation(run_mkv, "Stage 7: MKV (FLAC audio)"))
|
|
|
|
# ============================================================
|
|
# Stage 8: 720p downscale
|
|
# Uses: LocalEncodingService.encode_720p
|
|
# ============================================================
|
|
lossy_720p = str(output_dir / "final_lossy_720p.mp4")
|
|
|
|
def run_720p():
|
|
success = encoding_service.encode_720p(lossless_4k, lossy_720p)
|
|
if not success:
|
|
raise RuntimeError("720p failed")
|
|
return lossy_720p
|
|
|
|
if Path(lossless_4k).exists():
|
|
results.append(time_operation(run_720p, "Stage 8: 720p Downscale"))
|
|
|
|
return results
|
|
|
|
|
|
def print_results(system_info: SystemInfo, results: List[BenchmarkResult]):
|
|
"""Print benchmark results."""
|
|
print(f"\n{'='*60}")
|
|
print("BENCHMARK RESULTS")
|
|
print(f"{'='*60}")
|
|
|
|
print(f"\nSystem Information:")
|
|
print(f" Hostname: {system_info.hostname}")
|
|
print(f" Platform: {system_info.platform} {system_info.platform_version}")
|
|
print(f" CPU: {system_info.cpu_model}")
|
|
print(f" CPU Count: {system_info.cpu_count}")
|
|
print(f" Memory: {system_info.memory_gb} GB")
|
|
print(f" FFmpeg: {system_info.ffmpeg_version}")
|
|
|
|
print(f"\nResults:")
|
|
print(f"{'Operation':<55} {'Duration':>10} {'Size':>10} {'Status':>10}")
|
|
print("-" * 85)
|
|
|
|
total_time = 0.0
|
|
for r in results:
|
|
status = "OK" if r.success else "FAILED"
|
|
duration_str = f"{r.duration_seconds:.2f}s"
|
|
size_str = f"{r.output_size_mb:.1f}MB" if r.output_size_mb > 0 else "-"
|
|
print(f"{r.name:<55} {duration_str:>10} {size_str:>10} {status:>10}")
|
|
if r.success:
|
|
total_time += r.duration_seconds
|
|
if not r.success and r.error:
|
|
print(f" Error: {r.error[:80]}")
|
|
|
|
print("-" * 85)
|
|
print(f"{'TOTAL (successful operations)':<55} {total_time:.2f}s")
|
|
|
|
# Calculate key metrics
|
|
with_vocals = next((r for r in results if "With Vocals" in r.name), None)
|
|
lossless_concat = next((r for r in results if "Lossless 4K Concat" in r.name), None)
|
|
downscale_720p = next((r for r in results if "720p Downscale" in r.name), None)
|
|
|
|
print(f"\nKey Metrics:")
|
|
if with_vocals and with_vocals.success:
|
|
print(f" With Vocals (4K ASS render): {with_vocals.duration_seconds:.2f}s <- HEAVIEST OPERATION")
|
|
if lossless_concat and lossless_concat.success:
|
|
print(f" Lossless 4K concatenation: {lossless_concat.duration_seconds:.2f}s")
|
|
if downscale_720p and downscale_720p.success:
|
|
print(f" 720p downscale: {downscale_720p.duration_seconds:.2f}s")
|
|
|
|
|
|
def save_results(system_info: SystemInfo, results: List[BenchmarkResult], output_path: Path):
|
|
"""Save results to JSON file."""
|
|
data = {
|
|
"system": {
|
|
"hostname": system_info.hostname,
|
|
"platform": system_info.platform,
|
|
"platform_version": system_info.platform_version,
|
|
"cpu_model": system_info.cpu_model,
|
|
"cpu_count": system_info.cpu_count,
|
|
"memory_gb": system_info.memory_gb,
|
|
"ffmpeg_version": system_info.ffmpeg_version,
|
|
},
|
|
"results": [
|
|
{
|
|
"name": r.name,
|
|
"duration_seconds": r.duration_seconds,
|
|
"success": r.success,
|
|
"output_size_mb": r.output_size_mb,
|
|
"error": r.error,
|
|
}
|
|
for r in results
|
|
],
|
|
"total_time": sum(r.duration_seconds for r in results if r.success),
|
|
}
|
|
|
|
with open(output_path, "w") as f:
|
|
json.dump(data, f, indent=2)
|
|
print(f"\nResults saved to: {output_path}")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Encoding Performance Benchmark")
|
|
parser.add_argument("--download", action="store_true", help="Download test files from GCS")
|
|
parser.add_argument("--run", action="store_true", help="Run benchmarks locally")
|
|
parser.add_argument("--output", type=str, default="benchmark_results.json", help="Output JSON file")
|
|
parser.add_argument("--data-dir", type=str, default="benchmark_data", help="Directory for test data")
|
|
parser.add_argument("--output-dir", type=str, default=None, help="Directory for output files (default: temp)")
|
|
args = parser.parse_args()
|
|
|
|
benchmark_dir = Path(args.data_dir)
|
|
|
|
if not any([args.download, args.run]):
|
|
parser.print_help()
|
|
print("\nExample usage:")
|
|
print(" python scripts/benchmark_encoding.py --download --run")
|
|
sys.exit(1)
|
|
|
|
if args.download:
|
|
if not download_test_files(benchmark_dir):
|
|
print("ERROR: Failed to download test files")
|
|
sys.exit(1)
|
|
|
|
if args.run:
|
|
# Check for required files
|
|
required = ["karaoke.ass", "title.mov", "end.mov", "instrumental_clean.flac", "vocals.flac", "background.png"]
|
|
missing = [f for f in required if not (benchmark_dir / f).exists()]
|
|
if missing:
|
|
print(f"ERROR: Missing files: {missing}")
|
|
print("Run with --download first")
|
|
sys.exit(1)
|
|
|
|
system_info = get_system_info()
|
|
|
|
# Use temp directory or specified output directory
|
|
if args.output_dir:
|
|
output_dir = Path(args.output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
results = run_benchmarks(benchmark_dir, output_dir)
|
|
print_results(system_info, results)
|
|
save_results(system_info, results, Path(args.output))
|
|
else:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
output_dir = Path(tmpdir)
|
|
results = run_benchmarks(benchmark_dir, output_dir)
|
|
print_results(system_info, results)
|
|
save_results(system_info, results, Path(args.output))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|