mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Upgrades the GCE encoding worker from c4-standard-8 (Intel Xeon) to c4d-highcpu-32 (AMD EPYC 9B45 Turin), achieving **4.92x faster encoding performance** based on rigorous benchmarking. ## Changes - **Infrastructure**: Updated `MachineTypes.ENCODING_WORKER` to `c4d-highcpu-32` in `config.py` - **Documentation**: Comprehensive benchmark results in `docs/archive/2026-01-08-performance-investigation.md` - **Lessons Learned**: Added "GCE Instance Type Selection for FFmpeg Encoding" section - **Benchmark Scripts**: Added reusable scripts for future infrastructure testing: - `scripts/benchmark_encoding.py` - Local Python benchmark using actual code paths - `scripts/benchmark_encoding_gce.sh` - Shell benchmark for GCE VMs - `scripts/benchmark_candidates.sh` - Multi-VM orchestration for comparative testing ## Benchmark Results | Instance | CPU | Total Time | vs Baseline | |----------|-----|------------|-------------| | c4-standard-8 (baseline) | Intel Xeon | 666s | 1.00x | | c4-highcpu-16 | Intel Xeon | 309s | 2.16x | | c4a-highcpu-16 | Google Axion ARM | 248s | 2.69x | | c4d-highcpu-16 | AMD EPYC 9B45 | 220s | 3.03x | | **c4d-highcpu-32** | **AMD EPYC 9B45** | **135s** | **4.92x** | Key finding: AMD EPYC Turin significantly outperforms Intel Granite Rapids for CPU-bound FFmpeg/libass workloads. ## Testing - [x] Benchmarked on actual GCE instances with production data - [x] Verified c4d-highcpu-32 uses hyperdisk-balanced (already configured) - [ ] Infrastructure deploys via CI after merge ## Deployment Impact ⚠️ Changing machine type will cause Pulumi to **replace** the VM (~1-2 min downtime). Jobs queued during restart will retry automatically. @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
246 lines
8.2 KiB
Bash
Executable File
246 lines
8.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# Benchmark Candidate GCE Instance Types
|
|
#
|
|
# This script creates temporary VMs with different instance types,
|
|
# runs the encoding benchmark on each, and collects results.
|
|
#
|
|
# Usage:
|
|
# ./scripts/benchmark_candidates.sh [candidate_number]
|
|
#
|
|
# # Run specific candidate (1-5)
|
|
# ./scripts/benchmark_candidates.sh 1
|
|
#
|
|
# # Run all candidates
|
|
# ./scripts/benchmark_candidates.sh all
|
|
|
|
set -e
|
|
|
|
PROJECT="nomadkaraoke"
|
|
ZONE="us-central1-a"
|
|
BENCHMARK_SCRIPT="benchmark_encoding_gce.sh"
|
|
RESULTS_DIR="benchmark_results"
|
|
|
|
# Candidate configurations
|
|
# Format: "name:machine_type"
|
|
CANDIDATES=(
|
|
"c4d-highcpu-16:c4d-highcpu-16"
|
|
"c4-highcpu-16:c4-highcpu-16"
|
|
"c4-highcpu-32:c4-highcpu-32"
|
|
"c3d-highcpu-30:c3d-highcpu-30"
|
|
"c4a-highcpu-16:c4a-highcpu-16"
|
|
"c4d-highcpu-32:c4d-highcpu-32"
|
|
)
|
|
|
|
# Create results directory
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
# Function to create a benchmark VM
|
|
create_vm() {
|
|
local name=$1
|
|
local machine_type=$2
|
|
local vm_name="benchmark-${name}"
|
|
|
|
echo "Creating VM: $vm_name ($machine_type)..."
|
|
|
|
# Determine disk type based on machine type
|
|
# C4, C4D, C4A all require hyperdisk-balanced
|
|
local disk_type="pd-balanced"
|
|
if [[ "$machine_type" == c4* ]]; then
|
|
disk_type="hyperdisk-balanced"
|
|
fi
|
|
|
|
# Check if ARM (c4a) - needs different image
|
|
if [[ "$machine_type" == c4a* ]]; then
|
|
# ARM-based instance needs ARM image
|
|
gcloud compute instances create "$vm_name" \
|
|
--project="$PROJECT" \
|
|
--zone="$ZONE" \
|
|
--machine-type="$machine_type" \
|
|
--image-family="debian-12-arm64" \
|
|
--image-project="debian-cloud" \
|
|
--boot-disk-size="100GB" \
|
|
--boot-disk-type="$disk_type" \
|
|
--scopes="storage-ro" \
|
|
--metadata="startup-script=apt-get update && apt-get install -y python3 wget && wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz -O /tmp/ffmpeg.tar.xz && tar -xf /tmp/ffmpeg.tar.xz -C /tmp && cp /tmp/ffmpeg-*-arm64-static/ffmpeg /usr/local/bin/ && cp /tmp/ffmpeg-*-arm64-static/ffprobe /usr/local/bin/"
|
|
else
|
|
# x86 instance
|
|
gcloud compute instances create "$vm_name" \
|
|
--project="$PROJECT" \
|
|
--zone="$ZONE" \
|
|
--machine-type="$machine_type" \
|
|
--image-family="debian-12" \
|
|
--image-project="debian-cloud" \
|
|
--boot-disk-size="100GB" \
|
|
--boot-disk-type="$disk_type" \
|
|
--scopes="storage-ro" \
|
|
--metadata="startup-script=apt-get update && apt-get install -y python3 wget && wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -O /tmp/ffmpeg.tar.xz && tar -xf /tmp/ffmpeg.tar.xz -C /tmp && cp /tmp/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ && cp /tmp/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/"
|
|
fi
|
|
|
|
echo "Waiting for VM to be ready..."
|
|
sleep 30
|
|
|
|
# Wait for FFmpeg to be installed
|
|
local max_attempts=20
|
|
local attempt=0
|
|
while ! gcloud compute ssh "$vm_name" --zone="$ZONE" --project="$PROJECT" --command="which ffmpeg" &>/dev/null; do
|
|
attempt=$((attempt + 1))
|
|
if [ $attempt -ge $max_attempts ]; then
|
|
echo "ERROR: FFmpeg not ready after $max_attempts attempts"
|
|
return 1
|
|
fi
|
|
echo " Waiting for FFmpeg installation... (attempt $attempt/$max_attempts)"
|
|
sleep 15
|
|
done
|
|
|
|
echo "VM ready: $vm_name"
|
|
}
|
|
|
|
# Function to run benchmark on a VM
|
|
run_benchmark() {
|
|
local name=$1
|
|
local vm_name="benchmark-${name}"
|
|
local result_file="$RESULTS_DIR/${name}_results.txt"
|
|
|
|
echo ""
|
|
echo "============================================================"
|
|
echo "Running benchmark on: $vm_name"
|
|
echo "============================================================"
|
|
|
|
# Copy benchmark script to VM
|
|
gcloud compute scp "scripts/$BENCHMARK_SCRIPT" "${vm_name}:/tmp/" \
|
|
--zone="$ZONE" --project="$PROJECT"
|
|
|
|
# Run benchmark and capture output
|
|
gcloud compute ssh "$vm_name" --zone="$ZONE" --project="$PROJECT" \
|
|
--command="bash /tmp/$BENCHMARK_SCRIPT" | tee "$result_file"
|
|
|
|
echo ""
|
|
echo "Results saved to: $result_file"
|
|
}
|
|
|
|
# Function to delete a VM
|
|
delete_vm() {
|
|
local name=$1
|
|
local vm_name="benchmark-${name}"
|
|
|
|
echo "Deleting VM: $vm_name..."
|
|
gcloud compute instances delete "$vm_name" \
|
|
--zone="$ZONE" --project="$PROJECT" --quiet || true
|
|
}
|
|
|
|
# Function to run a single candidate
|
|
run_candidate() {
|
|
local index=$1
|
|
local candidate="${CANDIDATES[$index]}"
|
|
local name="${candidate%%:*}"
|
|
local machine_type="${candidate##*:}"
|
|
|
|
echo ""
|
|
echo "########################################################"
|
|
echo "# Candidate $((index + 1)): $name ($machine_type)"
|
|
echo "########################################################"
|
|
|
|
# Create VM
|
|
if ! create_vm "$name" "$machine_type"; then
|
|
echo "ERROR: Failed to create VM for $name"
|
|
return 1
|
|
fi
|
|
|
|
# Run benchmark
|
|
if ! run_benchmark "$name"; then
|
|
echo "ERROR: Benchmark failed for $name"
|
|
delete_vm "$name"
|
|
return 1
|
|
fi
|
|
|
|
# Delete VM
|
|
delete_vm "$name"
|
|
|
|
echo ""
|
|
echo "Candidate $name completed successfully!"
|
|
}
|
|
|
|
# Function to parse results and create summary
|
|
create_summary() {
|
|
echo ""
|
|
echo "============================================================"
|
|
echo "BENCHMARK SUMMARY"
|
|
echo "============================================================"
|
|
echo ""
|
|
echo "Instance Type | Total Time | vs Baseline | With Vocals | Concat | 720p"
|
|
echo "------------------------|------------|-------------|-------------|--------|------"
|
|
|
|
# Baseline (from existing encoding-worker)
|
|
echo "c4-standard-8 (baseline)| 666.19s | 1.00x | 324.16s | 164.86s| 50.20s"
|
|
|
|
# Parse each result file
|
|
for candidate in "${CANDIDATES[@]}"; do
|
|
local name="${candidate%%:*}"
|
|
local result_file="$RESULTS_DIR/${name}_results.txt"
|
|
|
|
if [ -f "$result_file" ]; then
|
|
# Extract total time
|
|
local total=$(grep "TOTAL" "$result_file" | awk '{print $NF}' | tr -d 's')
|
|
# Extract individual stage times
|
|
local with_vocals=$(grep "Stage 2:" "$result_file" | awk -F'|' '{print $2}' | tr -d 's ' 2>/dev/null || echo "-")
|
|
local concat=$(grep "Stage 5:" "$result_file" | awk -F'|' '{print $2}' | tr -d 's ' 2>/dev/null || echo "-")
|
|
local p720=$(grep "Stage 8:" "$result_file" | awk -F'|' '{print $2}' | tr -d 's ' 2>/dev/null || echo "-")
|
|
|
|
if [ -n "$total" ]; then
|
|
local ratio=$(python3 -c "print(f'{666.19 / float(\"$total\"):.2f}x')" 2>/dev/null || echo "-")
|
|
printf "%-24s|%11ss |%12s |%12ss |%7ss|%6ss\n" "$name" "$total" "$ratio" "$with_vocals" "$concat" "$p720"
|
|
else
|
|
printf "%-24s| - | - | - | - | -\n" "$name"
|
|
fi
|
|
else
|
|
printf "%-24s| - | - | - | - | - (no results)\n" "$name"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "Results saved to: $RESULTS_DIR/"
|
|
}
|
|
|
|
# Main
|
|
main() {
|
|
local target="${1:-help}"
|
|
|
|
case "$target" in
|
|
1|2|3|4|5|6)
|
|
run_candidate $((target - 1))
|
|
;;
|
|
all)
|
|
echo "Running all 5 candidates..."
|
|
for i in {0..4}; do
|
|
run_candidate $i
|
|
done
|
|
create_summary
|
|
;;
|
|
summary)
|
|
create_summary
|
|
;;
|
|
help|*)
|
|
echo "Usage: $0 [candidate_number|all|summary]"
|
|
echo ""
|
|
echo "Candidates:"
|
|
for i in "${!CANDIDATES[@]}"; do
|
|
local candidate="${CANDIDATES[$i]}"
|
|
local name="${candidate%%:*}"
|
|
local machine_type="${candidate##*:}"
|
|
echo " $((i + 1)): $name ($machine_type)"
|
|
done
|
|
echo ""
|
|
echo "Commands:"
|
|
echo " 1-5 Run specific candidate"
|
|
echo " all Run all candidates sequentially"
|
|
echo " summary Show results summary"
|
|
echo ""
|
|
echo "Example:"
|
|
echo " $0 1 # Run c4d-highcpu-16"
|
|
echo " $0 all # Run all candidates"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|