mirror of
https://github.com/RevLayDust/RLD-UVR-Opt.git
synced 2026-11-20 13:07:58 +09:00
- Add official GNU Affero General Public License v3.0 to uvr_cli/LICENSE - Add SPDX-License-Identifier: AGPL-3.0-or-later and Copyright (C) 2026 RevLayDust headers to all uvr_cli modules - Document multi-license architecture in root README.md and uvr_cli/README.md - Add automated regression tests in tests/test_uvr_cli.py verifying license file and module SPDX headers
271 lines
12 KiB
Python
271 lines
12 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# Copyright (C) 2026 RevLayDust
|
|
|
|
"""Benchmarking engine managing warmup, measurement rounds, timing, and export."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
import uuid
|
|
|
|
from .exporter import export_benchmark_csv, export_benchmark_json
|
|
from .model_resolver import HeadlessModelData
|
|
from .runner import execute_inference, get_audio_metadata
|
|
from .telemetry import TelemetrySampler, collect_system_info, sync_cuda
|
|
|
|
|
|
def _get_git_commit() -> str:
|
|
"""Return the current Git commit SHA, or 'unknown' if unavailable."""
|
|
try:
|
|
result = subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
capture_output=True, text=True, timeout=5,
|
|
cwd=str(Path(__file__).resolve().parent.parent),
|
|
)
|
|
if result.returncode == 0:
|
|
return result.stdout.strip()
|
|
except Exception:
|
|
pass
|
|
return "unknown"
|
|
|
|
|
|
def _compute_file_sha256(file_path: Path) -> str:
|
|
"""Compute a SHA-256 hash of a file."""
|
|
sha = hashlib.sha256()
|
|
try:
|
|
with file_path.open("rb") as f:
|
|
for chunk in iter(lambda: f.read(8192), b""):
|
|
sha.update(chunk)
|
|
return sha.hexdigest()
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
class BenchmarkEngine:
|
|
"""Orchestrates end-to-end performance benchmarking for UVR inference."""
|
|
|
|
def __init__(
|
|
self,
|
|
model_data: HeadlessModelData,
|
|
audio_path: str | Path,
|
|
results_dir: str | Path = "benchmark_results",
|
|
rounds: int = 1,
|
|
warmup: int = 0,
|
|
device_index: int = 0,
|
|
) -> None:
|
|
self.model_data = model_data
|
|
self.audio_path = Path(audio_path).resolve()
|
|
self.results_dir = Path(results_dir).resolve()
|
|
self.rounds = max(1, rounds)
|
|
self.warmup = max(0, warmup)
|
|
self.device_index = device_index
|
|
|
|
self.session_id = f"session_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
|
self.results_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def run(self, verbose: bool = True) -> Dict[str, Any]:
|
|
"""Run warmup and measurement rounds, gather telemetry, and export results."""
|
|
if verbose:
|
|
print(f"\n{'='*70}")
|
|
print(f" UVR BENCHMARK SESSION: {self.session_id}")
|
|
print(f"{'='*70}")
|
|
print(f"Model : {self.model_data.model_name}")
|
|
print(f"Backend : {self.model_data.process_method}")
|
|
print(f"Precision : {self.model_data.model_precision}")
|
|
print(f"Audio File : {self.audio_path.name}")
|
|
print(f"Rounds : {self.rounds} (Warmup: {self.warmup})")
|
|
print(f"Device Index : {self.device_index}")
|
|
print(f"{'='*70}\n")
|
|
|
|
system_info = collect_system_info(self.device_index)
|
|
audio_info = get_audio_metadata(self.audio_path)
|
|
|
|
git_commit = _get_git_commit()
|
|
python_version = sys.version.split()[0]
|
|
input_hash = _compute_file_sha256(self.audio_path)
|
|
|
|
all_runs_data: List[Dict[str, Any]] = []
|
|
csv_rows: List[Dict[str, Any]] = []
|
|
|
|
total_iterations = self.warmup + self.rounds
|
|
current_iter = 0
|
|
|
|
temp_stems_dir = self.results_dir / "temp_stems" / self.session_id
|
|
temp_stems_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
for i in range(total_iterations):
|
|
current_iter += 1
|
|
is_warmup_run = current_iter <= self.warmup
|
|
round_number = current_iter if is_warmup_run else (current_iter - self.warmup)
|
|
run_label = f"Warmup {round_number}" if is_warmup_run else f"Measurement Round {round_number}/{self.rounds}"
|
|
|
|
if verbose:
|
|
print(f"--> Starting {run_label}...")
|
|
|
|
round_export_dir = temp_stems_dir / f"round_{current_iter}"
|
|
round_export_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
sampler = TelemetrySampler(device_index=self.device_index, interval_sec=0.1)
|
|
sync_cuda(self.device_index)
|
|
|
|
sampler.start()
|
|
start_time = time.perf_counter()
|
|
|
|
result = execute_inference(
|
|
model_data=self.model_data,
|
|
audio_path=self.audio_path,
|
|
export_dir=round_export_dir,
|
|
)
|
|
stems, is_valid = result
|
|
|
|
sync_cuda(self.device_index)
|
|
end_time = time.perf_counter()
|
|
telemetry = sampler.stop()
|
|
|
|
inference_time = getattr(result, "inference_time", None)
|
|
if inference_time is not None and inference_time > 0.0:
|
|
total_time = round(inference_time, 4)
|
|
else:
|
|
total_time = round(end_time - start_time, 4)
|
|
duration_sec = audio_info.get("duration_sec", 0.0)
|
|
speed_factor = round(duration_sec / total_time, 2) if total_time > 0 and duration_sec > 0 else 0.0
|
|
|
|
run_record = {
|
|
"iteration": current_iter,
|
|
"is_warmup": is_warmup_run,
|
|
"round": round_number,
|
|
"total_time_sec": total_time,
|
|
"speed_factor_rt": speed_factor,
|
|
"is_valid": is_valid,
|
|
"output_stems": [s.name for s in stems],
|
|
"telemetry": telemetry,
|
|
}
|
|
all_runs_data.append(run_record)
|
|
|
|
csv_row = {
|
|
"session_id": self.session_id,
|
|
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
"gpu_name": system_info.get("gpu_name", "N/A"),
|
|
"cpu_name": system_info.get("cpu_name") or system_info.get("cpu", "N/A"),
|
|
"model_name": self.model_data.model_name,
|
|
"model_hash": self.model_data.model_hash,
|
|
"backend": self.model_data.process_method,
|
|
"precision": self.model_data.model_precision,
|
|
"device": getattr(self.model_data, "device_name", "cpu"),
|
|
"segment_size": getattr(self.model_data, "mdx_segment_size", None) or getattr(self.model_data, "segment_size", None),
|
|
"overlap": (
|
|
self.model_data.overlap_mdx
|
|
if getattr(self.model_data, "overlap_mdx", "Default") != "Default"
|
|
else getattr(self.model_data, "overlap", 0.25)
|
|
),
|
|
"batch_size": getattr(self.model_data, "mdx_batch_size", getattr(self.model_data, "batch_size", 1)),
|
|
"audio_file": self.audio_path.name,
|
|
"audio_duration_sec": duration_sec,
|
|
"round": f"warmup_{round_number}" if is_warmup_run else str(round_number),
|
|
"total_time_sec": total_time,
|
|
"inference_time_sec": total_time, # Pure separator execution
|
|
"speed_factor_rt": speed_factor,
|
|
"peak_torch_vram_mb": telemetry.get("torch_peak_allocated_vram_mb", 0.0),
|
|
"peak_device_vram_mb": telemetry.get("peak_device_vram_mb", 0.0),
|
|
"avg_gpu_util_percent": telemetry.get("avg_gpu_utilization_percent", 0.0),
|
|
"peak_gpu_util_percent": telemetry.get("peak_gpu_utilization_percent", 0.0),
|
|
"avg_cpu_util_percent": telemetry.get("avg_process_cpu_percent", 0.0),
|
|
"peak_process_ram_mb": telemetry.get("peak_process_ram_mb", 0.0),
|
|
"git_commit": git_commit,
|
|
"python_version": python_version,
|
|
"input_hash": input_hash,
|
|
"status": "COMPLETED" if is_valid else "VALIDATION_FAILED",
|
|
}
|
|
csv_rows.append(csv_row)
|
|
|
|
if verbose:
|
|
print(
|
|
f" Time: {total_time:.2f}s ({speed_factor}x RT) | "
|
|
f"Peak Torch VRAM: {telemetry.get('torch_peak_allocated_vram_mb', 0):.1f} MiB | "
|
|
f"Peak Device VRAM: {telemetry.get('peak_device_vram_mb', 0):.1f} MiB | "
|
|
f"GPU Util: {telemetry.get('avg_gpu_utilization_percent', 0):.1f}% (Peak: {telemetry.get('peak_gpu_utilization_percent', 0):.1f}%)"
|
|
)
|
|
|
|
finally:
|
|
# Clean up temporary stems after benchmarking
|
|
try:
|
|
shutil.rmtree(temp_stems_dir, ignore_errors=True)
|
|
except Exception:
|
|
pass
|
|
|
|
# Compute summary metrics across non-warmup rounds (or all if only warmup was run)
|
|
measured_runs = [r for r in all_runs_data if not r["is_warmup"]]
|
|
if not measured_runs:
|
|
measured_runs = all_runs_data
|
|
|
|
measured_times = [r["total_time_sec"] for r in measured_runs]
|
|
measured_speed_factors = [r["speed_factor_rt"] for r in measured_runs]
|
|
measured_torch_vram = [r["telemetry"].get("torch_peak_allocated_vram_mb", 0.0) for r in measured_runs]
|
|
measured_device_vram = [r["telemetry"].get("peak_device_vram_mb", 0.0) for r in measured_runs]
|
|
|
|
summary = {
|
|
"rounds_measured": len(measured_runs),
|
|
"warmup_rounds": self.warmup,
|
|
"avg_total_time_sec": round(sum(measured_times) / len(measured_times), 4),
|
|
"min_total_time_sec": min(measured_times),
|
|
"max_total_time_sec": max(measured_times),
|
|
"avg_speed_factor_rt": round(sum(measured_speed_factors) / len(measured_speed_factors), 2),
|
|
"peak_torch_allocated_vram_mb": max(measured_torch_vram) if measured_torch_vram else 0.0,
|
|
"peak_device_vram_mb": max(measured_device_vram) if measured_device_vram else 0.0,
|
|
}
|
|
|
|
report_data: Dict[str, Any] = {
|
|
"session_id": self.session_id,
|
|
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
"system_info": system_info,
|
|
"audio_info": audio_info,
|
|
"model_info": {
|
|
"name": self.model_data.model_name,
|
|
"hash": self.model_data.model_hash,
|
|
"backend": self.model_data.process_method,
|
|
"precision": self.model_data.model_precision,
|
|
"device": getattr(self.model_data, "device_name", "cpu"),
|
|
"segment_size": getattr(self.model_data, "mdx_segment_size", None) or getattr(self.model_data, "segment_size", None),
|
|
"overlap": (
|
|
self.model_data.overlap_mdx
|
|
if getattr(self.model_data, "overlap_mdx", "Default") != "Default"
|
|
else getattr(self.model_data, "overlap", 0.25)
|
|
),
|
|
"batch_size": getattr(self.model_data, "mdx_batch_size", getattr(self.model_data, "batch_size", 1)),
|
|
},
|
|
"git_commit": git_commit,
|
|
"python_version": python_version,
|
|
"input_hash": input_hash,
|
|
"summary": summary,
|
|
"runs": all_runs_data,
|
|
}
|
|
|
|
# Export JSON and CSV
|
|
json_file = self.results_dir / f"{self.session_id}.json"
|
|
csv_file = self.results_dir / "summary.csv"
|
|
|
|
export_benchmark_json(report_data, json_file)
|
|
export_benchmark_csv(csv_rows, csv_file)
|
|
|
|
if verbose:
|
|
print(f"\n{'='*70}")
|
|
print(f" BENCHMARK SUMMARY")
|
|
print(f"{'='*70}")
|
|
print(f"Avg Time : {summary['avg_total_time_sec']:.2f}s (Min: {summary['min_total_time_sec']:.2f}s, Max: {summary['max_total_time_sec']:.2f}s)")
|
|
print(f"Speed Factor : {summary['avg_speed_factor_rt']:.2f}x real-time")
|
|
print(f"Peak VRAM : Torch: {summary['peak_torch_allocated_vram_mb']:.1f} MiB | Device: {summary['peak_device_vram_mb']:.1f} MiB")
|
|
print(f"JSON Report : {json_file}")
|
|
print(f"CSV Summary : {csv_file}")
|
|
print(f"{'='*70}\n")
|
|
|
|
return report_data
|