mirror of
https://github.com/RevLayDust/RLD-UVR-Opt.git
synced 2026-11-20 13:07:58 +09:00
1049 lines
43 KiB
Python
1049 lines
43 KiB
Python
from __future__ import annotations
|
|
|
|
import functools
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import re
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
FALSE_VALUES = {"0", "false", "no", "off", "disabled"}
|
|
SEPARATOR_BACKENDS = {
|
|
"SeperateVR": "VR",
|
|
"SeperateMDX": "MDX-Net ONNX",
|
|
"SeperateMDXC": "MDX PyTorch",
|
|
"SeperateDemucs": "Demucs",
|
|
}
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_CACHE_ROOT = PROJECT_ROOT / "benchmark_cache"
|
|
RECORDER_VERSION = "1.0.0"
|
|
|
|
_WINDOWS_PATH = re.compile(r"(?i)(?<![A-Za-z0-9])[A-Z]:[\\/][^\r\n\"']+")
|
|
_UNC_PATH = re.compile(r"(?<![\\A-Za-z0-9])\\\\[^\\\r\n\"']+[\\/][^\r\n\"']+")
|
|
_ABSOLUTE_UNIX_PATH = re.compile(r"(?<![:/A-Za-z0-9])/(?!/)[^\r\n\"']+")
|
|
_SECRET_KEY = re.compile(r"(?i)(token|secret|password|passwd|api[_-]?key|credential)")
|
|
_SECRET_ASSIGNMENT = re.compile(
|
|
r"(?i)\b(token|secret|password|passwd|api[_-]?key|credential)\b\s*[:=]\s*[^\s,;]+"
|
|
)
|
|
_ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
|
|
|
|
|
|
def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(chunk_size), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def slugify(value: str, fallback: str = "unknown") -> str:
|
|
slug = re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip("-")
|
|
return slug or fallback
|
|
|
|
|
|
def sanitize_text(value: str) -> str:
|
|
sanitized = _ANSI_ESCAPE.sub("", str(value))
|
|
sanitized = _WINDOWS_PATH.sub("<redacted-path>", sanitized)
|
|
sanitized = _UNC_PATH.sub("<redacted-path>", sanitized)
|
|
sanitized = _ABSOLUTE_UNIX_PATH.sub("<redacted-path>", sanitized)
|
|
return _SECRET_ASSIGNMENT.sub(lambda match: f"{match.group(1)}=<redacted>", sanitized)
|
|
|
|
|
|
def sanitize_data(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
return {
|
|
key: "<redacted>" if _SECRET_KEY.search(str(key)) else sanitize_data(item)
|
|
for key, item in value.items()
|
|
}
|
|
if isinstance(value, (list, tuple)):
|
|
return [sanitize_data(item) for item in value]
|
|
if isinstance(value, str):
|
|
return sanitize_text(value)
|
|
return value
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(sanitize_data(value), indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
|
|
def collect_environment(device_index: int = 0) -> dict[str, Any]:
|
|
hardware: dict[str, Any] = {
|
|
"os": platform.platform(),
|
|
"cpu": platform.processor() or platform.machine() or "unknown",
|
|
"system_ram_gb": _system_ram_gb(),
|
|
"gpu_name": None,
|
|
"gpu_vram_mb": None,
|
|
"driver_version": None,
|
|
"compute_capability": None,
|
|
"form_factor": "unknown",
|
|
"power_limit_w": None,
|
|
}
|
|
software: dict[str, Any] = {
|
|
"python_version": platform.python_version(),
|
|
"torch_version": None,
|
|
"cuda_runtime_version": None,
|
|
"cudnn_version": None,
|
|
"torch_cuda_available": False,
|
|
"onnxruntime_version": None,
|
|
"onnxruntime_providers": [],
|
|
}
|
|
warnings: list[str] = []
|
|
try:
|
|
import torch
|
|
|
|
software["torch_version"] = torch.__version__
|
|
software["cuda_runtime_version"] = torch.version.cuda
|
|
software["cudnn_version"] = torch.backends.cudnn.version()
|
|
software["torch_cuda_available"] = bool(torch.cuda.is_available())
|
|
if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count():
|
|
properties = torch.cuda.get_device_properties(device_index)
|
|
hardware["gpu_name"] = properties.name
|
|
hardware["gpu_vram_mb"] = properties.total_memory / (1024 * 1024)
|
|
hardware["compute_capability"] = f"{properties.major}.{properties.minor}"
|
|
hardware["form_factor"] = "laptop" if "laptop" in properties.name.lower() else "unknown"
|
|
except Exception as error:
|
|
warnings.append(f"PyTorch metadata unavailable: {type(error).__name__}")
|
|
try:
|
|
import onnxruntime as ort
|
|
|
|
software["onnxruntime_version"] = ort.__version__
|
|
software["onnxruntime_providers"] = list(ort.get_available_providers())
|
|
except Exception as error:
|
|
warnings.append(f"ONNX Runtime metadata unavailable: {type(error).__name__}")
|
|
_add_nvml_metadata(hardware, device_index, warnings)
|
|
return sanitize_data({
|
|
"recorder_version": RECORDER_VERSION,
|
|
"timestamp_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"),
|
|
"hardware": hardware,
|
|
"software": software,
|
|
"warnings": warnings,
|
|
})
|
|
|
|
|
|
def _system_ram_gb() -> float | None:
|
|
try:
|
|
import psutil
|
|
|
|
return psutil.virtual_memory().total / (1024 ** 3)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _add_nvml_metadata(hardware: dict[str, Any], device_index: int, warnings: list[str]) -> None:
|
|
try:
|
|
import pynvml
|
|
|
|
pynvml.nvmlInit()
|
|
try:
|
|
handle, mapped_by_pci = nvml_handle_for_torch_device(pynvml, device_index)
|
|
if not mapped_by_pci:
|
|
warnings.append("NVML metadata used device-index fallback")
|
|
name = pynvml.nvmlDeviceGetName(handle)
|
|
hardware["gpu_name"] = name.decode(errors="replace") if isinstance(name, bytes) else str(name)
|
|
hardware["gpu_vram_mb"] = pynvml.nvmlDeviceGetMemoryInfo(handle).total / (1024 * 1024)
|
|
driver = pynvml.nvmlSystemGetDriverVersion()
|
|
hardware["driver_version"] = driver.decode() if isinstance(driver, bytes) else str(driver)
|
|
try:
|
|
hardware["power_limit_w"] = pynvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000.0
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
pynvml.nvmlShutdown()
|
|
except Exception as error:
|
|
warnings.append(f"NVML metadata unavailable: {type(error).__name__}")
|
|
|
|
|
|
def nvml_handle_for_torch_device(nvml: Any, device_index: int) -> tuple[Any, bool]:
|
|
try:
|
|
import torch
|
|
|
|
properties = torch.cuda.get_device_properties(device_index)
|
|
candidates = (
|
|
f"{int(properties.pci_domain_id):08x}:{int(properties.pci_bus_id):02x}:{int(properties.pci_device_id):02x}.0",
|
|
f"{int(properties.pci_domain_id):04x}:{int(properties.pci_bus_id):02x}:{int(properties.pci_device_id):02x}.0",
|
|
)
|
|
for candidate in candidates:
|
|
for value in (candidate, candidate.encode("ascii")):
|
|
try:
|
|
return nvml.nvmlDeviceGetHandleByPciBusId(value), True
|
|
except Exception:
|
|
continue
|
|
except Exception:
|
|
pass
|
|
return nvml.nvmlDeviceGetHandleByIndex(device_index), False
|
|
|
|
|
|
def install_gui_runtime_benchmark_hooks(
|
|
namespace: dict[str, Any],
|
|
recorder: "GUIRuntimeBenchmarkRecorder | None" = None,
|
|
) -> "GUIRuntimeBenchmarkRecorder | None":
|
|
if os.environ.get("UVR_GUI_BENCHMARK", "1").strip().lower() in FALSE_VALUES:
|
|
return None
|
|
existing = namespace.get("_GUI_RUNTIME_BENCHMARK_RECORDER")
|
|
runtime_recorder = recorder or existing or GUIRuntimeBenchmarkRecorder(namespace)
|
|
namespace["_GUI_RUNTIME_BENCHMARK_RECORDER"] = runtime_recorder
|
|
for class_name, backend in SEPARATOR_BACKENDS.items():
|
|
separator_class = namespace.get(class_name)
|
|
if separator_class is None:
|
|
continue
|
|
original = separator_class.seperate
|
|
if getattr(original, "_uvr_gui_benchmark_wrapped", False):
|
|
continue
|
|
|
|
@functools.wraps(original)
|
|
def wrapped(instance, *args, __original=original, __backend=backend, **kwargs):
|
|
return runtime_recorder.run(instance, __backend, __original, args, kwargs)
|
|
|
|
wrapped._uvr_gui_benchmark_wrapped = True
|
|
wrapped._uvr_gui_benchmark_original = original
|
|
separator_class.seperate = wrapped
|
|
return runtime_recorder
|
|
|
|
|
|
class GUIRuntimeBenchmarkRecorder:
|
|
def __init__(
|
|
self,
|
|
namespace: dict[str, Any],
|
|
output_root: Path | None = None,
|
|
sampler_factory: Callable[..., Any] | None = None,
|
|
environment_collector: Callable[[int], dict[str, Any]] = collect_environment,
|
|
progress_interval_seconds: float = 5.0,
|
|
):
|
|
self.namespace = namespace
|
|
configured_root = os.environ.get("UVR_GUI_BENCHMARK_DIR")
|
|
selected_root = Path(configured_root).expanduser() if configured_root else (
|
|
output_root or DEFAULT_CACHE_ROOT / "gui_runtime"
|
|
)
|
|
self.output_root = selected_root.resolve()
|
|
self.sampler_factory = sampler_factory or RuntimeResourceSampler
|
|
self.environment_collector = environment_collector
|
|
self.progress_interval_seconds = max(float(progress_interval_seconds), 1.0)
|
|
self._lock = threading.RLock()
|
|
self._thread_state = threading.local()
|
|
self._inference_index = 0
|
|
self._last_model_signature: tuple[str, str] | None = None
|
|
self._model_round = 0
|
|
self._environment_cache: dict[int, dict[str, Any]] = {}
|
|
self._model_hash_cache: dict[tuple[str, int, int], str] = {}
|
|
self._session_directory: Path | None = None
|
|
self._session_log: Path | None = None
|
|
|
|
def run(
|
|
self,
|
|
instance: Any,
|
|
backend: str,
|
|
original: Callable[..., Any],
|
|
args: tuple[Any, ...],
|
|
kwargs: dict[str, Any],
|
|
) -> Any:
|
|
depth = getattr(self._thread_state, "depth", 0)
|
|
if depth:
|
|
return original(instance, *args, **kwargs)
|
|
self._thread_state.depth = depth + 1
|
|
try:
|
|
return self._run_top_level(instance, backend, original, args, kwargs)
|
|
finally:
|
|
self._thread_state.depth = depth
|
|
|
|
def _run_top_level(
|
|
self,
|
|
instance: Any,
|
|
backend: str,
|
|
original: Callable[..., Any],
|
|
args: tuple[Any, ...],
|
|
kwargs: dict[str, Any],
|
|
) -> Any:
|
|
started = time.perf_counter()
|
|
record = None
|
|
instrumentor = None
|
|
sampler = None
|
|
try:
|
|
record = self._begin_record(instance, backend)
|
|
instrumentor = RuntimeStageInstrumentor(
|
|
self.namespace,
|
|
instance,
|
|
record["device_index"],
|
|
lambda name, value: self._record_stage(record, name, value),
|
|
)
|
|
sampler = self.sampler_factory(
|
|
device_index=record["device_index"],
|
|
progress_interval_seconds=self.progress_interval_seconds,
|
|
progress=lambda snapshot: self._record_progress(record, snapshot),
|
|
)
|
|
instrumentor.install()
|
|
sampler.start()
|
|
except Exception as error:
|
|
setup_warning = f"Recorder setup failed: {type(error).__name__}"
|
|
cleanup_warnings = []
|
|
if instrumentor is not None:
|
|
try:
|
|
instrumentor.restore()
|
|
except Exception as cleanup_error:
|
|
cleanup_warnings.append(
|
|
f"Recorder hook cleanup failed: {type(cleanup_error).__name__}"
|
|
)
|
|
if sampler is not None:
|
|
try:
|
|
sampler.stop()
|
|
except Exception as cleanup_error:
|
|
cleanup_warnings.append(
|
|
f"Recorder sampler cleanup failed: {type(cleanup_error).__name__}"
|
|
)
|
|
warnings = [setup_warning, *cleanup_warnings]
|
|
if record is not None:
|
|
for warning in warnings:
|
|
self._record_warning(record, warning)
|
|
self._notify_gui(instance, f"[Benchmark warning] {setup_warning}; processing will continue")
|
|
failure: BaseException | None = None
|
|
try:
|
|
return original(instance, *args, **kwargs)
|
|
except BaseException as backend_error:
|
|
failure = backend_error
|
|
raise
|
|
finally:
|
|
synchronize_cuda(selected_device_index(instance))
|
|
if record is not None:
|
|
self._finish_record_safely(
|
|
record,
|
|
instance,
|
|
time.perf_counter() - started,
|
|
{"warnings": warnings},
|
|
failure,
|
|
)
|
|
|
|
failure: BaseException | None = None
|
|
synchronize_cuda(record["device_index"])
|
|
try:
|
|
return original(instance, *args, **kwargs)
|
|
except BaseException as error:
|
|
failure = error
|
|
raise
|
|
finally:
|
|
synchronize_cuda(record["device_index"])
|
|
total_seconds = time.perf_counter() - started
|
|
resources: dict[str, Any] = {}
|
|
recorder_warnings = []
|
|
try:
|
|
instrumentor.finish()
|
|
except Exception as recorder_error:
|
|
recorder_warnings.append(
|
|
f"Stage finalization failed: {type(recorder_error).__name__}"
|
|
)
|
|
try:
|
|
resources = sampler.stop() or {}
|
|
except Exception as recorder_error:
|
|
recorder_warnings.append(
|
|
f"Resource sampler finalization failed: {type(recorder_error).__name__}"
|
|
)
|
|
try:
|
|
instrumentor.restore()
|
|
except Exception as recorder_error:
|
|
recorder_warnings.append(
|
|
f"Recorder hook cleanup failed: {type(recorder_error).__name__}"
|
|
)
|
|
resources.setdefault("warnings", []).extend(recorder_warnings)
|
|
self._finish_record_safely(
|
|
record,
|
|
instance,
|
|
total_seconds,
|
|
resources,
|
|
failure,
|
|
)
|
|
|
|
def _begin_record(self, instance: Any, backend: str) -> dict[str, Any]:
|
|
model_path = Path(str(getattr(instance, "model_path", "unknown-model")))
|
|
model_name = model_path.name or str(getattr(instance, "model_basename", "unknown-model"))
|
|
precision = str(getattr(getattr(instance, "model_data", None), "model_precision", "unknown"))
|
|
device_index = selected_device_index(instance)
|
|
normalized_model = os.path.normcase(os.path.abspath(str(model_path)))
|
|
signature = (normalized_model, precision)
|
|
with self._lock:
|
|
self._inference_index += 1
|
|
inference_index = self._inference_index
|
|
if signature == self._last_model_signature:
|
|
self._model_round += 1
|
|
else:
|
|
self._last_model_signature = signature
|
|
self._model_round = 1
|
|
model_round = self._model_round
|
|
self._ensure_session()
|
|
|
|
environment = self._environment(device_index)
|
|
input_metadata = read_input_metadata(getattr(instance, "audio_file", None))
|
|
model_hash = self._model_hash(model_path)
|
|
cache_state = model_cache_state(self.namespace.get("MODEL_CACHE"), model_path, precision)
|
|
file_slug = slugify(f"{model_path.stem}-{precision}")
|
|
text_path = self._session_directory / f"inference_{inference_index:04d}_{file_slug}.log"
|
|
json_path = self._session_directory / f"inference_{inference_index:04d}_{file_slug}.json"
|
|
record = {
|
|
"inference_index": inference_index,
|
|
"inference_label": f"{ordinal(inference_index)} Inference",
|
|
"model": model_name,
|
|
"model_sha256": model_hash,
|
|
"model_round": model_round,
|
|
"backend": backend,
|
|
"precision": precision,
|
|
"device_index": device_index,
|
|
"cache_state": cache_state,
|
|
"os": environment.get("hardware", {}).get("os") or platform.platform(),
|
|
"environment": environment,
|
|
"input": input_metadata,
|
|
"started_utc": utc_now(),
|
|
"status": "running",
|
|
"total_time_seconds": None,
|
|
"stages": {},
|
|
"resources": {},
|
|
"runtime": {},
|
|
"outputs": {},
|
|
"warnings": [],
|
|
"error": None,
|
|
"text_path": text_path,
|
|
"json_path": json_path,
|
|
}
|
|
header = [
|
|
f"{record['inference_label']}:",
|
|
f"Model: {model_name}",
|
|
f"Model round: {model_round}",
|
|
f"OS: {record['os']}",
|
|
f"Backend: {backend}",
|
|
f"Precision: {precision}",
|
|
f"Model cache: {cache_state}",
|
|
f"Model SHA-256: {model_hash or 'unavailable'}",
|
|
f"Input: {input_metadata.get('name') or 'unavailable'}",
|
|
f"Input duration: {format_seconds(input_metadata.get('duration_seconds'))}",
|
|
f"Started UTC: {record['started_utc']}",
|
|
"Status: RUNNING",
|
|
]
|
|
text_path.write_text("\n".join(header) + "\n", encoding="utf-8")
|
|
with self._session_log.open("a", encoding="utf-8") as stream:
|
|
stream.write("\n".join(header) + "\n")
|
|
self._checkpoint(record)
|
|
relative_log = self._session_log.relative_to(self.output_root.parent)
|
|
self._notify_gui(instance, f"[Benchmark] Recording {record['inference_label']} -> {relative_log}")
|
|
return record
|
|
|
|
def _finish_record(
|
|
self,
|
|
record: dict[str, Any],
|
|
instance: Any,
|
|
total_seconds: float,
|
|
resources: dict[str, Any],
|
|
failure: BaseException | None,
|
|
) -> None:
|
|
record["status"] = "completed" if failure is None else "failed"
|
|
record["total_time_seconds"] = total_seconds
|
|
record["ended_utc"] = utc_now()
|
|
record["resources"] = resources
|
|
for warning in resources.get("warnings", []):
|
|
self._record_warning(record, warning)
|
|
try:
|
|
record["runtime"] = effective_runtime(self.namespace, instance)
|
|
except Exception as error:
|
|
record["runtime"] = {}
|
|
self._record_warning(record, f"Runtime metadata unavailable: {type(error).__name__}")
|
|
try:
|
|
record["outputs"] = output_metadata(instance)
|
|
except Exception as error:
|
|
record["outputs"] = {"count": 0, "validation": "unavailable"}
|
|
self._record_warning(record, f"Output metadata unavailable: {type(error).__name__}")
|
|
if failure is not None:
|
|
record["error"] = f"{type(failure).__name__}: {sanitize_text(str(failure))}"
|
|
final_lines = [
|
|
f"Effective precision: {record['runtime'].get('effective_precision') or 'unavailable'}",
|
|
f"Execution provider: {', '.join(record['runtime'].get('execution_provider') or []) or 'unavailable'}",
|
|
f"Peak process RAM: {format_mb(resources.get('peak_process_ram_mb'))}",
|
|
f"Peak device VRAM: {format_mb(resources.get('peak_device_vram_mb'))}",
|
|
f"Peak Torch allocated VRAM: {format_mb(resources.get('torch_peak_allocated_vram_mb'))}",
|
|
f"Peak Torch reserved VRAM: {format_mb(resources.get('torch_peak_reserved_vram_mb'))}",
|
|
f"Average GPU utilization: {format_percent(resources.get('average_gpu_utilization_percent'))}",
|
|
f"Peak GPU utilization: {format_percent(resources.get('peak_gpu_utilization_percent'))}",
|
|
f"Average process CPU: {format_percent(resources.get('average_process_cpu_percent'))}",
|
|
f"Peak process CPU: {format_percent(resources.get('peak_process_cpu_percent'))}",
|
|
f"Output files: {record['outputs'].get('count', 0)}",
|
|
f"Output validation: {record['outputs'].get('validation', 'unavailable')}",
|
|
f"Status: {record['status'].upper()}",
|
|
]
|
|
if record["error"]:
|
|
final_lines.append(f"Error: {record['error']}")
|
|
final_lines.extend([
|
|
f"Total-time: {total_seconds:.6f} s",
|
|
f"Ended UTC: {record['ended_utc']}",
|
|
"=======",
|
|
])
|
|
for line in final_lines:
|
|
self._append(record, line, checkpoint=False)
|
|
self._checkpoint(record)
|
|
self._notify_gui(
|
|
instance,
|
|
f"[Benchmark] {record['inference_label']} {record['status']} in {total_seconds:.3f}s",
|
|
)
|
|
|
|
def _finish_record_safely(
|
|
self,
|
|
record: dict[str, Any],
|
|
instance: Any,
|
|
total_seconds: float,
|
|
resources: dict[str, Any],
|
|
failure: BaseException | None,
|
|
) -> None:
|
|
try:
|
|
self._finish_record(record, instance, total_seconds, resources, failure)
|
|
except Exception as recorder_error:
|
|
self._notify_gui(
|
|
instance,
|
|
f"[Benchmark warning] Recorder finalization failed: {type(recorder_error).__name__}",
|
|
)
|
|
|
|
def _record_stage(self, record: dict[str, Any], name: str, value: float) -> None:
|
|
with self._lock:
|
|
is_cumulative = name in record["stages"]
|
|
record["stages"][name] = value
|
|
label = f"Stage {name} (cumulative)" if is_cumulative else f"Stage {name}"
|
|
self._append(record, f"{label}: {value:.6f} s")
|
|
|
|
def _record_progress(self, record: dict[str, Any], snapshot: dict[str, Any]) -> None:
|
|
with self._lock:
|
|
record["resources"].update(snapshot)
|
|
line = (
|
|
f"Progress: elapsed={snapshot.get('elapsed_seconds', 0.0):.1f}s, "
|
|
f"CPU={format_percent(snapshot.get('process_cpu_percent'))}, "
|
|
f"GPU={format_percent(snapshot.get('gpu_utilization_percent'))}, "
|
|
f"device VRAM={format_mb(snapshot.get('device_vram_mb'))}"
|
|
)
|
|
self._append(record, line)
|
|
|
|
def _record_warning(self, record: dict[str, Any], warning: str) -> None:
|
|
cleaned = sanitize_text(str(warning))
|
|
with self._lock:
|
|
if cleaned in record["warnings"]:
|
|
return
|
|
record["warnings"].append(cleaned)
|
|
self._append(record, f"Warning: {cleaned}")
|
|
|
|
def _append(self, record: dict[str, Any], line: str, checkpoint: bool = True) -> None:
|
|
with self._lock:
|
|
for path in (record["text_path"], self._session_log):
|
|
with path.open("a", encoding="utf-8") as stream:
|
|
stream.write(line + "\n")
|
|
if checkpoint:
|
|
self._checkpoint(record)
|
|
|
|
def _checkpoint(self, record: dict[str, Any]) -> None:
|
|
write_json(record["json_path"], public_record(record))
|
|
|
|
def _ensure_session(self) -> None:
|
|
if self._session_directory is not None:
|
|
return
|
|
session_name = datetime.now(timezone.utc).strftime("session_%Y%m%dT%H%M%SZ") + f"_{os.getpid()}"
|
|
self._session_directory = self.output_root / session_name
|
|
self._session_directory.mkdir(parents=True, exist_ok=True)
|
|
self._session_log = self._session_directory / "benchmark.log"
|
|
uvr_path = Path(__file__).resolve().parents[1] / "UVR.py"
|
|
uvr_hash = sha256_file(uvr_path) if uvr_path.is_file() else "unavailable"
|
|
self._session_log.write_text(
|
|
"UVR GUI Backend Benchmark\n"
|
|
f"Session started UTC: {utc_now()}\n"
|
|
f"UVR.py SHA-256: {uvr_hash}\n"
|
|
"=======\n",
|
|
encoding="utf-8",
|
|
)
|
|
self.output_root.mkdir(parents=True, exist_ok=True)
|
|
(self.output_root / "latest.txt").write_text(session_name + "\n", encoding="utf-8")
|
|
|
|
def _environment(self, device_index: int) -> dict[str, Any]:
|
|
if device_index not in self._environment_cache:
|
|
try:
|
|
self._environment_cache[device_index] = self.environment_collector(device_index)
|
|
except Exception as error:
|
|
self._environment_cache[device_index] = {
|
|
"hardware": {"os": platform.platform()},
|
|
"software": {},
|
|
"warnings": [f"Environment metadata unavailable: {type(error).__name__}"],
|
|
}
|
|
return self._environment_cache[device_index]
|
|
|
|
def _model_hash(self, model_path: Path) -> str | None:
|
|
try:
|
|
stat = model_path.stat()
|
|
key = (os.path.normcase(str(model_path.resolve())), stat.st_size, stat.st_mtime_ns)
|
|
if key not in self._model_hash_cache:
|
|
self._model_hash_cache[key] = sha256_file(model_path)
|
|
return self._model_hash_cache[key]
|
|
except Exception:
|
|
return None
|
|
|
|
@staticmethod
|
|
def _notify_gui(instance: Any, message: str) -> None:
|
|
writer = getattr(instance, "write_to_console", None)
|
|
if not callable(writer):
|
|
return
|
|
try:
|
|
writer(message, base_text="")
|
|
except TypeError:
|
|
try:
|
|
writer(message)
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
class RuntimeStageInstrumentor:
|
|
def __init__(
|
|
self,
|
|
namespace: dict[str, Any],
|
|
instance: Any,
|
|
device_index: int,
|
|
stage_callback: Callable[[str, float], None],
|
|
):
|
|
self.namespace = namespace
|
|
self.instance = instance
|
|
self.device_index = device_index
|
|
self.stage_callback = stage_callback
|
|
self.values: dict[str, float] = {}
|
|
self._restores: list[Callable[[], None]] = []
|
|
self._model_load_started: float | None = None
|
|
|
|
def install(self) -> None:
|
|
model_cache = self.namespace.get("MODEL_CACHE")
|
|
if model_cache is not None:
|
|
original_get = model_cache.get_model
|
|
original_set = model_cache.set_model
|
|
|
|
def get_model(*args, **kwargs):
|
|
model = original_get(*args, **kwargs)
|
|
if model is None and self._model_load_started is None:
|
|
synchronize_cuda(self.device_index)
|
|
self._model_load_started = time.perf_counter()
|
|
return model
|
|
|
|
def set_model(*args, **kwargs):
|
|
result = original_set(*args, **kwargs)
|
|
self._finish_model_loading()
|
|
return result
|
|
|
|
self._patch_attribute(model_cache, "get_model", get_model)
|
|
self._patch_attribute(model_cache, "set_model", set_model)
|
|
|
|
backend_name = type(self.instance).__name__
|
|
if backend_name in {"SeperateMDX", "SeperateMDXC", "SeperateDemucs"}:
|
|
self._patch_namespace("prepare_mix", self._timed(self.namespace["prepare_mix"], "preprocessing"))
|
|
if backend_name in {"SeperateMDX", "SeperateMDXC"}:
|
|
self._patch_attribute(self.instance, "demix", self._timed(self.instance.demix, "inference"))
|
|
elif backend_name == "SeperateDemucs":
|
|
self._patch_attribute(
|
|
self.instance,
|
|
"demix_demucs",
|
|
self._timed(self.instance.demix_demucs, "inference"),
|
|
)
|
|
elif backend_name == "SeperateVR":
|
|
self._patch_attribute(self.instance, "loading_mix", self._timed(self.instance.loading_mix, "preprocessing"))
|
|
self._patch_attribute(self.instance, "inference_vr", self._timed(self.instance.inference_vr, "inference"))
|
|
if hasattr(self.instance, "write_audio"):
|
|
self._patch_attribute(self.instance, "write_audio", self._timed(self.instance.write_audio, "output_writing"))
|
|
|
|
def finish(self) -> None:
|
|
self._finish_model_loading()
|
|
|
|
def restore(self) -> None:
|
|
errors = []
|
|
for restore in reversed(self._restores):
|
|
try:
|
|
restore()
|
|
except Exception as error:
|
|
errors.append(error)
|
|
self._restores.clear()
|
|
if errors:
|
|
raise RuntimeError(f"failed to restore {len(errors)} benchmark hook(s)") from errors[0]
|
|
|
|
def _timed(self, function: Callable[..., Any], stage_name: str) -> Callable[..., Any]:
|
|
@functools.wraps(function)
|
|
def wrapped(*args, **kwargs):
|
|
synchronize_cuda(self.device_index)
|
|
started = time.perf_counter()
|
|
try:
|
|
return function(*args, **kwargs)
|
|
finally:
|
|
synchronize_cuda(self.device_index)
|
|
elapsed = time.perf_counter() - started
|
|
self.values[stage_name] = self.values.get(stage_name, 0.0) + elapsed
|
|
self.stage_callback(stage_name, self.values[stage_name])
|
|
|
|
return wrapped
|
|
|
|
def _finish_model_loading(self) -> None:
|
|
if self._model_load_started is None:
|
|
return
|
|
synchronize_cuda(self.device_index)
|
|
elapsed = time.perf_counter() - self._model_load_started
|
|
self._model_load_started = None
|
|
self.values["model_loading"] = self.values.get("model_loading", 0.0) + elapsed
|
|
self.stage_callback("model_loading", self.values["model_loading"])
|
|
|
|
def _patch_namespace(self, name: str, replacement: Any) -> None:
|
|
original = self.namespace[name]
|
|
self.namespace[name] = replacement
|
|
self._restores.append(lambda: self.namespace.__setitem__(name, original))
|
|
|
|
def _patch_attribute(self, owner: Any, name: str, replacement: Any) -> None:
|
|
owner_dict = getattr(owner, "__dict__", {})
|
|
had_local_value = name in owner_dict
|
|
local_value = owner_dict.get(name)
|
|
setattr(owner, name, replacement)
|
|
|
|
def restore() -> None:
|
|
if had_local_value:
|
|
setattr(owner, name, local_value)
|
|
else:
|
|
try:
|
|
delattr(owner, name)
|
|
except AttributeError:
|
|
pass
|
|
|
|
self._restores.append(restore)
|
|
|
|
|
|
class RuntimeResourceSampler:
|
|
def __init__(
|
|
self,
|
|
device_index: int = 0,
|
|
interval_seconds: float = 0.25,
|
|
progress_interval_seconds: float = 5.0,
|
|
progress: Callable[[dict[str, Any]], None] | None = None,
|
|
):
|
|
self.device_index = device_index
|
|
self.interval_seconds = max(float(interval_seconds), 0.05)
|
|
self.progress_interval_seconds = max(float(progress_interval_seconds), self.interval_seconds)
|
|
self.progress = progress or (lambda snapshot: None)
|
|
self._stop = threading.Event()
|
|
self._thread: threading.Thread | None = None
|
|
self._started = 0.0
|
|
self._last_progress = 0.0
|
|
self._psutil: Any = None
|
|
self._process: Any = None
|
|
self._nvml: Any = None
|
|
self._handle: Any = None
|
|
self._sample_count = 0
|
|
self._process_cpu_total = 0.0
|
|
self._process_cpu_peak: float | None = None
|
|
self._system_cpu_total = 0.0
|
|
self._system_cpu_peak: float | None = None
|
|
self._process_ram_peak: float | None = None
|
|
self._gpu_util_total = 0.0
|
|
self._gpu_util_peak: float | None = None
|
|
self._device_vram_baseline: float | None = None
|
|
self._device_vram_peak: float | None = None
|
|
self._power_total = 0.0
|
|
self._power_peak: float | None = None
|
|
self._power_samples = 0
|
|
self.warnings: list[str] = []
|
|
|
|
def start(self) -> None:
|
|
self._setup_psutil()
|
|
self._setup_nvml()
|
|
reset_torch_peak_memory(self.device_index, self.warnings)
|
|
self._started = time.perf_counter()
|
|
self._last_progress = self._started
|
|
self._sample(emit_progress=False)
|
|
self._stop.clear()
|
|
self._thread = threading.Thread(target=self._run, name="uvr-gui-benchmark-sampler", daemon=True)
|
|
self._thread.start()
|
|
|
|
def stop(self) -> dict[str, Any]:
|
|
self._stop.set()
|
|
if self._thread is not None:
|
|
self._thread.join(timeout=max(1.0, self.interval_seconds * 4))
|
|
self._sample(emit_progress=False)
|
|
if self._nvml is not None:
|
|
try:
|
|
self._nvml.nvmlShutdown()
|
|
except Exception:
|
|
pass
|
|
torch_memory = torch_peak_memory(self.device_index)
|
|
count = max(self._sample_count, 1)
|
|
return {
|
|
"sampling_interval_ms": self.interval_seconds * 1000.0,
|
|
"average_process_cpu_percent": self._process_cpu_total / count if self._process_cpu_peak is not None else None,
|
|
"peak_process_cpu_percent": self._process_cpu_peak,
|
|
"average_system_cpu_percent": self._system_cpu_total / count if self._system_cpu_peak is not None else None,
|
|
"peak_system_cpu_percent": self._system_cpu_peak,
|
|
"peak_process_ram_mb": self._process_ram_peak,
|
|
"average_gpu_utilization_percent": self._gpu_util_total / count if self._gpu_util_peak is not None else None,
|
|
"peak_gpu_utilization_percent": self._gpu_util_peak,
|
|
"baseline_device_vram_mb": self._device_vram_baseline,
|
|
"peak_device_vram_mb": self._device_vram_peak,
|
|
"average_power_w": self._power_total / self._power_samples if self._power_samples else None,
|
|
"peak_power_w": self._power_peak,
|
|
**torch_memory,
|
|
"warnings": list(dict.fromkeys(self.warnings)),
|
|
}
|
|
|
|
def _run(self) -> None:
|
|
while not self._stop.wait(self.interval_seconds):
|
|
self._sample(emit_progress=True)
|
|
|
|
def _sample(self, emit_progress: bool) -> None:
|
|
snapshot: dict[str, Any] = {
|
|
"elapsed_seconds": max(0.0, time.perf_counter() - self._started) if self._started else 0.0,
|
|
"process_cpu_percent": None,
|
|
"gpu_utilization_percent": None,
|
|
"device_vram_mb": None,
|
|
}
|
|
if self._process is not None:
|
|
try:
|
|
process_cpu = float(self._process.cpu_percent(None))
|
|
system_cpu = float(self._psutil.cpu_percent(None))
|
|
process_ram = self._process.memory_info().rss / (1024 * 1024)
|
|
self._process_cpu_total += process_cpu
|
|
self._process_cpu_peak = max(self._process_cpu_peak or 0.0, process_cpu)
|
|
self._system_cpu_total += system_cpu
|
|
self._system_cpu_peak = max(self._system_cpu_peak or 0.0, system_cpu)
|
|
self._process_ram_peak = max(self._process_ram_peak or 0.0, process_ram)
|
|
snapshot["process_cpu_percent"] = process_cpu
|
|
except Exception as error:
|
|
self._warn_once(f"CPU/RAM sampling stopped: {type(error).__name__}")
|
|
self._process = None
|
|
if self._handle is not None:
|
|
try:
|
|
utilization = float(self._nvml.nvmlDeviceGetUtilizationRates(self._handle).gpu)
|
|
device_vram = self._nvml.nvmlDeviceGetMemoryInfo(self._handle).used / (1024 * 1024)
|
|
self._gpu_util_total += utilization
|
|
self._gpu_util_peak = max(self._gpu_util_peak or 0.0, utilization)
|
|
if self._device_vram_baseline is None:
|
|
self._device_vram_baseline = device_vram
|
|
self._device_vram_peak = max(self._device_vram_peak or 0.0, device_vram)
|
|
snapshot["gpu_utilization_percent"] = utilization
|
|
snapshot["device_vram_mb"] = device_vram
|
|
try:
|
|
power = self._nvml.nvmlDeviceGetPowerUsage(self._handle) / 1000.0
|
|
self._power_total += power
|
|
self._power_samples += 1
|
|
self._power_peak = max(self._power_peak or 0.0, power)
|
|
except Exception:
|
|
pass
|
|
except Exception as error:
|
|
self._warn_once(f"GPU sampling stopped: {type(error).__name__}")
|
|
self._handle = None
|
|
self._sample_count += 1
|
|
now = time.perf_counter()
|
|
if emit_progress and now - self._last_progress >= self.progress_interval_seconds:
|
|
self._last_progress = now
|
|
try:
|
|
self.progress(snapshot)
|
|
except Exception:
|
|
pass
|
|
|
|
def _setup_psutil(self) -> None:
|
|
try:
|
|
import psutil
|
|
|
|
self._psutil = psutil
|
|
self._process = psutil.Process(os.getpid())
|
|
self._process.cpu_percent(None)
|
|
psutil.cpu_percent(None)
|
|
except Exception as error:
|
|
self._warn_once(f"CPU/RAM sampling unavailable: {type(error).__name__}")
|
|
|
|
def _setup_nvml(self) -> None:
|
|
try:
|
|
import pynvml
|
|
|
|
pynvml.nvmlInit()
|
|
self._nvml = pynvml
|
|
self._handle, mapped_by_pci = nvml_handle_for_torch_device(pynvml, self.device_index)
|
|
if not mapped_by_pci:
|
|
self._warn_once("NVML used device-index fallback")
|
|
except Exception as error:
|
|
self._warn_once(f"GPU sampling unavailable: {type(error).__name__}")
|
|
self._handle = None
|
|
|
|
def _warn_once(self, warning: str) -> None:
|
|
if warning not in self.warnings:
|
|
self.warnings.append(warning)
|
|
|
|
|
|
def public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
key: value
|
|
for key, value in record.items()
|
|
if key not in {"text_path", "json_path"}
|
|
}
|
|
|
|
|
|
def selected_device_index(instance: Any) -> int:
|
|
candidates = (
|
|
getattr(getattr(instance, "model_data", None), "device_set", None),
|
|
getattr(instance, "device", None),
|
|
)
|
|
for candidate in candidates:
|
|
text = str(candidate or "")
|
|
if text.isdigit():
|
|
return int(text)
|
|
if text.startswith("cuda:") and text.split(":", 1)[1].isdigit():
|
|
return int(text.split(":", 1)[1])
|
|
return 0
|
|
|
|
|
|
def read_input_metadata(value: Any) -> dict[str, Any]:
|
|
if not value:
|
|
return {"name": None, "duration_seconds": None, "sample_rate": None, "channels": None}
|
|
path = Path(str(value))
|
|
metadata = {"name": path.name, "duration_seconds": None, "sample_rate": None, "channels": None}
|
|
try:
|
|
import soundfile as sf
|
|
|
|
info = sf.info(path)
|
|
metadata.update({
|
|
"duration_seconds": info.duration,
|
|
"sample_rate": info.samplerate,
|
|
"channels": info.channels,
|
|
})
|
|
except Exception:
|
|
pass
|
|
return metadata
|
|
|
|
|
|
def model_cache_state(model_cache: Any, model_path: Path, precision: str) -> str:
|
|
if model_cache is None:
|
|
return "unknown"
|
|
normalized = os.path.normpath(os.path.abspath(str(model_path)))
|
|
entry = getattr(model_cache, "cache", {}).get(normalized)
|
|
if not entry:
|
|
return "cold"
|
|
cached_precision = entry.get("precision_mode")
|
|
return "warm" if cached_precision in (None, precision) else "cold-precision-change"
|
|
|
|
|
|
def effective_runtime(namespace: dict[str, Any], instance: Any) -> dict[str, Any]:
|
|
model_path = Path(str(getattr(instance, "model_path", "")))
|
|
model_cache = namespace.get("MODEL_CACHE")
|
|
normalized = os.path.normpath(os.path.abspath(str(model_path)))
|
|
entry = getattr(model_cache, "cache", {}).get(normalized, {}) if model_cache is not None else {}
|
|
model = entry.get("model_obj")
|
|
if model is None:
|
|
model = getattr(instance, "model_run", None)
|
|
if model is None:
|
|
model = getattr(instance, "demucs", None)
|
|
if hasattr(model, "get_inputs"):
|
|
input_type = model.get_inputs()[0].type
|
|
return {
|
|
"effective_precision": "fp16" if "float16" in input_type else "fp32",
|
|
"execution_provider": list(model.get_providers()),
|
|
"onnx_input_type": input_type,
|
|
}
|
|
try:
|
|
import torch
|
|
|
|
if isinstance(model, torch.nn.Module):
|
|
dtypes = {parameter.dtype for parameter in model.parameters() if parameter.is_floating_point()}
|
|
mapping = {torch.float16: "fp16", torch.bfloat16: "bf16", torch.float32: "fp32"}
|
|
effective = {mapping[dtype] for dtype in dtypes if dtype in mapping}
|
|
return {
|
|
"effective_precision": effective.pop() if len(effective) == 1 else "unknown",
|
|
"execution_provider": ["CUDA"] if str(getattr(instance, "device", "")).startswith("cuda") else ["CPU"],
|
|
}
|
|
except Exception:
|
|
pass
|
|
providers = []
|
|
for provider in getattr(instance, "run_type", []) or []:
|
|
providers.append(str(provider[0] if isinstance(provider, tuple) else provider))
|
|
return {"effective_precision": "unknown", "execution_provider": providers}
|
|
|
|
|
|
def output_metadata(instance: Any) -> dict[str, Any]:
|
|
export_path = Path(str(getattr(instance, "export_path", "")))
|
|
base = str(getattr(instance, "audio_file_base", ""))
|
|
if not export_path.is_dir() or not base:
|
|
return {"count": 0, "total_size_mb": 0.0, "validation": "unavailable"}
|
|
files = sorted(path for path in export_path.iterdir() if path.is_file() and path.name.startswith(base))
|
|
readable_audio = 0
|
|
invalid_audio = 0
|
|
durations = []
|
|
for path in files:
|
|
if path.suffix.lower() not in {".wav", ".flac", ".mp3"}:
|
|
continue
|
|
try:
|
|
import soundfile as sf
|
|
|
|
info = sf.info(path)
|
|
if info.frames > 0 and info.samplerate > 0 and info.channels > 0:
|
|
readable_audio += 1
|
|
durations.append(info.duration)
|
|
else:
|
|
invalid_audio += 1
|
|
except Exception:
|
|
invalid_audio += 1
|
|
validation = "passed" if readable_audio and invalid_audio == 0 else "failed" if invalid_audio else "unavailable"
|
|
return {
|
|
"count": len(files),
|
|
"readable_audio_count": readable_audio,
|
|
"invalid_audio_count": invalid_audio,
|
|
"duration_seconds": durations,
|
|
"total_size_mb": sum(path.stat().st_size for path in files) / (1024 * 1024),
|
|
"validation": validation,
|
|
}
|
|
|
|
|
|
def reset_torch_peak_memory(device_index: int, warnings: list[str]) -> None:
|
|
try:
|
|
import torch
|
|
|
|
if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count():
|
|
torch.cuda.reset_peak_memory_stats(device_index)
|
|
except Exception as error:
|
|
warnings.append(f"Torch peak reset unavailable: {type(error).__name__}")
|
|
|
|
|
|
def torch_peak_memory(device_index: int) -> dict[str, float | None]:
|
|
try:
|
|
import torch
|
|
|
|
if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count():
|
|
return {
|
|
"torch_peak_allocated_vram_mb": torch.cuda.max_memory_allocated(device_index) / (1024 * 1024),
|
|
"torch_peak_reserved_vram_mb": torch.cuda.max_memory_reserved(device_index) / (1024 * 1024),
|
|
}
|
|
except Exception:
|
|
pass
|
|
return {"torch_peak_allocated_vram_mb": None, "torch_peak_reserved_vram_mb": None}
|
|
|
|
|
|
def synchronize_cuda(device_index: int) -> None:
|
|
try:
|
|
import torch
|
|
|
|
if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count():
|
|
torch.cuda.synchronize(device_index)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def ordinal(value: int) -> str:
|
|
if 10 <= value % 100 <= 20:
|
|
suffix = "th"
|
|
else:
|
|
suffix = {1: "st", 2: "nd", 3: "rd"}.get(value % 10, "th")
|
|
return f"{value}{suffix}"
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def format_seconds(value: Any) -> str:
|
|
return "unavailable" if value is None else f"{float(value):.6f} s"
|
|
|
|
|
|
def format_mb(value: Any) -> str:
|
|
return "unavailable" if value is None else f"{float(value):.2f} MiB"
|
|
|
|
|
|
def format_percent(value: Any) -> str:
|
|
return "unavailable" if value is None else f"{float(value):.2f}%"
|