diff --git a/.gitignore b/.gitignore index e1ce42d..d4c5692 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,8 @@ __pycache__/ venv/ .venv/ .agents/ +benchmark_cache/ +benchmark_results/ uvr_run.log data.pkl -benchmark_cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84b1f65..2a76d26 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,10 @@ # Contributions -Runtime benchmarking is recorded automatically for each top-level inference started from the UVR GUI. The implementation and output format are documented in `benchmarks/README.md`. +The CLI and performance benchmarking suite is located in `uvr_cli/`. The implementation and output format are documented in `uvr_cli/README.md`. -Do not commit private audio, model files, generated stems, or local `benchmark_cache/` logs. Before submitting recorder changes, run: +Do not commit private audio, model files, generated stems, or local `benchmark_results/` logs. Before submitting benchmark changes, run: ```powershell -venv\Scripts\python.exe -m unittest tests.test_gui_runtime_recorder tests.test_in_memory_model_cache -v +venv\Scripts\python.exe -m unittest tests.test_uvr_cli tests.test_in_memory_model_cache -v ``` + diff --git a/UVR.py b/UVR.py index 975b917..db60e3a 100644 --- a/UVR.py +++ b/UVR.py @@ -3358,7 +3358,7 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): model_precision_Label.grid(pady=MENU_PADDING_2) model_precision_Option = ComboBoxMenu(settings_menu_format_Frame, textvariable=self.model_precision_var, values=MODEL_PRECISION_OPTIONS, width=GEN_SETTINGS_WIDTH+1) model_precision_Option.grid(padx=20,pady=MENU_PADDING_1) - self.help_hints(model_precision_Label, text="Select precision preset:\n• Ultra Quality: FP32 (Max precision, mastering)\n• Balanced: BF16 (Recommended for general use)\n• Performance: FP16 (Fast, lower VRAM)") + self.help_hints(model_precision_Label, text="Select precision preset:\n• Ultra Quality: FP32 (Max precision, mastering)\n• Performance: FP16 (Fast, lower VRAM)") # if is_choose_arch: # is_use_opencl_Option = ttk.Checkbutton(settings_menu_format_Frame, diff --git a/benchmarks/README.md b/benchmarks/README.md deleted file mode 100644 index 8800513..0000000 --- a/benchmarks/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# GUI Runtime Benchmark Recorder - -UVR records runtime benchmark metadata automatically whenever the GUI backend enters a top-level `seperate()` call after **Start Processing**. There is no benchmark CLI, case matrix, profiler command, result packager, or upload workflow. - -Supported production backends: - -- VR Architecture -- MDX-Net ONNX -- MDX PyTorch/MDXC -- Demucs - -## Output - -Each application session writes progressively to: - -```text -benchmark_cache/gui_runtime/latest.txt -benchmark_cache/gui_runtime//benchmark.log -benchmark_cache/gui_runtime//inference__.log -benchmark_cache/gui_runtime//inference__.json -``` - -Every top-level audio job receives a separate block: - -```text -1st Inference: -Model: -Model round: 1 -OS: -... -Total-time: -======= -``` - -`Model round` increments while model path and requested precision remain unchanged. Changing either value resets the round to one. Nested secondary separators remain part of the parent pipeline and do not create overlapping benchmark records. - -The recorder stores model identity/hash, backend, requested and effective precision, execution provider, input metadata, cache state, stage timings, total time, CPU/RAM, GPU utilization, device/Torch VRAM, power when available, output validation, warnings, and failure state. Start, progress, stage, warning, and completion data are flushed incrementally. - -Recorder failures never cancel audio processing. When instrumentation setup fails, UVR continues without instrumentation and closes any record already opened with its final status and total time. - -## Configuration - -- Set `UVR_GUI_BENCHMARK=0` before launch to disable recording. -- Set `UVR_GUI_BENCHMARK_DIR=` to override the local output directory. -- Keep `nvidia-ml-py` installed for NVIDIA utilization, VRAM, driver, and power metadata. Missing NVML support produces a warning rather than stopping inference. - -## Tests - -```powershell -venv\Scripts\python.exe -m unittest tests.test_gui_runtime_recorder tests.test_in_memory_model_cache -v -``` - -`UVR.py` is not modified by the recorder. Hooks are installed from `separate.py` after all separator classes are defined. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py deleted file mode 100644 index 91eb2d5..0000000 --- a/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Automatic per-inference runtime recording for the UVR GUI.""" diff --git a/benchmarks/gui_runtime_recorder.py b/benchmarks/gui_runtime_recorder.py deleted file mode 100644 index 94be22e..0000000 --- a/benchmarks/gui_runtime_recorder.py +++ /dev/null @@ -1,1048 +0,0 @@ -from __future__ import annotations - -import functools -import hashlib -import json -import os -import platform -import re -import threading -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Callable - -FALSE_VALUES = {"0", "false", "no", "off", "disabled"} -SEPARATOR_BACKENDS = { - "SeperateVR": "VR", - "SeperateMDX": "MDX-Net ONNX", - "SeperateMDXC": "MDX PyTorch", - "SeperateDemucs": "Demucs", -} -PROJECT_ROOT = Path(__file__).resolve().parents[1] -DEFAULT_CACHE_ROOT = PROJECT_ROOT / "benchmark_cache" -RECORDER_VERSION = "1.0.0" - -_WINDOWS_PATH = re.compile(r"(?i)(? str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(chunk_size), b""): - digest.update(chunk) - return digest.hexdigest() - - -def slugify(value: str, fallback: str = "unknown") -> str: - slug = re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip("-") - return slug or fallback - - -def sanitize_text(value: str) -> str: - sanitized = _ANSI_ESCAPE.sub("", str(value)) - sanitized = _WINDOWS_PATH.sub("", sanitized) - sanitized = _UNC_PATH.sub("", sanitized) - sanitized = _ABSOLUTE_UNIX_PATH.sub("", sanitized) - return _SECRET_ASSIGNMENT.sub(lambda match: f"{match.group(1)}=", sanitized) - - -def sanitize_data(value: Any) -> Any: - if isinstance(value, dict): - return { - key: "" if _SECRET_KEY.search(str(key)) else sanitize_data(item) - for key, item in value.items() - } - if isinstance(value, (list, tuple)): - return [sanitize_data(item) for item in value] - if isinstance(value, str): - return sanitize_text(value) - return value - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(sanitize_data(value), indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def collect_environment(device_index: int = 0) -> dict[str, Any]: - hardware: dict[str, Any] = { - "os": platform.platform(), - "cpu": platform.processor() or platform.machine() or "unknown", - "system_ram_gb": _system_ram_gb(), - "gpu_name": None, - "gpu_vram_mb": None, - "driver_version": None, - "compute_capability": None, - "form_factor": "unknown", - "power_limit_w": None, - } - software: dict[str, Any] = { - "python_version": platform.python_version(), - "torch_version": None, - "cuda_runtime_version": None, - "cudnn_version": None, - "torch_cuda_available": False, - "onnxruntime_version": None, - "onnxruntime_providers": [], - } - warnings: list[str] = [] - try: - import torch - - software["torch_version"] = torch.__version__ - software["cuda_runtime_version"] = torch.version.cuda - software["cudnn_version"] = torch.backends.cudnn.version() - software["torch_cuda_available"] = bool(torch.cuda.is_available()) - if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count(): - properties = torch.cuda.get_device_properties(device_index) - hardware["gpu_name"] = properties.name - hardware["gpu_vram_mb"] = properties.total_memory / (1024 * 1024) - hardware["compute_capability"] = f"{properties.major}.{properties.minor}" - hardware["form_factor"] = "laptop" if "laptop" in properties.name.lower() else "unknown" - except Exception as error: - warnings.append(f"PyTorch metadata unavailable: {type(error).__name__}") - try: - import onnxruntime as ort - - software["onnxruntime_version"] = ort.__version__ - software["onnxruntime_providers"] = list(ort.get_available_providers()) - except Exception as error: - warnings.append(f"ONNX Runtime metadata unavailable: {type(error).__name__}") - _add_nvml_metadata(hardware, device_index, warnings) - return sanitize_data({ - "recorder_version": RECORDER_VERSION, - "timestamp_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), - "hardware": hardware, - "software": software, - "warnings": warnings, - }) - - -def _system_ram_gb() -> float | None: - try: - import psutil - - return psutil.virtual_memory().total / (1024 ** 3) - except Exception: - return None - - -def _add_nvml_metadata(hardware: dict[str, Any], device_index: int, warnings: list[str]) -> None: - try: - import pynvml - - pynvml.nvmlInit() - try: - handle, mapped_by_pci = nvml_handle_for_torch_device(pynvml, device_index) - if not mapped_by_pci: - warnings.append("NVML metadata used device-index fallback") - name = pynvml.nvmlDeviceGetName(handle) - hardware["gpu_name"] = name.decode(errors="replace") if isinstance(name, bytes) else str(name) - hardware["gpu_vram_mb"] = pynvml.nvmlDeviceGetMemoryInfo(handle).total / (1024 * 1024) - driver = pynvml.nvmlSystemGetDriverVersion() - hardware["driver_version"] = driver.decode() if isinstance(driver, bytes) else str(driver) - try: - hardware["power_limit_w"] = pynvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000.0 - except Exception: - pass - finally: - pynvml.nvmlShutdown() - except Exception as error: - warnings.append(f"NVML metadata unavailable: {type(error).__name__}") - - -def nvml_handle_for_torch_device(nvml: Any, device_index: int) -> tuple[Any, bool]: - try: - import torch - - properties = torch.cuda.get_device_properties(device_index) - candidates = ( - f"{int(properties.pci_domain_id):08x}:{int(properties.pci_bus_id):02x}:{int(properties.pci_device_id):02x}.0", - f"{int(properties.pci_domain_id):04x}:{int(properties.pci_bus_id):02x}:{int(properties.pci_device_id):02x}.0", - ) - for candidate in candidates: - for value in (candidate, candidate.encode("ascii")): - try: - return nvml.nvmlDeviceGetHandleByPciBusId(value), True - except Exception: - continue - except Exception: - pass - return nvml.nvmlDeviceGetHandleByIndex(device_index), False - - -def install_gui_runtime_benchmark_hooks( - namespace: dict[str, Any], - recorder: "GUIRuntimeBenchmarkRecorder | None" = None, -) -> "GUIRuntimeBenchmarkRecorder | None": - if os.environ.get("UVR_GUI_BENCHMARK", "1").strip().lower() in FALSE_VALUES: - return None - existing = namespace.get("_GUI_RUNTIME_BENCHMARK_RECORDER") - runtime_recorder = recorder or existing or GUIRuntimeBenchmarkRecorder(namespace) - namespace["_GUI_RUNTIME_BENCHMARK_RECORDER"] = runtime_recorder - for class_name, backend in SEPARATOR_BACKENDS.items(): - separator_class = namespace.get(class_name) - if separator_class is None: - continue - original = separator_class.seperate - if getattr(original, "_uvr_gui_benchmark_wrapped", False): - continue - - @functools.wraps(original) - def wrapped(instance, *args, __original=original, __backend=backend, **kwargs): - return runtime_recorder.run(instance, __backend, __original, args, kwargs) - - wrapped._uvr_gui_benchmark_wrapped = True - wrapped._uvr_gui_benchmark_original = original - separator_class.seperate = wrapped - return runtime_recorder - - -class GUIRuntimeBenchmarkRecorder: - def __init__( - self, - namespace: dict[str, Any], - output_root: Path | None = None, - sampler_factory: Callable[..., Any] | None = None, - environment_collector: Callable[[int], dict[str, Any]] = collect_environment, - progress_interval_seconds: float = 5.0, - ): - self.namespace = namespace - configured_root = os.environ.get("UVR_GUI_BENCHMARK_DIR") - selected_root = Path(configured_root).expanduser() if configured_root else ( - output_root or DEFAULT_CACHE_ROOT / "gui_runtime" - ) - self.output_root = selected_root.resolve() - self.sampler_factory = sampler_factory or RuntimeResourceSampler - self.environment_collector = environment_collector - self.progress_interval_seconds = max(float(progress_interval_seconds), 1.0) - self._lock = threading.RLock() - self._thread_state = threading.local() - self._inference_index = 0 - self._last_model_signature: tuple[str, str] | None = None - self._model_round = 0 - self._environment_cache: dict[int, dict[str, Any]] = {} - self._model_hash_cache: dict[tuple[str, int, int], str] = {} - self._session_directory: Path | None = None - self._session_log: Path | None = None - - def run( - self, - instance: Any, - backend: str, - original: Callable[..., Any], - args: tuple[Any, ...], - kwargs: dict[str, Any], - ) -> Any: - depth = getattr(self._thread_state, "depth", 0) - if depth: - return original(instance, *args, **kwargs) - self._thread_state.depth = depth + 1 - try: - return self._run_top_level(instance, backend, original, args, kwargs) - finally: - self._thread_state.depth = depth - - def _run_top_level( - self, - instance: Any, - backend: str, - original: Callable[..., Any], - args: tuple[Any, ...], - kwargs: dict[str, Any], - ) -> Any: - started = time.perf_counter() - record = None - instrumentor = None - sampler = None - try: - record = self._begin_record(instance, backend) - instrumentor = RuntimeStageInstrumentor( - self.namespace, - instance, - record["device_index"], - lambda name, value: self._record_stage(record, name, value), - ) - sampler = self.sampler_factory( - device_index=record["device_index"], - progress_interval_seconds=self.progress_interval_seconds, - progress=lambda snapshot: self._record_progress(record, snapshot), - ) - instrumentor.install() - sampler.start() - except Exception as error: - setup_warning = f"Recorder setup failed: {type(error).__name__}" - cleanup_warnings = [] - if instrumentor is not None: - try: - instrumentor.restore() - except Exception as cleanup_error: - cleanup_warnings.append( - f"Recorder hook cleanup failed: {type(cleanup_error).__name__}" - ) - if sampler is not None: - try: - sampler.stop() - except Exception as cleanup_error: - cleanup_warnings.append( - f"Recorder sampler cleanup failed: {type(cleanup_error).__name__}" - ) - warnings = [setup_warning, *cleanup_warnings] - if record is not None: - for warning in warnings: - self._record_warning(record, warning) - self._notify_gui(instance, f"[Benchmark warning] {setup_warning}; processing will continue") - failure: BaseException | None = None - try: - return original(instance, *args, **kwargs) - except BaseException as backend_error: - failure = backend_error - raise - finally: - synchronize_cuda(selected_device_index(instance)) - if record is not None: - self._finish_record_safely( - record, - instance, - time.perf_counter() - started, - {"warnings": warnings}, - failure, - ) - - failure: BaseException | None = None - synchronize_cuda(record["device_index"]) - try: - return original(instance, *args, **kwargs) - except BaseException as error: - failure = error - raise - finally: - synchronize_cuda(record["device_index"]) - total_seconds = time.perf_counter() - started - resources: dict[str, Any] = {} - recorder_warnings = [] - try: - instrumentor.finish() - except Exception as recorder_error: - recorder_warnings.append( - f"Stage finalization failed: {type(recorder_error).__name__}" - ) - try: - resources = sampler.stop() or {} - except Exception as recorder_error: - recorder_warnings.append( - f"Resource sampler finalization failed: {type(recorder_error).__name__}" - ) - try: - instrumentor.restore() - except Exception as recorder_error: - recorder_warnings.append( - f"Recorder hook cleanup failed: {type(recorder_error).__name__}" - ) - resources.setdefault("warnings", []).extend(recorder_warnings) - self._finish_record_safely( - record, - instance, - total_seconds, - resources, - failure, - ) - - def _begin_record(self, instance: Any, backend: str) -> dict[str, Any]: - model_path = Path(str(getattr(instance, "model_path", "unknown-model"))) - model_name = model_path.name or str(getattr(instance, "model_basename", "unknown-model")) - precision = str(getattr(getattr(instance, "model_data", None), "model_precision", "unknown")) - device_index = selected_device_index(instance) - normalized_model = os.path.normcase(os.path.abspath(str(model_path))) - signature = (normalized_model, precision) - with self._lock: - self._inference_index += 1 - inference_index = self._inference_index - if signature == self._last_model_signature: - self._model_round += 1 - else: - self._last_model_signature = signature - self._model_round = 1 - model_round = self._model_round - self._ensure_session() - - environment = self._environment(device_index) - input_metadata = read_input_metadata(getattr(instance, "audio_file", None)) - model_hash = self._model_hash(model_path) - cache_state = model_cache_state(self.namespace.get("MODEL_CACHE"), model_path, precision) - file_slug = slugify(f"{model_path.stem}-{precision}") - text_path = self._session_directory / f"inference_{inference_index:04d}_{file_slug}.log" - json_path = self._session_directory / f"inference_{inference_index:04d}_{file_slug}.json" - record = { - "inference_index": inference_index, - "inference_label": f"{ordinal(inference_index)} Inference", - "model": model_name, - "model_sha256": model_hash, - "model_round": model_round, - "backend": backend, - "precision": precision, - "device_index": device_index, - "cache_state": cache_state, - "os": environment.get("hardware", {}).get("os") or platform.platform(), - "environment": environment, - "input": input_metadata, - "started_utc": utc_now(), - "status": "running", - "total_time_seconds": None, - "stages": {}, - "resources": {}, - "runtime": {}, - "outputs": {}, - "warnings": [], - "error": None, - "text_path": text_path, - "json_path": json_path, - } - header = [ - f"{record['inference_label']}:", - f"Model: {model_name}", - f"Model round: {model_round}", - f"OS: {record['os']}", - f"Backend: {backend}", - f"Precision: {precision}", - f"Model cache: {cache_state}", - f"Model SHA-256: {model_hash or 'unavailable'}", - f"Input: {input_metadata.get('name') or 'unavailable'}", - f"Input duration: {format_seconds(input_metadata.get('duration_seconds'))}", - f"Started UTC: {record['started_utc']}", - "Status: RUNNING", - ] - text_path.write_text("\n".join(header) + "\n", encoding="utf-8") - with self._session_log.open("a", encoding="utf-8") as stream: - stream.write("\n".join(header) + "\n") - self._checkpoint(record) - relative_log = self._session_log.relative_to(self.output_root.parent) - self._notify_gui(instance, f"[Benchmark] Recording {record['inference_label']} -> {relative_log}") - return record - - def _finish_record( - self, - record: dict[str, Any], - instance: Any, - total_seconds: float, - resources: dict[str, Any], - failure: BaseException | None, - ) -> None: - record["status"] = "completed" if failure is None else "failed" - record["total_time_seconds"] = total_seconds - record["ended_utc"] = utc_now() - record["resources"] = resources - for warning in resources.get("warnings", []): - self._record_warning(record, warning) - try: - record["runtime"] = effective_runtime(self.namespace, instance) - except Exception as error: - record["runtime"] = {} - self._record_warning(record, f"Runtime metadata unavailable: {type(error).__name__}") - try: - record["outputs"] = output_metadata(instance) - except Exception as error: - record["outputs"] = {"count": 0, "validation": "unavailable"} - self._record_warning(record, f"Output metadata unavailable: {type(error).__name__}") - if failure is not None: - record["error"] = f"{type(failure).__name__}: {sanitize_text(str(failure))}" - final_lines = [ - f"Effective precision: {record['runtime'].get('effective_precision') or 'unavailable'}", - f"Execution provider: {', '.join(record['runtime'].get('execution_provider') or []) or 'unavailable'}", - f"Peak process RAM: {format_mb(resources.get('peak_process_ram_mb'))}", - f"Peak device VRAM: {format_mb(resources.get('peak_device_vram_mb'))}", - f"Peak Torch allocated VRAM: {format_mb(resources.get('torch_peak_allocated_vram_mb'))}", - f"Peak Torch reserved VRAM: {format_mb(resources.get('torch_peak_reserved_vram_mb'))}", - f"Average GPU utilization: {format_percent(resources.get('average_gpu_utilization_percent'))}", - f"Peak GPU utilization: {format_percent(resources.get('peak_gpu_utilization_percent'))}", - f"Average process CPU: {format_percent(resources.get('average_process_cpu_percent'))}", - f"Peak process CPU: {format_percent(resources.get('peak_process_cpu_percent'))}", - f"Output files: {record['outputs'].get('count', 0)}", - f"Output validation: {record['outputs'].get('validation', 'unavailable')}", - f"Status: {record['status'].upper()}", - ] - if record["error"]: - final_lines.append(f"Error: {record['error']}") - final_lines.extend([ - f"Total-time: {total_seconds:.6f} s", - f"Ended UTC: {record['ended_utc']}", - "=======", - ]) - for line in final_lines: - self._append(record, line, checkpoint=False) - self._checkpoint(record) - self._notify_gui( - instance, - f"[Benchmark] {record['inference_label']} {record['status']} in {total_seconds:.3f}s", - ) - - def _finish_record_safely( - self, - record: dict[str, Any], - instance: Any, - total_seconds: float, - resources: dict[str, Any], - failure: BaseException | None, - ) -> None: - try: - self._finish_record(record, instance, total_seconds, resources, failure) - except Exception as recorder_error: - self._notify_gui( - instance, - f"[Benchmark warning] Recorder finalization failed: {type(recorder_error).__name__}", - ) - - def _record_stage(self, record: dict[str, Any], name: str, value: float) -> None: - with self._lock: - is_cumulative = name in record["stages"] - record["stages"][name] = value - label = f"Stage {name} (cumulative)" if is_cumulative else f"Stage {name}" - self._append(record, f"{label}: {value:.6f} s") - - def _record_progress(self, record: dict[str, Any], snapshot: dict[str, Any]) -> None: - with self._lock: - record["resources"].update(snapshot) - line = ( - f"Progress: elapsed={snapshot.get('elapsed_seconds', 0.0):.1f}s, " - f"CPU={format_percent(snapshot.get('process_cpu_percent'))}, " - f"GPU={format_percent(snapshot.get('gpu_utilization_percent'))}, " - f"device VRAM={format_mb(snapshot.get('device_vram_mb'))}" - ) - self._append(record, line) - - def _record_warning(self, record: dict[str, Any], warning: str) -> None: - cleaned = sanitize_text(str(warning)) - with self._lock: - if cleaned in record["warnings"]: - return - record["warnings"].append(cleaned) - self._append(record, f"Warning: {cleaned}") - - def _append(self, record: dict[str, Any], line: str, checkpoint: bool = True) -> None: - with self._lock: - for path in (record["text_path"], self._session_log): - with path.open("a", encoding="utf-8") as stream: - stream.write(line + "\n") - if checkpoint: - self._checkpoint(record) - - def _checkpoint(self, record: dict[str, Any]) -> None: - write_json(record["json_path"], public_record(record)) - - def _ensure_session(self) -> None: - if self._session_directory is not None: - return - session_name = datetime.now(timezone.utc).strftime("session_%Y%m%dT%H%M%SZ") + f"_{os.getpid()}" - self._session_directory = self.output_root / session_name - self._session_directory.mkdir(parents=True, exist_ok=True) - self._session_log = self._session_directory / "benchmark.log" - uvr_path = Path(__file__).resolve().parents[1] / "UVR.py" - uvr_hash = sha256_file(uvr_path) if uvr_path.is_file() else "unavailable" - self._session_log.write_text( - "UVR GUI Backend Benchmark\n" - f"Session started UTC: {utc_now()}\n" - f"UVR.py SHA-256: {uvr_hash}\n" - "=======\n", - encoding="utf-8", - ) - self.output_root.mkdir(parents=True, exist_ok=True) - (self.output_root / "latest.txt").write_text(session_name + "\n", encoding="utf-8") - - def _environment(self, device_index: int) -> dict[str, Any]: - if device_index not in self._environment_cache: - try: - self._environment_cache[device_index] = self.environment_collector(device_index) - except Exception as error: - self._environment_cache[device_index] = { - "hardware": {"os": platform.platform()}, - "software": {}, - "warnings": [f"Environment metadata unavailable: {type(error).__name__}"], - } - return self._environment_cache[device_index] - - def _model_hash(self, model_path: Path) -> str | None: - try: - stat = model_path.stat() - key = (os.path.normcase(str(model_path.resolve())), stat.st_size, stat.st_mtime_ns) - if key not in self._model_hash_cache: - self._model_hash_cache[key] = sha256_file(model_path) - return self._model_hash_cache[key] - except Exception: - return None - - @staticmethod - def _notify_gui(instance: Any, message: str) -> None: - writer = getattr(instance, "write_to_console", None) - if not callable(writer): - return - try: - writer(message, base_text="") - except TypeError: - try: - writer(message) - except Exception: - pass - except Exception: - pass - - -class RuntimeStageInstrumentor: - def __init__( - self, - namespace: dict[str, Any], - instance: Any, - device_index: int, - stage_callback: Callable[[str, float], None], - ): - self.namespace = namespace - self.instance = instance - self.device_index = device_index - self.stage_callback = stage_callback - self.values: dict[str, float] = {} - self._restores: list[Callable[[], None]] = [] - self._model_load_started: float | None = None - - def install(self) -> None: - model_cache = self.namespace.get("MODEL_CACHE") - if model_cache is not None: - original_get = model_cache.get_model - original_set = model_cache.set_model - - def get_model(*args, **kwargs): - model = original_get(*args, **kwargs) - if model is None and self._model_load_started is None: - synchronize_cuda(self.device_index) - self._model_load_started = time.perf_counter() - return model - - def set_model(*args, **kwargs): - result = original_set(*args, **kwargs) - self._finish_model_loading() - return result - - self._patch_attribute(model_cache, "get_model", get_model) - self._patch_attribute(model_cache, "set_model", set_model) - - backend_name = type(self.instance).__name__ - if backend_name in {"SeperateMDX", "SeperateMDXC", "SeperateDemucs"}: - self._patch_namespace("prepare_mix", self._timed(self.namespace["prepare_mix"], "preprocessing")) - if backend_name in {"SeperateMDX", "SeperateMDXC"}: - self._patch_attribute(self.instance, "demix", self._timed(self.instance.demix, "inference")) - elif backend_name == "SeperateDemucs": - self._patch_attribute( - self.instance, - "demix_demucs", - self._timed(self.instance.demix_demucs, "inference"), - ) - elif backend_name == "SeperateVR": - self._patch_attribute(self.instance, "loading_mix", self._timed(self.instance.loading_mix, "preprocessing")) - self._patch_attribute(self.instance, "inference_vr", self._timed(self.instance.inference_vr, "inference")) - if hasattr(self.instance, "write_audio"): - self._patch_attribute(self.instance, "write_audio", self._timed(self.instance.write_audio, "output_writing")) - - def finish(self) -> None: - self._finish_model_loading() - - def restore(self) -> None: - errors = [] - for restore in reversed(self._restores): - try: - restore() - except Exception as error: - errors.append(error) - self._restores.clear() - if errors: - raise RuntimeError(f"failed to restore {len(errors)} benchmark hook(s)") from errors[0] - - def _timed(self, function: Callable[..., Any], stage_name: str) -> Callable[..., Any]: - @functools.wraps(function) - def wrapped(*args, **kwargs): - synchronize_cuda(self.device_index) - started = time.perf_counter() - try: - return function(*args, **kwargs) - finally: - synchronize_cuda(self.device_index) - elapsed = time.perf_counter() - started - self.values[stage_name] = self.values.get(stage_name, 0.0) + elapsed - self.stage_callback(stage_name, self.values[stage_name]) - - return wrapped - - def _finish_model_loading(self) -> None: - if self._model_load_started is None: - return - synchronize_cuda(self.device_index) - elapsed = time.perf_counter() - self._model_load_started - self._model_load_started = None - self.values["model_loading"] = self.values.get("model_loading", 0.0) + elapsed - self.stage_callback("model_loading", self.values["model_loading"]) - - def _patch_namespace(self, name: str, replacement: Any) -> None: - original = self.namespace[name] - self.namespace[name] = replacement - self._restores.append(lambda: self.namespace.__setitem__(name, original)) - - def _patch_attribute(self, owner: Any, name: str, replacement: Any) -> None: - owner_dict = getattr(owner, "__dict__", {}) - had_local_value = name in owner_dict - local_value = owner_dict.get(name) - setattr(owner, name, replacement) - - def restore() -> None: - if had_local_value: - setattr(owner, name, local_value) - else: - try: - delattr(owner, name) - except AttributeError: - pass - - self._restores.append(restore) - - -class RuntimeResourceSampler: - def __init__( - self, - device_index: int = 0, - interval_seconds: float = 0.25, - progress_interval_seconds: float = 5.0, - progress: Callable[[dict[str, Any]], None] | None = None, - ): - self.device_index = device_index - self.interval_seconds = max(float(interval_seconds), 0.05) - self.progress_interval_seconds = max(float(progress_interval_seconds), self.interval_seconds) - self.progress = progress or (lambda snapshot: None) - self._stop = threading.Event() - self._thread: threading.Thread | None = None - self._started = 0.0 - self._last_progress = 0.0 - self._psutil: Any = None - self._process: Any = None - self._nvml: Any = None - self._handle: Any = None - self._sample_count = 0 - self._process_cpu_total = 0.0 - self._process_cpu_peak: float | None = None - self._system_cpu_total = 0.0 - self._system_cpu_peak: float | None = None - self._process_ram_peak: float | None = None - self._gpu_util_total = 0.0 - self._gpu_util_peak: float | None = None - self._device_vram_baseline: float | None = None - self._device_vram_peak: float | None = None - self._power_total = 0.0 - self._power_peak: float | None = None - self._power_samples = 0 - self.warnings: list[str] = [] - - def start(self) -> None: - self._setup_psutil() - self._setup_nvml() - reset_torch_peak_memory(self.device_index, self.warnings) - self._started = time.perf_counter() - self._last_progress = self._started - self._sample(emit_progress=False) - self._stop.clear() - self._thread = threading.Thread(target=self._run, name="uvr-gui-benchmark-sampler", daemon=True) - self._thread.start() - - def stop(self) -> dict[str, Any]: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=max(1.0, self.interval_seconds * 4)) - self._sample(emit_progress=False) - if self._nvml is not None: - try: - self._nvml.nvmlShutdown() - except Exception: - pass - torch_memory = torch_peak_memory(self.device_index) - count = max(self._sample_count, 1) - return { - "sampling_interval_ms": self.interval_seconds * 1000.0, - "average_process_cpu_percent": self._process_cpu_total / count if self._process_cpu_peak is not None else None, - "peak_process_cpu_percent": self._process_cpu_peak, - "average_system_cpu_percent": self._system_cpu_total / count if self._system_cpu_peak is not None else None, - "peak_system_cpu_percent": self._system_cpu_peak, - "peak_process_ram_mb": self._process_ram_peak, - "average_gpu_utilization_percent": self._gpu_util_total / count if self._gpu_util_peak is not None else None, - "peak_gpu_utilization_percent": self._gpu_util_peak, - "baseline_device_vram_mb": self._device_vram_baseline, - "peak_device_vram_mb": self._device_vram_peak, - "average_power_w": self._power_total / self._power_samples if self._power_samples else None, - "peak_power_w": self._power_peak, - **torch_memory, - "warnings": list(dict.fromkeys(self.warnings)), - } - - def _run(self) -> None: - while not self._stop.wait(self.interval_seconds): - self._sample(emit_progress=True) - - def _sample(self, emit_progress: bool) -> None: - snapshot: dict[str, Any] = { - "elapsed_seconds": max(0.0, time.perf_counter() - self._started) if self._started else 0.0, - "process_cpu_percent": None, - "gpu_utilization_percent": None, - "device_vram_mb": None, - } - if self._process is not None: - try: - process_cpu = float(self._process.cpu_percent(None)) - system_cpu = float(self._psutil.cpu_percent(None)) - process_ram = self._process.memory_info().rss / (1024 * 1024) - self._process_cpu_total += process_cpu - self._process_cpu_peak = max(self._process_cpu_peak or 0.0, process_cpu) - self._system_cpu_total += system_cpu - self._system_cpu_peak = max(self._system_cpu_peak or 0.0, system_cpu) - self._process_ram_peak = max(self._process_ram_peak or 0.0, process_ram) - snapshot["process_cpu_percent"] = process_cpu - except Exception as error: - self._warn_once(f"CPU/RAM sampling stopped: {type(error).__name__}") - self._process = None - if self._handle is not None: - try: - utilization = float(self._nvml.nvmlDeviceGetUtilizationRates(self._handle).gpu) - device_vram = self._nvml.nvmlDeviceGetMemoryInfo(self._handle).used / (1024 * 1024) - self._gpu_util_total += utilization - self._gpu_util_peak = max(self._gpu_util_peak or 0.0, utilization) - if self._device_vram_baseline is None: - self._device_vram_baseline = device_vram - self._device_vram_peak = max(self._device_vram_peak or 0.0, device_vram) - snapshot["gpu_utilization_percent"] = utilization - snapshot["device_vram_mb"] = device_vram - try: - power = self._nvml.nvmlDeviceGetPowerUsage(self._handle) / 1000.0 - self._power_total += power - self._power_samples += 1 - self._power_peak = max(self._power_peak or 0.0, power) - except Exception: - pass - except Exception as error: - self._warn_once(f"GPU sampling stopped: {type(error).__name__}") - self._handle = None - self._sample_count += 1 - now = time.perf_counter() - if emit_progress and now - self._last_progress >= self.progress_interval_seconds: - self._last_progress = now - try: - self.progress(snapshot) - except Exception: - pass - - def _setup_psutil(self) -> None: - try: - import psutil - - self._psutil = psutil - self._process = psutil.Process(os.getpid()) - self._process.cpu_percent(None) - psutil.cpu_percent(None) - except Exception as error: - self._warn_once(f"CPU/RAM sampling unavailable: {type(error).__name__}") - - def _setup_nvml(self) -> None: - try: - import pynvml - - pynvml.nvmlInit() - self._nvml = pynvml - self._handle, mapped_by_pci = nvml_handle_for_torch_device(pynvml, self.device_index) - if not mapped_by_pci: - self._warn_once("NVML used device-index fallback") - except Exception as error: - self._warn_once(f"GPU sampling unavailable: {type(error).__name__}") - self._handle = None - - def _warn_once(self, warning: str) -> None: - if warning not in self.warnings: - self.warnings.append(warning) - - -def public_record(record: dict[str, Any]) -> dict[str, Any]: - return { - key: value - for key, value in record.items() - if key not in {"text_path", "json_path"} - } - - -def selected_device_index(instance: Any) -> int: - candidates = ( - getattr(getattr(instance, "model_data", None), "device_set", None), - getattr(instance, "device", None), - ) - for candidate in candidates: - text = str(candidate or "") - if text.isdigit(): - return int(text) - if text.startswith("cuda:") and text.split(":", 1)[1].isdigit(): - return int(text.split(":", 1)[1]) - return 0 - - -def read_input_metadata(value: Any) -> dict[str, Any]: - if not value: - return {"name": None, "duration_seconds": None, "sample_rate": None, "channels": None} - path = Path(str(value)) - metadata = {"name": path.name, "duration_seconds": None, "sample_rate": None, "channels": None} - try: - import soundfile as sf - - info = sf.info(path) - metadata.update({ - "duration_seconds": info.duration, - "sample_rate": info.samplerate, - "channels": info.channels, - }) - except Exception: - pass - return metadata - - -def model_cache_state(model_cache: Any, model_path: Path, precision: str) -> str: - if model_cache is None: - return "unknown" - normalized = os.path.normpath(os.path.abspath(str(model_path))) - entry = getattr(model_cache, "cache", {}).get(normalized) - if not entry: - return "cold" - cached_precision = entry.get("precision_mode") - return "warm" if cached_precision in (None, precision) else "cold-precision-change" - - -def effective_runtime(namespace: dict[str, Any], instance: Any) -> dict[str, Any]: - model_path = Path(str(getattr(instance, "model_path", ""))) - model_cache = namespace.get("MODEL_CACHE") - normalized = os.path.normpath(os.path.abspath(str(model_path))) - entry = getattr(model_cache, "cache", {}).get(normalized, {}) if model_cache is not None else {} - model = entry.get("model_obj") - if model is None: - model = getattr(instance, "model_run", None) - if model is None: - model = getattr(instance, "demucs", None) - if hasattr(model, "get_inputs"): - input_type = model.get_inputs()[0].type - return { - "effective_precision": "fp16" if "float16" in input_type else "fp32", - "execution_provider": list(model.get_providers()), - "onnx_input_type": input_type, - } - try: - import torch - - if isinstance(model, torch.nn.Module): - dtypes = {parameter.dtype for parameter in model.parameters() if parameter.is_floating_point()} - mapping = {torch.float16: "fp16", torch.bfloat16: "bf16", torch.float32: "fp32"} - effective = {mapping[dtype] for dtype in dtypes if dtype in mapping} - return { - "effective_precision": effective.pop() if len(effective) == 1 else "unknown", - "execution_provider": ["CUDA"] if str(getattr(instance, "device", "")).startswith("cuda") else ["CPU"], - } - except Exception: - pass - providers = [] - for provider in getattr(instance, "run_type", []) or []: - providers.append(str(provider[0] if isinstance(provider, tuple) else provider)) - return {"effective_precision": "unknown", "execution_provider": providers} - - -def output_metadata(instance: Any) -> dict[str, Any]: - export_path = Path(str(getattr(instance, "export_path", ""))) - base = str(getattr(instance, "audio_file_base", "")) - if not export_path.is_dir() or not base: - return {"count": 0, "total_size_mb": 0.0, "validation": "unavailable"} - files = sorted(path for path in export_path.iterdir() if path.is_file() and path.name.startswith(base)) - readable_audio = 0 - invalid_audio = 0 - durations = [] - for path in files: - if path.suffix.lower() not in {".wav", ".flac", ".mp3"}: - continue - try: - import soundfile as sf - - info = sf.info(path) - if info.frames > 0 and info.samplerate > 0 and info.channels > 0: - readable_audio += 1 - durations.append(info.duration) - else: - invalid_audio += 1 - except Exception: - invalid_audio += 1 - validation = "passed" if readable_audio and invalid_audio == 0 else "failed" if invalid_audio else "unavailable" - return { - "count": len(files), - "readable_audio_count": readable_audio, - "invalid_audio_count": invalid_audio, - "duration_seconds": durations, - "total_size_mb": sum(path.stat().st_size for path in files) / (1024 * 1024), - "validation": validation, - } - - -def reset_torch_peak_memory(device_index: int, warnings: list[str]) -> None: - try: - import torch - - if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count(): - torch.cuda.reset_peak_memory_stats(device_index) - except Exception as error: - warnings.append(f"Torch peak reset unavailable: {type(error).__name__}") - - -def torch_peak_memory(device_index: int) -> dict[str, float | None]: - try: - import torch - - if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count(): - return { - "torch_peak_allocated_vram_mb": torch.cuda.max_memory_allocated(device_index) / (1024 * 1024), - "torch_peak_reserved_vram_mb": torch.cuda.max_memory_reserved(device_index) / (1024 * 1024), - } - except Exception: - pass - return {"torch_peak_allocated_vram_mb": None, "torch_peak_reserved_vram_mb": None} - - -def synchronize_cuda(device_index: int) -> None: - try: - import torch - - if torch.cuda.is_available() and 0 <= device_index < torch.cuda.device_count(): - torch.cuda.synchronize(device_index) - except Exception: - pass - - -def ordinal(value: int) -> str: - if 10 <= value % 100 <= 20: - suffix = "th" - else: - suffix = {1: "st", 2: "nd", 3: "rd"}.get(value % 10, "th") - return f"{value}{suffix}" - - -def utc_now() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def format_seconds(value: Any) -> str: - return "unavailable" if value is None else f"{float(value):.6f} s" - - -def format_mb(value: Any) -> str: - return "unavailable" if value is None else f"{float(value):.2f} MiB" - - -def format_percent(value: Any) -> str: - return "unavailable" if value is None else f"{float(value):.2f}%" diff --git a/gui_data/constants.py b/gui_data/constants.py index fb5b8fb..eb2fc11 100644 --- a/gui_data/constants.py +++ b/gui_data/constants.py @@ -17,13 +17,15 @@ MODEL_PRECISION_BF16 = 'Balanced (BF16)' MODEL_PRECISION_FP16 = 'Performance (FP16)' MODEL_PRECISION_OPTIONS = ( MODEL_PRECISION_FP32, - MODEL_PRECISION_BF16, MODEL_PRECISION_FP16, ) -MODEL_PRECISION_DEFAULT = MODEL_PRECISION_BF16 +MODEL_PRECISION_DEFAULT = MODEL_PRECISION_FP16 def normalize_model_precision(value): - return value if value in MODEL_PRECISION_OPTIONS else MODEL_PRECISION_DEFAULT + if value in MODEL_PRECISION_OPTIONS or value == MODEL_PRECISION_BF16: + return value + return MODEL_PRECISION_DEFAULT + #MAIN_FONT_NAME = "Century Gothic" OPT_SEPARATOR_SAVE = '─'*25 diff --git a/separate.py b/separate.py index 890d2a2..127ccba 100644 --- a/separate.py +++ b/separate.py @@ -2237,9 +2237,4 @@ def loading_mix(X, mp): return X_spec -try: - from benchmarks.gui_runtime_recorder import install_gui_runtime_benchmark_hooks - install_gui_runtime_benchmark_hooks(globals()) -except Exception as gui_benchmark_error: - print(f"[Benchmark warning] GUI runtime recorder unavailable: {type(gui_benchmark_error).__name__}") diff --git a/tests/test_gui_runtime_recorder.py b/tests/test_gui_runtime_recorder.py deleted file mode 100644 index e521f95..0000000 --- a/tests/test_gui_runtime_recorder.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Regression coverage for automatic GUI per-inference recording.""" - -from __future__ import annotations - -import json -import tempfile -import unittest -import wave -from pathlib import Path -from types import SimpleNamespace - -from benchmarks.gui_runtime_recorder import ( - GUIRuntimeBenchmarkRecorder, - install_gui_runtime_benchmark_hooks, - sha256_file, -) - - -class FakeCache: - def __init__(self): - self.cache = {} - - def get_model(self, model_path, precision_mode=None): - normalized = str(Path(model_path).resolve()) - entry = self.cache.get(normalized) - if entry and entry["precision_mode"] == precision_mode: - return entry["model_obj"] - if entry: - self.cache.pop(normalized) - return None - - def set_model(self, model_path, model_obj, size_bytes=0, precision_mode=None): - self.cache[str(Path(model_path).resolve())] = { - "model_obj": model_obj, - "precision_mode": precision_mode, - } - - -class FakeOrtSession: - def __init__(self, precision): - self.precision = precision - - def get_inputs(self): - input_type = "tensor(float16)" if "FP16" in self.precision else "tensor(float)" - return [SimpleNamespace(type=input_type)] - - def get_providers(self): - return ["CUDAExecutionProvider", "CPUExecutionProvider"] - - -class FakeSampler: - def __init__(self, progress=None, **kwargs): - self.progress = progress or (lambda snapshot: None) - - def start(self): - self.progress({ - "elapsed_seconds": 0.0, - "process_cpu_percent": 12.5, - "gpu_utilization_percent": 75.0, - "device_vram_mb": 1024.0, - }) - - def stop(self): - return { - "sampling_interval_ms": 250.0, - "average_process_cpu_percent": 12.5, - "peak_process_cpu_percent": 20.0, - "average_system_cpu_percent": 10.0, - "peak_system_cpu_percent": 15.0, - "peak_process_ram_mb": 512.0, - "average_gpu_utilization_percent": 75.0, - "peak_gpu_utilization_percent": 90.0, - "baseline_device_vram_mb": 512.0, - "peak_device_vram_mb": 1536.0, - "average_power_w": 100.0, - "peak_power_w": 125.0, - "torch_peak_allocated_vram_mb": 768.0, - "torch_peak_reserved_vram_mb": 896.0, - "warnings": [], - } - - -class FailingSampler(FakeSampler): - def start(self): - raise RuntimeError("synthetic sampler failure") - - -class SeperateMDX: - def __init__(self, root: Path, cache: FakeCache, precision: str): - self.model_path = str(root / "UVR-MDX-NET-Test.onnx") - self.audio_file = str(root / "input.wav") - self.audio_file_base = "input" - self.export_path = str(root / "outputs") - self.model_data = SimpleNamespace(model_precision=precision, device_set="0") - self.device = "cuda:0" - self.process_data = {} - self.cache = cache - self.messages = [] - - def write_to_console(self, message, **kwargs): - self.messages.append(str(message)) - - def demix(self): - return "separated" - - def write_audio(self): - output = Path(self.export_path) - output.mkdir(parents=True, exist_ok=True) - with wave.open(str(output / "input_(Vocals).wav"), "wb") as stream: - stream.setnchannels(2) - stream.setsampwidth(2) - stream.setframerate(44100) - stream.writeframes(b"\0\0\0\0" * 4410) - - def seperate(self): - precision = self.model_data.model_precision - model = self.cache.get_model(self.model_path, precision_mode=precision) - if model is None: - model = FakeOrtSession(precision) - self.cache.set_model(self.model_path, model, precision_mode=precision) - self.demix() - self.write_audio() - return "done" - - -def fake_environment(device_index: int): - return { - "hardware": { - "os": "Test Windows", - "cpu": "Test CPU", - "gpu_name": "Test NVIDIA GPU", - }, - "software": { - "python_version": "3.11", - "torch_version": "test", - }, - "warnings": [], - } - - -class GUIRuntimeRecorderTests(unittest.TestCase): - def tearDown(self): - method = SeperateMDX.seperate - original = getattr(method, "_uvr_gui_benchmark_original", None) - if original is not None: - SeperateMDX.seperate = original - - def test_blocks_are_separate_and_model_round_resets_on_precision_change(self): - project_root = Path(__file__).resolve().parents[1] - uvr_hash_before = sha256_file(project_root / "UVR.py") - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - (root / "UVR-MDX-NET-Test.onnx").write_bytes(b"model") - with wave.open(str(root / "input.wav"), "wb") as stream: - stream.setnchannels(2) - stream.setsampwidth(2) - stream.setframerate(44100) - stream.writeframes(b"\0\0\0\0" * 4410) - cache = FakeCache() - namespace = { - "SeperateMDX": SeperateMDX, - "MODEL_CACHE": cache, - "prepare_mix": lambda value: value, - } - recorder = GUIRuntimeBenchmarkRecorder( - namespace, - output_root=root / "runtime", - sampler_factory=FakeSampler, - environment_collector=fake_environment, - progress_interval_seconds=1.0, - ) - install_gui_runtime_benchmark_hooks(namespace, recorder=recorder) - - first = SeperateMDX(root, cache, "Performance (FP16)") - second = SeperateMDX(root, cache, "Performance (FP16)") - third = SeperateMDX(root, cache, "Ultra Quality (FP32)") - self.assertEqual(first.seperate(), "done") - self.assertEqual(second.seperate(), "done") - self.assertEqual(third.seperate(), "done") - - session_name = (root / "runtime" / "latest.txt").read_text(encoding="utf-8").strip() - session = root / "runtime" / session_name - combined = (session / "benchmark.log").read_text(encoding="utf-8") - records = [ - json.loads(path.read_text(encoding="utf-8")) - for path in sorted(session.glob("inference_*.json")) - ] - - self.assertIn("1st Inference:", combined) - self.assertIn("2nd Inference:", combined) - self.assertIn("3rd Inference:", combined) - self.assertEqual([record["model_round"] for record in records], [1, 2, 1]) - self.assertEqual([record["status"] for record in records], ["completed"] * 3) - self.assertEqual(records[0]["runtime"]["effective_precision"], "fp16") - self.assertEqual(records[2]["runtime"]["effective_precision"], "fp32") - self.assertIn("Stage inference:", combined) - self.assertIn("Total-time:", combined) - self.assertGreaterEqual(combined.count("======="), 4) - self.assertTrue(any("[Benchmark] Recording" in message for message in first.messages)) - - self.assertEqual(sha256_file(project_root / "UVR.py"), uvr_hash_before) - - def test_sampler_setup_failure_keeps_processing_and_closes_record(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - (root / "UVR-MDX-NET-Test.onnx").write_bytes(b"model") - with wave.open(str(root / "input.wav"), "wb") as stream: - stream.setnchannels(2) - stream.setsampwidth(2) - stream.setframerate(44100) - stream.writeframes(b"\0\0\0\0" * 100) - cache = FakeCache() - prepare_mix = lambda value: value - namespace = { - "SeperateMDX": SeperateMDX, - "MODEL_CACHE": cache, - "prepare_mix": prepare_mix, - } - recorder = GUIRuntimeBenchmarkRecorder( - namespace, - output_root=root / "runtime", - sampler_factory=FailingSampler, - environment_collector=fake_environment, - ) - install_gui_runtime_benchmark_hooks(namespace, recorder=recorder) - - separator = SeperateMDX(root, cache, "Performance (FP16)") - result = separator.seperate() - - session_name = (root / "runtime" / "latest.txt").read_text(encoding="utf-8").strip() - session = root / "runtime" / session_name - record_path = next(session.glob("inference_*.json")) - record = json.loads(record_path.read_text(encoding="utf-8")) - text_log = next(session.glob("inference_*.log")).read_text(encoding="utf-8") - self.assertEqual(result, "done") - self.assertEqual(record["status"], "completed") - self.assertTrue(any("Recorder setup failed" in warning for warning in record["warnings"])) - self.assertIn("Warning: Recorder setup failed: RuntimeError", text_log) - self.assertIn("Status: COMPLETED", text_log) - self.assertIn("Total-time:", text_log) - self.assertIs(namespace["prepare_mix"], prepare_mix) - self.assertNotIn("demix", separator.__dict__) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_uvr_cli.py b/tests/test_uvr_cli.py new file mode 100644 index 0000000..405c909 --- /dev/null +++ b/tests/test_uvr_cli.py @@ -0,0 +1,170 @@ +"""Automated unit and regression tests for UVR CLI and Benchmarking suite.""" + +from __future__ import annotations + +import csv +import json +import tempfile +import unittest +from pathlib import Path + +from uvr_cli.cli import build_parser, create_synthetic_audio, normalize_cli_precision +from uvr_cli.exporter import export_benchmark_csv, export_benchmark_json, sanitize_path +from uvr_cli.model_resolver import ( + HeadlessModelData, + build_headless_model_data, + list_available_models, + resolve_model_file, +) +from uvr_cli.telemetry import TelemetrySampler, collect_system_info + + +class BenchmarkCLITests(unittest.TestCase): + """Test coverage for model resolution, telemetry, CLI parsing, and export.""" + + def test_model_discovery_finds_installed_models(self): + models = list_available_models() + self.assertIsInstance(models, list) + self.assertGreater(len(models), 0, "Should detect at least one installed UVR model") + first = models[0] + self.assertIn("name", first) + self.assertIn("architecture", first) + self.assertIn("path", first) + + def test_model_resolution_and_headless_data(self): + models = list_available_models() + target = models[0]["name"] + path, arch = resolve_model_file(target) + self.assertTrue(path.is_file()) + + model_data = build_headless_model_data( + model_path=path, + architecture=arch, + precision="Performance (FP16)", + device="cpu", + ) + self.assertIsInstance(model_data, HeadlessModelData) + self.assertEqual(model_data.process_method, arch) + self.assertEqual(model_data.model_precision, "Performance (FP16)") + self.assertIsNotNone(model_data.model_hash) + + def test_telemetry_sampler_lifecycle(self): + sampler = TelemetrySampler(device_index=0, interval_sec=0.05) + sampler.start() + # Brief pause to collect samples + import time + time.sleep(0.2) + metrics = sampler.stop() + + self.assertIn("avg_process_cpu_percent", metrics) + self.assertIn("peak_process_cpu_percent", metrics) + self.assertIn("peak_process_ram_mb", metrics) + self.assertIn("samples_count", metrics) + self.assertGreaterEqual(metrics["samples_count"], 1) + + def test_system_info_collection(self): + info = collect_system_info(device_index=0) + self.assertIn("os", info) + self.assertIn("cpu", info) + self.assertIn("system_ram_gb", info) + self.assertIn("python_version", info) + + def test_path_sanitization(self): + raw_path = r"C:\Users\JohnDoe\AppData\Local\test.wav" + sanitized = sanitize_path(raw_path) + self.assertNotIn("JohnDoe", sanitized) + self.assertIn("/", sanitized) + + def test_export_json_and_csv(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + json_file = temp_path / "test_session.json" + csv_file = temp_path / "summary.csv" + + dummy_data = { + "session_id": "test_123", + "system_info": {"os": "TestOS"}, + "summary": {"avg_total_time_sec": 1.23}, + } + export_benchmark_json(dummy_data, json_file) + self.assertTrue(json_file.is_file()) + with json_file.open("r", encoding="utf-8") as f: + loaded = json.load(f) + self.assertEqual(loaded["session_id"], "test_123") + + dummy_rows = [{ + "session_id": "test_123", + "timestamp_utc": "2026-09-06T00:00:00Z", + "gpu_name": "TestGPU", + "cpu_name": "TestCPU", + "model_name": "test_model.onnx", + "model_hash": "abc123hash", + "backend": "MDX-Net", + "precision": "Performance (FP16)", + "audio_file": "sample.wav", + "audio_duration_sec": 10.0, + "round": "1", + "cache_state": "cold", + "total_time_sec": 2.5, + "inference_time_sec": 2.5, + "speed_factor_rt": 4.0, + "peak_torch_vram_mb": 1024.0, + "peak_device_vram_mb": 2048.0, + "avg_gpu_util_percent": 85.0, + "peak_gpu_util_percent": 95.0, + "avg_cpu_util_percent": 50.0, + "peak_process_ram_mb": 512.0, + "status": "COMPLETED", + }] + export_benchmark_csv(dummy_rows, csv_file) + self.assertTrue(csv_file.is_file()) + with csv_file.open("r", encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["model_name"], "test_model.onnx") + + def test_cli_argument_parsing(self): + parser = build_parser() + + # Test 'list-models' + args_list = parser.parse_args(["list-models"]) + self.assertEqual(args_list.command, "list-models") + + # Test 'run' + args_run = parser.parse_args([ + "run", + "--audio", "test.wav", + "--model", "UVR-MDX-NET-Inst_HQ_4.onnx", + "--precision", "fp16", + "--device", "cuda:0", + ]) + self.assertEqual(args_run.command, "run") + self.assertEqual(args_run.precision, "fp16") + + # Test 'bench' + args_bench = parser.parse_args([ + "bench", + "--audio", "test.wav", + "--model", "UVR-MDX-NET-Inst_HQ_4.onnx", + "--rounds", "3", + "--warmup", "1", + ]) + self.assertEqual(args_bench.command, "bench") + self.assertEqual(args_bench.rounds, 3) + self.assertEqual(args_bench.warmup, 1) + + def test_synthetic_audio_creation(self): + audio_path = create_synthetic_audio(duration_sec=1.0, samplerate=44100) + try: + self.assertTrue(audio_path.is_file()) + self.assertGreater(audio_path.stat().st_size, 1000) + finally: + if audio_path.is_file(): + audio_path.unlink() + if audio_path.parent.is_dir(): + audio_path.parent.rmdir() + + +if __name__ == "__main__": + unittest.main() diff --git a/uvr_cli/README.md b/uvr_cli/README.md new file mode 100644 index 0000000..c8078c2 --- /dev/null +++ b/uvr_cli/README.md @@ -0,0 +1,105 @@ +# Ultimate Vocal Remover (UVR) - CLI & Performance Benchmark Suite + +This package provides a **Command-Line Interface (CLI)** and a **Real Performance Benchmarking Suite** for UVR. It connects directly to UVR's actual inference engine in `separate.py` without requiring the Tkinter GUI or desktop automation. + +--- + +## Folder Structure + +```text +uvr_cli/ +├── __init__.py # Package exports and version +├── __main__.py # Package runner for `python -m uvr_cli` +├── cli.py # Command-line interface (`run`, `bench`, `list-models`, `smoke-test`) +├── model_resolver.py # Headless model discovery & parameter builder +├── runner.py # Direct execution bridge to separate.py +├── telemetry.py # Real-time resource sampler (CPU, RAM, GPU %, VRAM, Power) +├── engine.py # Benchmark orchestrator (warmup, rounds, statistics) +├── exporter.py # Machine-readable JSON & CSV exporters with path sanitization +└── README.md # This documentation +``` + +--- + +## How It Works + +1. **Headless Model Resolution (`model_resolver.py`)**: + Reads UVR's model directories (`models/MDX_Net_Models`, `models/VR_Models`, `models/Demucs_Models`), computes the exact model MD5 hash using UVR's chunked hash algorithm, and loads the model's architecture parameters (FFT, hop size, frequency bins, stems, compensation) from `model_data.json` without depending on Tkinter GUI variables. +2. **Direct Core Separation (`runner.py`)**: + Directly instantiates `SeperateMDX`, `SeperateMDXC`, `SeperateVR`, or `SeperateDemucs` from `separate.py` and triggers `.seperate()` directly. +3. **Real-time Telemetry (`telemetry.py`)**: + Runs a background sampling daemon measuring process CPU %, system CPU %, process RAM, GPU utilization %, and device VRAM. Synchronizes CUDA devices and resets PyTorch peak allocation statistics before/after inference to capture true peak VRAM. +4. **Machine-Readable Export (`exporter.py`)**: + Outputs complete structured JSON documents and appends runs to a shared tabular CSV (`summary.csv`) suitable for cross-GPU comparisons and committing to GitHub. All user profile paths are automatically redacted for privacy. + +--- + +## CLI Commands + +### 1. List Available Models +Scan and display all downloaded models across MDX-Net, VR, and Demucs: +```powershell +venv\Scripts\python.exe -m uvr_cli list-models +``` + +### 2. Direct Separation (CLI-Only) +Separate an audio file directly from the terminal without GUI: +```powershell +venv\Scripts\python.exe -m uvr_cli run ` + --audio "path/to/song.wav" ` + --model "UVR-MDX-NET-Inst_HQ_4.onnx" ` + --precision fp16 ` + --device cuda:0 ` + --output-dir "separated_outputs" +``` +Supported precisions: `fp32`, `fp16`, `fp8_e4m3`, `nvfp4`. + +### 3. Real Performance Benchmark +Benchmark one or more models across multiple rounds with warmup: +```powershell +venv\Scripts\python.exe -m uvr_cli bench ` + --audio "path/to/song.wav" ` + --model "UVR-MDX-NET-Inst_HQ_4.onnx" ` + --precision fp16 ` + --rounds 3 ` + --warmup 1 ` + --export-dir "benchmark_results" +``` + +### 4. Automated Smoke Test +Run an end-to-end verification test using an automatically generated synthetic audio clip: +```powershell +venv\Scripts\python.exe -m uvr_cli smoke-test +``` + +--- + +## Output Data Format + +### 1. Tabular CSV (`benchmark_results/summary.csv`) +Columns included in the CSV summary: +- `session_id`: Unique identifier for the benchmark session +- `timestamp_utc`: UTC timestamp of the run +- `gpu_name`: Hardware GPU device name +- `cpu_name`: Hardware CPU processor model +- `model_name`: File name of the model +- `model_hash`: MD5 hash of the model +- `backend`: UVR backend architecture (MDX-Net, VR Arc, Demucs) +- `precision`: Effective precision policy (FP32, FP16, BF16, etc.) +- `audio_file`: Name of the benchmarked audio file +- `audio_duration_sec`: Duration of the audio file in seconds +- `round`: Measurement round index or `warmup_1` +- `cache_state`: `cold` (first run) or `warm` (subsequent runs) +- `total_time_sec`: Total wall-clock execution time +- `inference_time_sec`: Pure inference time +- `speed_factor_rt`: Real-time processing speed multiplier (`audio_duration / total_time`) +- `peak_torch_vram_mb`: Peak VRAM allocated by PyTorch tensors (MiB) +- `peak_device_vram_mb`: Peak overall hardware VRAM in use (MiB) +- `avg_gpu_util_percent`: Average GPU core utilization percentage +- `peak_gpu_util_percent`: Peak GPU core utilization percentage +- `avg_cpu_util_percent`: Average CPU process utilization percentage +- `peak_process_ram_mb`: Peak system RAM consumed by the process (MiB) +- `status`: Execution outcome (`COMPLETED` or error) + +### 2. Structured JSON (`benchmark_results/.json`) +Contains the full system hardware report, audio metadata, model configuration parameters, individual per-round telemetry profiles, and aggregated statistics (mean, min, max). diff --git a/uvr_cli/__init__.py b/uvr_cli/__init__.py new file mode 100644 index 0000000..49fa234 --- /dev/null +++ b/uvr_cli/__init__.py @@ -0,0 +1,35 @@ +"""Ultimate Vocal Remover - CLI-Only Inference & Real Performance Benchmarking Suite.""" + +from __future__ import annotations + +from .engine import BenchmarkEngine +from .exporter import export_benchmark_csv, export_benchmark_json, sanitize_path +from .model_resolver import ( + HeadlessModelData, + build_headless_model_data, + compute_uvr_hash, + list_available_models, + resolve_model_file, +) +from .runner import execute_inference, get_audio_metadata +from .telemetry import TelemetrySampler, collect_system_info, sync_cuda + +__version__ = "2.0.0" + +__all__ = [ + "BenchmarkEngine", + "HeadlessModelData", + "TelemetrySampler", + "build_headless_model_data", + "collect_system_info", + "compute_uvr_hash", + "execute_inference", + "export_benchmark_csv", + "export_benchmark_json", + "get_audio_metadata", + "list_available_models", + "resolve_model_file", + "sanitize_path", + "sync_cuda", + "__version__", +] diff --git a/uvr_cli/__main__.py b/uvr_cli/__main__.py new file mode 100644 index 0000000..a7d34eb --- /dev/null +++ b/uvr_cli/__main__.py @@ -0,0 +1,6 @@ +"""Package entry point allowing execution via `python -m uvr_cli`.""" + +from uvr_cli.cli import main + +if __name__ == "__main__": + main() diff --git a/uvr_cli/cli.py b/uvr_cli/cli.py new file mode 100644 index 0000000..38cbd75 --- /dev/null +++ b/uvr_cli/cli.py @@ -0,0 +1,308 @@ +"""Command-Line Interface for UVR CLI-Only inference and performance benchmarking.""" + +from __future__ import annotations + +import argparse +import sys +import tempfile +import wave +from pathlib import Path +from typing import Optional + +import numpy as np + +from gui_data.constants import ( + MODEL_PRECISION_BF16, + MODEL_PRECISION_DEFAULT, + MODEL_PRECISION_FP16, + MODEL_PRECISION_FP32, + MODEL_PRECISION_OPTIONS, +) +from uvr_cli.engine import BenchmarkEngine +from uvr_cli.model_resolver import ( + build_headless_model_data, + list_available_models, + resolve_model_file, +) +from uvr_cli.runner import execute_inference + +# Map shorthand precision arguments to UVR canonical precision strings +PRECISION_MAP = { + "fp32": MODEL_PRECISION_FP32, + "fp16": MODEL_PRECISION_FP16, + "bf16": MODEL_PRECISION_BF16, + "fp8": "Performance (FP8-E4M3)", + "fp8_e4m3": "Performance (FP8-E4M3)", + "nvfp4": "Performance (NVFP4)", +} + + +def normalize_cli_precision(prec_arg: Optional[str]) -> str: + """Map user CLI input (e.g. 'fp16') to UVR canonical precision name.""" + if not prec_arg: + return MODEL_PRECISION_DEFAULT + clean = prec_arg.strip().lower() + if clean in PRECISION_MAP: + return PRECISION_MAP[clean] + # Check if user passed the full canonical string + for opt in MODEL_PRECISION_OPTIONS: + if clean == opt.lower(): + return opt + return MODEL_PRECISION_DEFAULT + + +def create_synthetic_audio(duration_sec: float = 2.0, samplerate: int = 44100) -> Path: + """Create a temporary sine-wave test WAV file for smoke testing.""" + temp_dir = Path(tempfile.mkdtemp(prefix="uvr_smoke_test_")) + audio_path = temp_dir / "synthetic_test_input.wav" + + total_samples = int(duration_sec * samplerate) + t = np.linspace(0, duration_sec, total_samples, endpoint=False) + # 440 Hz (A4) sine wave in stereo + signal = 0.5 * np.sin(2 * np.pi * 440 * t) + stereo_signal = np.vstack((signal, signal)).T + int_signal = (stereo_signal * 32767).astype(np.int16) + + with wave.open(str(audio_path), "wb") as wf: + wf.setnchannels(2) + wf.setsampwidth(2) + wf.setframerate(samplerate) + wf.writeframes(int_signal.tobytes()) + + return audio_path + + +def handle_list_models(args: argparse.Namespace) -> int: + """List all installed UVR models.""" + models = list_available_models() + if not models: + print("No models found in models/ directories.") + return 0 + + print(f"\n{'='*75}") + print(f" INSTALLED UVR MODELS ({len(models)} found)") + print(f"{'='*75}") + print(f"{'Architecture':<15} | {'Size (MB)':<10} | {'Model Name'}") + print(f"{'-'*15}-+-{'-'*10}-+-{'-'*45}") + for m in models: + print(f"{m['architecture']:<15} | {m['size_mb']:<10.2f} | {m['name']}") + print(f"{'='*75}\n") + return 0 + + +def handle_run(args: argparse.Namespace) -> int: + """Perform CLI-only separation directly through UVR core.""" + model_path, arch = resolve_model_file(args.model) + precision = normalize_cli_precision(args.precision) + + print(f"Resolving model: {model_path.name} ({arch})") + model_data = build_headless_model_data( + model_path=model_path, + architecture=arch, + precision=precision, + device=args.device, + segment_size=args.segment_size, + overlap=args.overlap, + batch_size=args.batch_size, + ) + + audio_path = Path(args.audio).resolve() + output_dir = Path(args.output_dir).resolve() + + print(f"Separating : {audio_path.name}") + print(f"Precision : {precision}") + print(f"Output folder : {output_dir}") + + stems, is_valid = execute_inference( + model_data=model_data, + audio_path=audio_path, + export_dir=output_dir, + console_callback=lambda text, base="": print(f"[UVR Core] {text}"), + ) + + if is_valid and stems: + print(f"\n[OK] Separation completed successfully! Generated {len(stems)} stem(s):") + for stem in stems: + print(f" -> {stem.name} ({round(stem.stat().st_size / (1024*1024), 2)} MB)") + return 0 + else: + print(f"\n[ERROR] Separation completed but no valid outputs were generated.") + return 1 + + +def handle_bench(args: argparse.Namespace) -> int: + """Run real performance benchmark on UVR models.""" + model_path, arch = resolve_model_file(args.model) + precision = normalize_cli_precision(args.precision) + + model_data = build_headless_model_data( + model_path=model_path, + architecture=arch, + precision=precision, + device=args.device, + segment_size=args.segment_size, + overlap=args.overlap, + batch_size=args.batch_size, + ) + + device_idx = int(args.device.split(":")[-1]) if ":" in args.device else 0 + + engine = BenchmarkEngine( + model_data=model_data, + audio_path=args.audio, + results_dir=args.export_dir, + rounds=args.rounds, + warmup=args.warmup, + device_index=device_idx, + ) + + results = engine.run(verbose=True) + return 0 if results["summary"]["rounds_measured"] > 0 else 1 + + +def handle_smoke_test(args: argparse.Namespace) -> int: + """Run end-to-end smoke test with synthetic audio.""" + print("Generating 2.0-second synthetic stereo audio for smoke test...") + audio_path = create_synthetic_audio(duration_sec=2.0) + + try: + # Select model: use specified or pick first available + if args.model: + model_target = args.model + else: + models = list_available_models() + if not models: + print("[ERROR] No models installed to test with. Download a model first.") + return 1 + model_target = models[0]["path"] + + model_path, arch = resolve_model_file(model_target) + precision = normalize_cli_precision(args.precision) + + print(f"Selected smoke test model: {model_path.name} ({arch})") + + model_data = build_headless_model_data( + model_path=model_path, + architecture=arch, + precision=precision, + device=args.device, + ) + + device_idx = int(args.device.split(":")[-1]) if ":" in args.device else 0 + + engine = BenchmarkEngine( + model_data=model_data, + audio_path=audio_path, + results_dir=args.export_dir, + rounds=1, + warmup=1, + device_index=device_idx, + ) + + results = engine.run(verbose=True) + if results["summary"]["rounds_measured"] > 0: + print("\n[OK] Smoke test passed successfully!") + return 0 + else: + print("\n[FAIL] Smoke test completed without successful rounds.") + return 1 + + finally: + # Clean up synthetic audio file and its folder + try: + if audio_path.is_file(): + audio_path.unlink() + if audio_path.parent.is_dir(): + audio_path.parent.rmdir() + except Exception: + pass + + +def build_parser() -> argparse.ArgumentParser: + """Build command-line parser.""" + parser = argparse.ArgumentParser( + prog="uvr_cli", + description="Ultimate Vocal Remover - CLI-Only Inference & Performance Benchmarking Suite", + ) + subparsers = parser.add_subparsers(dest="command", help="Available subcommands") + + # Command: list-models + subparsers.add_parser("list-models", help="List all available installed models") + + # Command: run + run_parser = subparsers.add_parser("run", help="Run UVR separation directly via CLI (No GUI)") + run_parser.add_argument("--audio", "-a", required=True, help="Path to input audio file") + run_parser.add_argument("--model", "-m", required=True, help="Model name, filename, or absolute path") + run_parser.add_argument( + "--precision", + "-p", + default="fp16", + help="Inference precision: fp32, fp16, fp8, nvfp4 (default: fp16)", + ) + run_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)") + run_parser.add_argument("--output-dir", "-o", default="separated_outputs", help="Output directory for stems") + run_parser.add_argument("--segment-size", type=int, default=None, help="Segment size (MDX chunk size)") + run_parser.add_argument("--overlap", type=float, default=None, help="Overlap ratio (e.g. 0.25)") + run_parser.add_argument("--batch-size", type=int, default=1, help="Inference batch size (default: 1)") + + # Command: bench + bench_parser = subparsers.add_parser("bench", help="Run real benchmark with hardware telemetry & metric export") + bench_parser.add_argument("--audio", "-a", required=True, help="Path to input audio file") + bench_parser.add_argument("--model", "-m", required=True, help="Model name, filename, or absolute path") + bench_parser.add_argument( + "--precision", + "-p", + default="fp16", + help="Inference precision: fp32, fp16, fp8, nvfp4 (default: fp16)", + ) + bench_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)") + bench_parser.add_argument("--rounds", "-r", type=int, default=3, help="Number of measurement rounds (default: 3)") + bench_parser.add_argument("--warmup", "-w", type=int, default=1, help="Number of warmup rounds (default: 1)") + bench_parser.add_argument( + "--export-dir", + "-e", + default="benchmark_results", + help="Directory to save JSON & CSV reports (default: benchmark_results)", + ) + bench_parser.add_argument("--segment-size", type=int, default=None, help="Segment size (MDX chunk size)") + bench_parser.add_argument("--overlap", type=float, default=None, help="Overlap ratio (e.g. 0.25)") + bench_parser.add_argument("--batch-size", type=int, default=1, help="Inference batch size (default: 1)") + + # Command: smoke-test + smoke_parser = subparsers.add_parser("smoke-test", help="Run automated smoke test using synthetic audio") + smoke_parser.add_argument("--model", "-m", default=None, help="Optional model name (defaults to first available)") + smoke_parser.add_argument("--precision", "-p", default="fp16", help="Inference precision (default: fp16)") + smoke_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)") + smoke_parser.add_argument( + "--export-dir", + "-e", + default="benchmark_results", + help="Directory to save smoke test reports (default: benchmark_results)", + ) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + if not args.command: + parser.print_help() + sys.exit(1) + + if args.command == "list-models": + sys.exit(handle_list_models(args)) + elif args.command == "run": + sys.exit(handle_run(args)) + elif args.command == "bench": + sys.exit(handle_bench(args)) + elif args.command == "smoke-test": + sys.exit(handle_smoke_test(args)) + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/uvr_cli/engine.py b/uvr_cli/engine.py new file mode 100644 index 0000000..148c9d0 --- /dev/null +++ b/uvr_cli/engine.py @@ -0,0 +1,211 @@ +"""Benchmarking engine managing warmup, measurement rounds, timing, and export.""" + +from __future__ import annotations + +import os +import shutil +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional +import uuid + +from .exporter import export_benchmark_csv, export_benchmark_json +from .model_resolver import HeadlessModelData +from .runner import execute_inference, get_audio_metadata +from .telemetry import TelemetrySampler, collect_system_info, sync_cuda + + +class BenchmarkEngine: + """Orchestrates end-to-end performance benchmarking for UVR inference.""" + + def __init__( + self, + model_data: HeadlessModelData, + audio_path: str | Path, + results_dir: str | Path = "benchmark_results", + rounds: int = 1, + warmup: int = 0, + device_index: int = 0, + ) -> None: + self.model_data = model_data + self.audio_path = Path(audio_path).resolve() + self.results_dir = Path(results_dir).resolve() + self.rounds = max(1, rounds) + self.warmup = max(0, warmup) + self.device_index = device_index + + self.session_id = f"session_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}" + self.results_dir.mkdir(parents=True, exist_ok=True) + + def run(self, verbose: bool = True) -> Dict[str, Any]: + """Run warmup and measurement rounds, gather telemetry, and export results.""" + if verbose: + print(f"\n{'='*70}") + print(f" UVR BENCHMARK SESSION: {self.session_id}") + print(f"{'='*70}") + print(f"Model : {self.model_data.model_name}") + print(f"Backend : {self.model_data.process_method}") + print(f"Precision : {self.model_data.model_precision}") + print(f"Audio File : {self.audio_path.name}") + print(f"Rounds : {self.rounds} (Warmup: {self.warmup})") + print(f"Device Index : {self.device_index}") + print(f"{'='*70}\n") + + system_info = collect_system_info(self.device_index) + audio_info = get_audio_metadata(self.audio_path) + + all_runs_data: List[Dict[str, Any]] = [] + csv_rows: List[Dict[str, Any]] = [] + + total_iterations = self.warmup + self.rounds + current_iter = 0 + + temp_stems_dir = self.results_dir / "temp_stems" / self.session_id + temp_stems_dir.mkdir(parents=True, exist_ok=True) + + try: + for i in range(total_iterations): + current_iter += 1 + is_warmup_run = current_iter <= self.warmup + round_number = current_iter if is_warmup_run else (current_iter - self.warmup) + cache_state = "cold" if current_iter == 1 else "warm" + run_label = f"Warmup {round_number}" if is_warmup_run else f"Measurement Round {round_number}/{self.rounds}" + + if verbose: + print(f"--> Starting {run_label} (Cache: {cache_state})...") + + round_export_dir = temp_stems_dir / f"round_{current_iter}" + round_export_dir.mkdir(parents=True, exist_ok=True) + + sampler = TelemetrySampler(device_index=self.device_index, interval_sec=0.1) + sync_cuda(self.device_index) + + sampler.start() + start_time = time.perf_counter() + + stems, is_valid = execute_inference( + model_data=self.model_data, + audio_path=self.audio_path, + export_dir=round_export_dir, + ) + + sync_cuda(self.device_index) + end_time = time.perf_counter() + telemetry = sampler.stop() + + total_time = round(end_time - start_time, 4) + duration_sec = audio_info.get("duration_sec", 0.0) + speed_factor = round(duration_sec / total_time, 2) if total_time > 0 and duration_sec > 0 else 0.0 + + run_record = { + "iteration": current_iter, + "is_warmup": is_warmup_run, + "round": round_number, + "cache_state": cache_state, + "total_time_sec": total_time, + "speed_factor_rt": speed_factor, + "is_valid": is_valid, + "output_stems": [s.name for s in stems], + "telemetry": telemetry, + } + all_runs_data.append(run_record) + + csv_row = { + "session_id": self.session_id, + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "gpu_name": system_info.get("gpu_name", "N/A"), + "cpu_name": system_info.get("cpu", "N/A"), + "model_name": self.model_data.model_name, + "model_hash": self.model_data.model_hash, + "backend": self.model_data.process_method, + "precision": self.model_data.model_precision, + "audio_file": self.audio_path.name, + "audio_duration_sec": duration_sec, + "round": f"warmup_{round_number}" if is_warmup_run else str(round_number), + "cache_state": cache_state, + "total_time_sec": total_time, + "inference_time_sec": total_time, # Pure separator execution + "speed_factor_rt": speed_factor, + "peak_torch_vram_mb": telemetry.get("torch_peak_allocated_vram_mb", 0.0), + "peak_device_vram_mb": telemetry.get("peak_device_vram_mb", 0.0), + "avg_gpu_util_percent": telemetry.get("avg_gpu_utilization_percent", 0.0), + "peak_gpu_util_percent": telemetry.get("peak_gpu_utilization_percent", 0.0), + "avg_cpu_util_percent": telemetry.get("avg_process_cpu_percent", 0.0), + "peak_process_ram_mb": telemetry.get("peak_process_ram_mb", 0.0), + "status": "COMPLETED" if is_valid else "VALIDATION_FAILED", + } + csv_rows.append(csv_row) + + if verbose: + print( + f" Time: {total_time:.2f}s ({speed_factor}x RT) | " + f"Peak Torch VRAM: {telemetry.get('torch_peak_allocated_vram_mb', 0):.1f} MiB | " + f"Peak Device VRAM: {telemetry.get('peak_device_vram_mb', 0):.1f} MiB | " + f"GPU Util: {telemetry.get('avg_gpu_utilization_percent', 0):.1f}% (Peak: {telemetry.get('peak_gpu_utilization_percent', 0):.1f}%)" + ) + + finally: + # Clean up temporary stems after benchmarking + try: + shutil.rmtree(temp_stems_dir, ignore_errors=True) + except Exception: + pass + + # Compute summary metrics across non-warmup rounds (or all if only warmup was run) + measured_runs = [r for r in all_runs_data if not r["is_warmup"]] + if not measured_runs: + measured_runs = all_runs_data + + measured_times = [r["total_time_sec"] for r in measured_runs] + measured_speed_factors = [r["speed_factor_rt"] for r in measured_runs] + measured_torch_vram = [r["telemetry"].get("torch_peak_allocated_vram_mb", 0.0) for r in measured_runs] + measured_device_vram = [r["telemetry"].get("peak_device_vram_mb", 0.0) for r in measured_runs] + + summary = { + "rounds_measured": len(measured_runs), + "warmup_rounds": self.warmup, + "avg_total_time_sec": round(sum(measured_times) / len(measured_times), 4), + "min_total_time_sec": min(measured_times), + "max_total_time_sec": max(measured_times), + "avg_speed_factor_rt": round(sum(measured_speed_factors) / len(measured_speed_factors), 2), + "peak_torch_allocated_vram_mb": max(measured_torch_vram) if measured_torch_vram else 0.0, + "peak_device_vram_mb": max(measured_device_vram) if measured_device_vram else 0.0, + } + + report_data: Dict[str, Any] = { + "session_id": self.session_id, + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "system_info": system_info, + "audio_info": audio_info, + "model_info": { + "name": self.model_data.model_name, + "hash": self.model_data.model_hash, + "backend": self.model_data.process_method, + "precision": self.model_data.model_precision, + "segment_size": getattr(self.model_data, "mdx_segment_size", None), + "batch_size": getattr(self.model_data, "mdx_batch_size", 1), + }, + "summary": summary, + "runs": all_runs_data, + } + + # Export JSON and CSV + json_file = self.results_dir / f"{self.session_id}.json" + csv_file = self.results_dir / "summary.csv" + + export_benchmark_json(report_data, json_file) + export_benchmark_csv(csv_rows, csv_file) + + if verbose: + print(f"\n{'='*70}") + print(f" BENCHMARK SUMMARY") + print(f"{'='*70}") + print(f"Avg Time : {summary['avg_total_time_sec']:.2f}s (Min: {summary['min_total_time_sec']:.2f}s, Max: {summary['max_total_time_sec']:.2f}s)") + print(f"Speed Factor : {summary['avg_speed_factor_rt']:.2f}x real-time") + print(f"Peak VRAM : Torch: {summary['peak_torch_allocated_vram_mb']:.1f} MiB | Device: {summary['peak_device_vram_mb']:.1f} MiB") + print(f"JSON Report : {json_file}") + print(f"CSV Summary : {csv_file}") + print(f"{'='*70}\n") + + return report_data diff --git a/uvr_cli/exporter.py b/uvr_cli/exporter.py new file mode 100644 index 0000000..4b9cb90 --- /dev/null +++ b/uvr_cli/exporter.py @@ -0,0 +1,81 @@ +"""Machine-readable JSON and CSV exporters for UVR benchmarks with privacy sanitization.""" + +from __future__ import annotations + +import csv +import json +import os +import re +from pathlib import Path +from typing import Any, Dict, List + +_WINDOWS_USER_PATH = re.compile(r"(?i)[A-Z]:[\\/]Users[\\/][^\\/\r\n\"']+[\\/]") +_GENERIC_ABS_PATH = re.compile(r"(?i)[A-Z]:[\\/][^\r\n\"']+") + + +def sanitize_path(value: Any) -> Any: + """Sanitize paths and redact local username for safe GitHub uploads.""" + if isinstance(value, str): + # Redact specific user profile directory while keeping project relative path + redacted = _WINDOWS_USER_PATH.sub("/", value) + return redacted.replace("\\", "/") + if isinstance(value, dict): + return {k: sanitize_path(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [sanitize_path(v) for v in value] + return value + + +def export_benchmark_json(data: Dict[str, Any], output_path: str | Path) -> Path: + """Export benchmark result to structured machine-readable JSON.""" + path = Path(output_path).resolve() + path.parent.mkdir(parents=True, exist_ok=True) + sanitized = sanitize_path(data) + with path.open("w", encoding="utf-8") as f: + json.dump(sanitized, f, indent=2, sort_keys=True) + return path + + +CSV_FIELDS = [ + "session_id", + "timestamp_utc", + "gpu_name", + "cpu_name", + "model_name", + "model_hash", + "backend", + "precision", + "audio_file", + "audio_duration_sec", + "round", + "cache_state", + "total_time_sec", + "inference_time_sec", + "speed_factor_rt", + "peak_torch_vram_mb", + "peak_device_vram_mb", + "avg_gpu_util_percent", + "peak_gpu_util_percent", + "avg_cpu_util_percent", + "peak_process_ram_mb", + "status", +] + + +def export_benchmark_csv(runs_summary: List[Dict[str, Any]], csv_path: str | Path) -> Path: + """Append benchmark runs to a tabular summary CSV file.""" + path = Path(csv_path).resolve() + path.parent.mkdir(parents=True, exist_ok=True) + + file_exists = path.is_file() and path.stat().st_size > 0 + + with path.open("a", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=CSV_FIELDS, extrasaction="ignore") + if not file_exists: + writer.writeheader() + + for run in runs_summary: + clean_run = sanitize_path(run) + writer.writerow(clean_run) + + return path diff --git a/uvr_cli/model_resolver.py b/uvr_cli/model_resolver.py new file mode 100644 index 0000000..57d6594 --- /dev/null +++ b/uvr_cli/model_resolver.py @@ -0,0 +1,419 @@ +"""Model resolution and headless configuration adapter for UVR models. + +Discovers models, computes UVR MD5 hashes, parses model configurations, +and constructs HeadlessModelData objects suitable for direct ingestion by +separate.py separator classes (SeperateMDX, SeperateMDXC, SeperateVR, SeperateDemucs). +""" + +from __future__ import annotations + +import hashlib +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import yaml +from ml_collections import ConfigDict + +from gui_data.constants import ( + ALL_STEMS, + CKPT, + DEMUCS_2_SOURCE, + DEMUCS_2_SOURCE_MAPPER, + DEMUCS_4_SOURCE, + DEMUCS_4_SOURCE_MAPPER, + DEMUCS_ARCH_TYPE, + DEMUCS_NEWER_TAGS, + DEMUCS_UVR_MODEL, + DEMUCS_V1, + DEMUCS_V1_TAG, + DEMUCS_V2, + DEMUCS_V2_TAG, + DEMUCS_V3, + DEMUCS_V3_TAG, + DEMUCS_V4, + DEMUCS_V4_TAG, + DEMUCS_VERSION_MAPPER, + INST_STEM, + MDX_ARCH_TYPE, + MODEL_PRECISION_DEFAULT, + MODEL_PRECISION_FP16, + MODEL_PRECISION_FP32, + MODEL_PRECISION_BF16, + MODEL_PRECISION_OPTIONS, + NO_MODEL, + ONNX, + PRIMARY_STEM, + PTH, + VOCAL_STEM, + VR_ARCH_TYPE, + YAML, + normalize_model_precision, + secondary_stem, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MODELS_DIR = PROJECT_ROOT / "models" +MDX_MODELS_DIR = MODELS_DIR / "MDX_Net_Models" +VR_MODELS_DIR = MODELS_DIR / "VR_Models" +DEMUCS_MODELS_DIR = MODELS_DIR / "Demucs_Models" +DEMUCS_NEWER_REPO_DIR = DEMUCS_MODELS_DIR / "v3_v4_repo" + +MDX_HASH_DIR = MDX_MODELS_DIR / "model_data" +MDX_HASH_JSON = MDX_HASH_DIR / "model_data.json" +MDX_C_CONFIG_PATH = MDX_HASH_DIR / "mdx_c_configs" +MDX_NAME_MAPPER_PATH = MDX_HASH_DIR / "model_name_mapper.json" + +VR_HASH_DIR = VR_MODELS_DIR / "model_data" +VR_HASH_JSON = VR_HASH_DIR / "model_data.json" +VR_PARAM_DIR = PROJECT_ROOT / "lib_v5" / "vr_network" / "modelparams" + +DEMUCS_NAME_MAPPER_PATH = DEMUCS_MODELS_DIR / "model_data" / "model_name_mapper.json" + + +def compute_uvr_hash(model_path: str | Path) -> str: + """Compute the model hash using UVR's exact algorithm. + + Reads the last 10,000 * 1024 bytes (or full file if smaller) and computes MD5. + """ + path = Path(model_path).resolve() + with open(path, "rb") as stream: + try: + stream.seek(-10000 * 1024, 2) + return hashlib.md5(stream.read()).hexdigest() + except (OSError, ValueError): + stream.seek(0) + return hashlib.md5(stream.read()).hexdigest() + + +class HeadlessModelData: + """Lightweight configuration container compatible with SeperateAttributes.""" + + def __init__(self, **kwargs: Any) -> None: + # Default flags matching SeperateAttributes expectations + self.process_method: str = MDX_ARCH_TYPE + self.model_path: str = "" + self.model_name: str = "" + self.model_basename: str = "" + self.model_hash: Optional[str] = None + self.model_status: bool = True + self.device_set: str = "0" + self.is_gpu_conversion: int = 0 + self.is_use_opencl: bool = False + self.model_precision: str = MODEL_PRECISION_DEFAULT + self.save_format: str = "WAV" + self.wav_type_set: str = "PCM_16" + self.mp3_bit_set: str = "320k" + self.is_normalization: bool = False + self.is_primary_stem_only: bool = False + self.is_secondary_stem_only: bool = False + self.is_pitch_change: bool = False + self.semitone_shift: float = 0.0 + self.is_match_frequency_pitch: bool = False + self.is_invert_spec: bool = False + self.is_deverb_vocals: bool = False + self.is_mixer_mode: bool = False + self.primary_stem: str = "Instrumental" + self.secondary_stem: str = "Vocals" + self.primary_stem_native: str = "Instrumental" + self.model_samplerate: int = 44100 + self.model_capacity: Tuple[int, int] = (32, 128) + self.is_vr_51_model: bool = False + + # Secondary / chain model placeholders + self.is_secondary_model: bool = False + self.is_secondary_model_activated: bool = False + self.secondary_model: Any = None + self.secondary_model_scale: Any = None + self.primary_model_primary_stem: Any = None + self.is_primary_model_primary_stem_only: bool = False + self.is_primary_model_secondary_stem_only: bool = False + self.is_pre_proc_model: bool = False + self.pre_proc_model_activated: bool = False + self.pre_proc_model: Any = None + self.vocal_split_model: Any = None + self.is_vocal_split_model: bool = False + self.is_vocal_split_model_activated: bool = False + self.is_save_inst_vocal_splitter: bool = False + self.is_inst_only_voc_splitter: bool = False + self.is_save_vocal_only: bool = False + self.is_karaoke: bool = False + self.is_bv_model: bool = False + self.bv_model_rebalance: float = 0.0 + self.is_sec_bv_rebalance: bool = False + self.deverb_vocal_opt: Any = None + self.DENOISER_MODEL: Any = None + self.DEVERBER_MODEL: Any = None + + # Ensemble flags + self.is_ensemble_mode: bool = False + self.is_4_stem_ensemble: bool = False + self.is_multi_stem_ensemble: bool = False + self.ensemble_primary_stem: Any = None + self.ensemble_secondary_stem: Any = None + + # MDX specific + self.is_mdx_ckpt: bool = False + self.is_mdx_c: bool = False + self.mdx_c_configs: Any = None + self.mdx_dim_f_set: int = 2048 + self.mdx_dim_t_set: int = 8 + self.mdx_n_fft_scale_set: int = 6144 + self.compensate: Optional[float] = 1.035 + self.mdxnet_stem_select: str = "Vocals" + self.mdx_batch_size: int = 1 + self.mdx_segment_size: int = 256 + self.overlap: float = 0.25 + self.overlap_mdx: Any = "Default" + self.overlap_mdx23: int = 8 + self.chunks: int = 0 + self.margin: int = 44100 + self.is_adaptive_chunk: bool = False + self.is_denoise: bool = False + self.is_denoise_model: bool = False + self.is_mdx_combine_stems: bool = False + self.is_mdx_c_seg_def: bool = False + self.mixer_path: str = str(MDX_MODELS_DIR / "mixer_val.ckpt") + + # VR specific + self.vr_model_param: Any = None + self.is_high_end_process: str = "mirroring" + self.is_tta: bool = False + self.is_post_process: bool = False + self.window_size: int = 512 + self.batch_size: int = 1 + self.crop_size: int = 256 + self.post_process_threshold: float = 0.2 + self.aggression_setting: float = 0.05 + + # Demucs specific + self.demucs_version: str = DEMUCS_V4 + self.demucs_stems: str = ALL_STEMS + self.demucs_source_list: List[str] = list(DEMUCS_4_SOURCE) + self.demucs_source_map: Dict[str, Any] = dict(DEMUCS_4_SOURCE_MAPPER) + self.demucs_stem_count: int = 4 + self.shifts: int = 2 + self.segment: Any = "Default" + self.is_split_mode: bool = True + self.is_chunk_demucs: bool = False + self.is_demucs_combine_stems: bool = False + self.secondary_model_4_stem: List[Any] = [] + self.secondary_model_4_stem_scale: List[Any] = [] + self.is_demucs_pre_proc_model_inst_mix: bool = False + + for k, v in kwargs.items(): + setattr(self, k, v) + + +def _load_json(path: Path) -> Dict[str, Any]: + if path.is_file(): + try: + with path.open("r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return {} + return {} + + +def list_available_models() -> List[Dict[str, Any]]: + """Scan and list all installed models across MDX, VR, and Demucs architectures.""" + results: List[Dict[str, Any]] = [] + + # MDX Models + if MDX_MODELS_DIR.is_dir(): + for file in MDX_MODELS_DIR.iterdir(): + if file.is_file() and file.suffix.lower() in [ONNX, CKPT]: + results.append({ + "name": file.name, + "basename": file.stem, + "path": str(file), + "architecture": MDX_ARCH_TYPE, + "size_mb": round(file.stat().st_size / (1024 * 1024), 2), + "format": file.suffix.lower(), + }) + + # VR Models + if VR_MODELS_DIR.is_dir(): + for file in VR_MODELS_DIR.iterdir(): + if file.is_file() and file.suffix.lower() in [PTH]: + results.append({ + "name": file.name, + "basename": file.stem, + "path": str(file), + "architecture": VR_ARCH_TYPE, + "size_mb": round(file.stat().st_size / (1024 * 1024), 2), + "format": file.suffix.lower(), + }) + + # Demucs Models + if DEMUCS_MODELS_DIR.is_dir(): + for file in DEMUCS_MODELS_DIR.iterdir(): + if file.is_file() and file.suffix.lower() in [".th", YAML]: + results.append({ + "name": file.name, + "basename": file.stem, + "path": str(file), + "architecture": DEMUCS_ARCH_TYPE, + "size_mb": round(file.stat().st_size / (1024 * 1024), 2), + "format": file.suffix.lower(), + }) + if DEMUCS_NEWER_REPO_DIR.is_dir(): + for file in DEMUCS_NEWER_REPO_DIR.iterdir(): + if file.is_file() and file.suffix.lower() in [YAML, ".th"]: + results.append({ + "name": file.name, + "basename": file.stem, + "path": str(file), + "architecture": DEMUCS_ARCH_TYPE, + "size_mb": round(file.stat().st_size / (1024 * 1024), 2), + "format": file.suffix.lower(), + }) + + return results + + +def resolve_model_file(model_identifier: str) -> Tuple[Path, str]: + """Resolve a model name or file path to an existing file and its architecture.""" + # 1. Direct path check + direct_path = Path(model_identifier).resolve() + if direct_path.is_file(): + ext = direct_path.suffix.lower() + if ext in [ONNX, CKPT]: + return direct_path, MDX_ARCH_TYPE + if ext in [PTH]: + return direct_path, VR_ARCH_TYPE + if ext in [YAML, ".th"]: + return direct_path, DEMUCS_ARCH_TYPE + return direct_path, MDX_ARCH_TYPE + + # 2. Search by filename or stem in MDX_Net_Models + for candidate in [MDX_MODELS_DIR / model_identifier, MDX_MODELS_DIR / f"{model_identifier}.onnx", MDX_MODELS_DIR / f"{model_identifier}.ckpt"]: + if candidate.is_file(): + return candidate, MDX_ARCH_TYPE + + # 3. Search in VR_Models + for candidate in [VR_MODELS_DIR / model_identifier, VR_MODELS_DIR / f"{model_identifier}.pth"]: + if candidate.is_file(): + return candidate, VR_ARCH_TYPE + + # 4. Search in Demucs_Models + for candidate in [DEMUCS_MODELS_DIR / model_identifier, DEMUCS_NEWER_REPO_DIR / model_identifier, DEMUCS_NEWER_REPO_DIR / f"{model_identifier}.yaml"]: + if candidate.is_file(): + return candidate, DEMUCS_ARCH_TYPE + + # 5. Fallback search through all available models + for m in list_available_models(): + if model_identifier.lower() in [m["name"].lower(), m["basename"].lower()]: + return Path(m["path"]), m["architecture"] + + raise FileNotFoundError(f"Model '{model_identifier}' could not be located in models directories.") + + +def build_headless_model_data( + model_path: Path, + architecture: str, + precision: str = MODEL_PRECISION_DEFAULT, + device: str = "cuda:0", + segment_size: Optional[int] = None, + overlap: Optional[float] = None, + batch_size: int = 1, +) -> HeadlessModelData: + """Build a complete HeadlessModelData object configured for the requested model.""" + model_path = model_path.resolve() + model_name = model_path.name + model_basename = model_path.stem + model_hash = compute_uvr_hash(model_path) + + is_gpu = "cuda" in device.lower() or "directml" in device.lower() + device_set = device.split(":")[-1] if ":" in device else "0" + + norm_precision = normalize_model_precision(precision) + + data = HeadlessModelData( + process_method=architecture, + model_path=str(model_path), + model_name=model_name, + model_basename=model_basename, + model_hash=model_hash, + model_precision=norm_precision, + device_set=device_set, + is_gpu_conversion=0 if is_gpu else -1, + ) + + if architecture == MDX_ARCH_TYPE: + data.is_mdx_ckpt = model_name.endswith(CKPT) + if segment_size is not None: + data.mdx_segment_size = segment_size + if overlap is not None: + data.overlap_mdx = overlap + data.mdx_batch_size = batch_size + + # Check for model settings in hash json, then model_data.json + hash_file = MDX_HASH_DIR / f"{model_hash}.json" + hash_data = _load_json(hash_file) + if not hash_data: + master_data = _load_json(MDX_HASH_JSON) + hash_data = master_data.get(model_hash, {}) + + if hash_data: + if "config_yaml" in hash_data: + data.is_mdx_c = True + cfg_path = MDX_C_CONFIG_PATH / hash_data["config_yaml"] + if cfg_path.is_file(): + with cfg_path.open("r", encoding="utf-8") as f: + data.mdx_c_configs = ConfigDict(yaml.load(f, Loader=yaml.FullLoader)) + target = getattr(data.mdx_c_configs.training, "target_instrument", None) + if target: + data.primary_stem = target + else: + stems = getattr(data.mdx_c_configs.training, "instruments", ["Vocals"]) + data.primary_stem = stems[0] + else: + data.compensate = hash_data.get("compensate", 1.035) + data.mdx_dim_f_set = hash_data.get("mdx_dim_f_set", 2048) + data.mdx_dim_t_set = hash_data.get("mdx_dim_t_set", 8) + data.mdx_n_fft_scale_set = hash_data.get("mdx_n_fft_scale_set", 6144) + data.primary_stem = hash_data.get("primary_stem", "Instrumental") + data.primary_stem_native = data.primary_stem + data.is_karaoke = hash_data.get("is_karaoke", False) + + data.secondary_stem = secondary_stem(data.primary_stem) + + elif architecture == VR_ARCH_TYPE: + hash_file = VR_HASH_DIR / f"{model_hash}.json" + hash_data = _load_json(hash_file) + if not hash_data: + master_data = _load_json(VR_HASH_JSON) + hash_data = master_data.get(model_hash, {}) + + if hash_data: + data.primary_stem = hash_data.get("primary_stem", "Vocals") + data.secondary_stem = secondary_stem(data.primary_stem) + vr_param_file = VR_PARAM_DIR / f"{hash_data.get('vr_model_param', '1_HP-UVR')}.json" + if vr_param_file.is_file(): + from lib_v5.vr_network.model_param_init import ModelParameters + data.vr_model_param = ModelParameters(str(vr_param_file)) + data.model_samplerate = data.vr_model_param.param.get("sr", 44100) + + if "nout" in hash_data and "nout_lstm" in hash_data: + data.model_capacity = (hash_data["nout"], hash_data["nout_lstm"]) + data.is_vr_51_model = True + + elif architecture == DEMUCS_ARCH_TYPE: + data.demucs_version = DEMUCS_V4 + for ver, tag in DEMUCS_VERSION_MAPPER.items(): + if tag in model_name: + data.demucs_version = ver + + if DEMUCS_UVR_MODEL in model_name: + data.demucs_source_list = list(DEMUCS_2_SOURCE) + data.demucs_source_map = dict(DEMUCS_2_SOURCE_MAPPER) + data.demucs_stem_count = 2 + else: + data.demucs_source_list = list(DEMUCS_4_SOURCE) + data.demucs_source_map = dict(DEMUCS_4_SOURCE_MAPPER) + data.demucs_stem_count = 4 + + return data diff --git a/uvr_cli/runner.py b/uvr_cli/runner.py new file mode 100644 index 0000000..9ae51c5 --- /dev/null +++ b/uvr_cli/runner.py @@ -0,0 +1,118 @@ +"""Headless inference runner bridging directly to UVR's separate.py backends.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple + +import soundfile as sf + +from gui_data.constants import DEMUCS_ARCH_TYPE, MDX_ARCH_TYPE, VR_ARCH_TYPE +from .model_resolver import HeadlessModelData +from separate import SeperateDemucs, SeperateMDX, SeperateMDXC, SeperateVR + + +def get_audio_metadata(audio_path: str | Path) -> Dict[str, Any]: + """Inspect audio file duration, sample rate, channels, and format.""" + path = Path(audio_path).resolve() + try: + info = sf.info(str(path)) + return { + "file_name": path.name, + "duration_sec": round(info.duration, 4), + "samplerate": info.samplerate, + "channels": info.channels, + "format": info.format, + "subtype": info.subtype, + "size_bytes": path.stat().st_size, + } + except Exception: + # Fallback if soundfile info fails + return { + "file_name": path.name, + "duration_sec": 0.0, + "samplerate": 44100, + "channels": 2, + "format": "UNKNOWN", + "subtype": "UNKNOWN", + "size_bytes": path.stat().st_size if path.is_file() else 0, + } + + +def execute_inference( + model_data: HeadlessModelData, + audio_path: str | Path, + export_dir: str | Path, + progress_callback: Optional[Callable[[float, float], None]] = None, + console_callback: Optional[Callable[[str, str], None]] = None, +) -> Tuple[List[Path], bool]: + """Execute UVR separation directly using core separate.py classes. + + Returns: + (generated_stems, is_valid) + """ + audio_path = Path(audio_path).resolve() + export_path = Path(export_dir).resolve() + export_path.mkdir(parents=True, exist_ok=True) + + if not audio_path.is_file(): + raise FileNotFoundError(f"Input audio file not found: {audio_path}") + + # Snapshot existing files in export directory to detect new outputs + existing_files = set(export_path.glob("*")) + + def default_progress(step: float, inference_iterations: float = 0.0) -> None: + if progress_callback: + progress_callback(step, inference_iterations) + + def default_console(text: str, base_text: str = "") -> None: + if console_callback: + console_callback(text, base_text) + + process_data: Dict[str, Any] = { + "model_data": model_data, + "export_path": str(export_path), + "audio_file_base": audio_path.stem, + "audio_file": str(audio_path), + "set_progress_bar": default_progress, + "write_to_console": default_console, + "process_iteration": 0, + "cached_source_callback": lambda process_method, model_name=None: (None, None), + "cached_model_source_holder": lambda *args, **kwargs: None, + "list_all_models": [], + "is_ensemble_master": False, + "is_4_stem_ensemble": False, + } + + # Instantiate the corresponding UVR separator + if model_data.process_method == VR_ARCH_TYPE: + separator = SeperateVR(model_data, process_data) + elif model_data.process_method == MDX_ARCH_TYPE: + if model_data.is_mdx_c: + separator = SeperateMDXC(model_data, process_data) + else: + separator = SeperateMDX(model_data, process_data) + elif model_data.process_method == DEMUCS_ARCH_TYPE: + separator = SeperateDemucs(model_data, process_data) + else: + raise ValueError(f"Unsupported architecture: {model_data.process_method}") + + # Run inference directly + separator.seperate() + + # Detect generated stems + all_current_files = set(export_path.glob("*")) + new_files = [f for f in (all_current_files - existing_files) if f.is_file()] + + # Validate output files + is_valid = True + if not new_files: + is_valid = False + else: + for stem_file in new_files: + if stem_file.stat().st_size == 0: + is_valid = False + break + + return sorted(new_files), is_valid diff --git a/uvr_cli/telemetry.py b/uvr_cli/telemetry.py new file mode 100644 index 0000000..288e541 --- /dev/null +++ b/uvr_cli/telemetry.py @@ -0,0 +1,249 @@ +"""Hardware telemetry and performance monitoring for UVR benchmarks. + +Provides background resource sampling (CPU, GPU, VRAM, Power) and system +environment discovery, with graceful fallback if NVML or CUDA is unavailable. +""" + +from __future__ import annotations + +import os +import platform +import threading +import time +from typing import Any, Dict, List, Optional + +import psutil +import torch + +try: + import onnxruntime as ort + ORT_AVAILABLE = True + ORT_VERSION = ort.__version__ +except Exception: + ORT_AVAILABLE = False + ORT_VERSION = "N/A" + +try: + import pynvml + pynvml.nvmlInit() + NVML_AVAILABLE = True +except Exception: + NVML_AVAILABLE = False + + +def sync_cuda(device_index: int = 0) -> None: + """Synchronize CUDA device if available.""" + if torch.cuda.is_available(): + try: + torch.cuda.synchronize(device_index) + except Exception: + pass + + +def collect_system_info(device_index: int = 0) -> Dict[str, Any]: + """Collect hardware and software runtime environment details.""" + gpu_name = None + driver_version = None + total_gpu_vram_mb = 0.0 + + if torch.cuda.is_available(): + try: + gpu_name = torch.cuda.get_device_name(device_index) + except Exception: + pass + + if NVML_AVAILABLE: + try: + handle = pynvml.nvmlDeviceGetHandleByIndex(device_index) + if not gpu_name: + gpu_name = pynvml.nvmlDeviceGetName(handle) + if isinstance(gpu_name, bytes): + gpu_name = gpu_name.decode("utf-8") + driver_version = pynvml.nvmlSystemGetDriverVersion() + if isinstance(driver_version, bytes): + driver_version = driver_version.decode("utf-8") + mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + total_gpu_vram_mb = round(mem_info.total / (1024 * 1024), 2) + except Exception: + pass + + return { + "os": platform.platform(), + "cpu": platform.processor() or platform.machine() or "Unknown CPU", + "cpu_count_logical": psutil.cpu_count(logical=True) or 0, + "cpu_count_physical": psutil.cpu_count(logical=False) or 0, + "system_ram_gb": round(psutil.virtual_memory().total / (1024**3), 2), + "gpu_name": gpu_name or "N/A", + "total_gpu_vram_mb": total_gpu_vram_mb, + "driver_version": driver_version or "N/A", + "cuda_available": torch.cuda.is_available(), + "cuda_version": torch.version.cuda or "N/A", + "torch_version": torch.__version__, + "onnxruntime_version": ORT_VERSION, + "python_version": platform.python_version(), + } + + +class TelemetrySampler: + """Background sampling monitor measuring CPU, RAM, GPU utilization, VRAM, and power.""" + + def __init__(self, device_index: int = 0, interval_sec: float = 0.1) -> None: + self.device_index = device_index + self.interval_sec = interval_sec + self._stop_event = threading.Event() + self._thread: Optional[threading.Thread] = None + + self._process = psutil.Process(os.getpid()) + self._nvml_handle: Any = None + if NVML_AVAILABLE: + try: + self._nvml_handle = pynvml.nvmlDeviceGetHandleByIndex(self.device_index) + except Exception: + self._nvml_handle = None + + # Sample storage + self.process_cpu_samples: List[float] = [] + self.system_cpu_samples: List[float] = [] + self.process_ram_samples: List[float] = [] + self.gpu_util_samples: List[float] = [] + self.device_vram_samples: List[float] = [] + self.power_samples: List[float] = [] + + self.baseline_device_vram_mb: float = 0.0 + self.baseline_torch_allocated_mb: float = 0.0 + + def start(self) -> None: + """Start the background sampling thread and reset peak GPU stats.""" + # Initial CPU reading primes psutil + try: + self._process.cpu_percent() + psutil.cpu_percent() + except Exception: + pass + + if torch.cuda.is_available(): + try: + sync_cuda(self.device_index) + torch.cuda.reset_peak_memory_stats(self.device_index) + self.baseline_torch_allocated_mb = round( + torch.cuda.memory_allocated(self.device_index) / (1024 * 1024), 2 + ) + except Exception: + pass + + if self._nvml_handle: + try: + mem = pynvml.nvmlDeviceGetMemoryInfo(self._nvml_handle) + self.baseline_device_vram_mb = round(mem.used / (1024 * 1024), 2) + except Exception: + pass + + self._stop_event.clear() + self._thread = threading.Thread(target=self._sample_loop, daemon=True, name="TelemetrySampler") + self._thread.start() + + def _sample_loop(self) -> None: + while not self._stop_event.is_set(): + try: + # CPU & RAM + proc_cpu = self._process.cpu_percent() + sys_cpu = psutil.cpu_percent() + proc_ram = self._process.memory_info().rss / (1024 * 1024) + + self.process_cpu_samples.append(proc_cpu) + self.system_cpu_samples.append(sys_cpu) + self.process_ram_samples.append(proc_ram) + + # NVML metrics + if self._nvml_handle: + rates = pynvml.nvmlDeviceGetUtilizationRates(self._nvml_handle) + self.gpu_util_samples.append(float(rates.gpu)) + + mem = pynvml.nvmlDeviceGetMemoryInfo(self._nvml_handle) + self.device_vram_samples.append(mem.used / (1024 * 1024)) + + try: + power_w = pynvml.nvmlDeviceGetPowerUsage(self._nvml_handle) / 1000.0 + self.power_samples.append(power_w) + except Exception: + pass + except Exception: + pass + + time.sleep(self.interval_sec) + + def stop(self) -> Dict[str, Any]: + """Stop sampling and return aggregated hardware metrics.""" + self._stop_event.set() + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=1.0) + + sync_cuda(self.device_index) + + torch_peak_alloc = 0.0 + torch_peak_reserved = 0.0 + if torch.cuda.is_available(): + try: + torch_peak_alloc = round( + torch.cuda.max_memory_allocated(self.device_index) / (1024 * 1024), 2 + ) + torch_peak_reserved = round( + torch.cuda.max_memory_reserved(self.device_index) / (1024 * 1024), 2 + ) + except Exception: + pass + + avg_proc_cpu = ( + round(sum(self.process_cpu_samples) / len(self.process_cpu_samples), 2) + if self.process_cpu_samples + else 0.0 + ) + peak_proc_cpu = round(max(self.process_cpu_samples), 2) if self.process_cpu_samples else 0.0 + + avg_sys_cpu = ( + round(sum(self.system_cpu_samples) / len(self.system_cpu_samples), 2) + if self.system_cpu_samples + else 0.0 + ) + peak_sys_cpu = round(max(self.system_cpu_samples), 2) if self.system_cpu_samples else 0.0 + + peak_proc_ram = ( + round(max(self.process_ram_samples), 2) if self.process_ram_samples else 0.0 + ) + + avg_gpu_util = ( + round(sum(self.gpu_util_samples) / len(self.gpu_util_samples), 2) + if self.gpu_util_samples + else 0.0 + ) + peak_gpu_util = round(max(self.gpu_util_samples), 2) if self.gpu_util_samples else 0.0 + + peak_device_vram = ( + round(max(self.device_vram_samples), 2) + if self.device_vram_samples + else (self.baseline_device_vram_mb or torch_peak_alloc) + ) + + avg_power = ( + round(sum(self.power_samples) / len(self.power_samples), 2) + if self.power_samples + else 0.0 + ) + peak_power = round(max(self.power_samples), 2) if self.power_samples else 0.0 + + return { + "samples_count": len(self.process_cpu_samples), + "avg_process_cpu_percent": avg_proc_cpu, + "peak_process_cpu_percent": peak_proc_cpu, + "avg_system_cpu_percent": avg_sys_cpu, + "peak_system_cpu_percent": peak_sys_cpu, + "peak_process_ram_mb": peak_proc_ram, + "avg_gpu_utilization_percent": avg_gpu_util, + "peak_gpu_utilization_percent": peak_gpu_util, + "baseline_device_vram_mb": self.baseline_device_vram_mb, + "peak_device_vram_mb": peak_device_vram, + "torch_peak_allocated_vram_mb": torch_peak_alloc, + "torch_peak_reserved_vram_mb": torch_peak_reserved, + "avg_power_watts": avg_power, + "peak_power_watts": peak_power, + }