mirror of
https://github.com/RevLayDust/RLD-UVR-Opt.git
synced 2026-11-20 13:07:58 +09:00
feat(engine): add TensorRT-RTX execution provider backend with in-memory FP16 downcast
- Implement TensorRT-RTX backend via onnxruntime-ep-nv-tensorrt-rtx plugin in engine/tensorrt_rtx/ - Support in-memory FP16 downcast for MDX-Net ONNX models with zero disk writes - Implement automatic fallback chain: TensorRT-RTX -> CUDA -> CPU - Add GUI execution provider selection with dynamic hardware gating - Add CLI provider flags and model resolver support for trt_rtx - Add install/requirements_trt_rtx.txt and stack profile tracking - Add test suite in tests/test_tensorrt_rtx.py and test_uvr_cli.py - Update run_uvr.bat to execute UVR.py via native CMD
This commit is contained in:
1 parent
af2d3ccb28
commit
90050a048e
17 files changed
+1197
-64
No files matched your search
+6
-1
@@ -9,8 +9,11 @@ __pycache__/
|
||||
.vscode/
|
||||
venv/
|
||||
.venv/
|
||||
venv_trt_test/
|
||||
.agents/
|
||||
benchmark_cache/
|
||||
models/**/trt_cache/
|
||||
install_logs/
|
||||
|
||||
output/*
|
||||
!output/*.txt
|
||||
@@ -21,5 +24,7 @@ error.log
|
||||
CONTRIBUTING.md
|
||||
rubberband.exe
|
||||
ffmpeg.exe
|
||||
sndfile.dll
|
||||
|
||||
install/install_packages.sh
|
||||
install/install_packages.sh
|
||||
install/install_record.json
|
||||
@@ -373,6 +373,8 @@ class ModelData():
|
||||
self.mdx_n_fft_scale_set = None
|
||||
self.wav_type_set = root.wav_type_set#
|
||||
self.device_set = device_set.split(':')[-1].strip() if ':' in device_set else device_set
|
||||
ep_var = getattr(root, 'execution_provider_var', None)
|
||||
self.execution_provider = ep_var.get() if ep_var is not None else 'CUDAExecutionProvider'
|
||||
self.mp3_bit_set = root.mp3_bit_set_var.get()
|
||||
self.save_format = root.save_format_var.get()
|
||||
self.is_invert_spec = root.is_invert_spec_var.get()#
|
||||
@@ -3173,23 +3175,72 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk):
|
||||
|
||||
def fill_gpu_list(self):
|
||||
try:
|
||||
if cuda_available:
|
||||
from engine.tensorrt_rtx import is_ort_gpu_available
|
||||
ort_gpu = is_ort_gpu_available()
|
||||
except Exception:
|
||||
ort_gpu = False
|
||||
|
||||
try:
|
||||
if cuda_available and ort_gpu:
|
||||
self.cuda_device_list = [f"{torch.cuda.get_device_properties(i).name}:{i}" for i in range(torch.cuda.device_count())]
|
||||
self.cuda_device_list.insert(0, DEFAULT)
|
||||
#print(self.cuda_device_list)
|
||||
|
||||
# if directml_available:
|
||||
# self.opencl_list = [f"{torch_directml.device_name(i)}:{i}" for i in range(torch_directml.device_count())]
|
||||
# self.opencl_list.insert(0, DEFAULT)
|
||||
else:
|
||||
self.cuda_device_list = [DEFAULT]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
self.cuda_device_list = [DEFAULT]
|
||||
|
||||
# if is_cuda_only:
|
||||
# self.is_use_opencl_var.set(False)
|
||||
|
||||
check_gpu_list = self.cuda_device_list#self.opencl_list if is_opencl_only or self.is_use_opencl_var.get() else self.cuda_device_list
|
||||
check_gpu_list = self.cuda_device_list
|
||||
if not self.device_set_var.get() in check_gpu_list:
|
||||
self.device_set_var.set(DEFAULT)
|
||||
self.update_execution_provider_ui()
|
||||
|
||||
def update_execution_provider_ui(self, ep_option=None, ep_label=None):
|
||||
"""Update Execution Provider selection and widget state (task 2)."""
|
||||
from engine.tensorrt_rtx import get_supported_execution_providers
|
||||
|
||||
is_gpu_selected = self.device_set_var.get() != DEFAULT
|
||||
dev_id = 0
|
||||
if is_gpu_selected:
|
||||
try:
|
||||
dev_id = int(self.device_set_var.get().split(':')[-1].strip())
|
||||
except Exception:
|
||||
dev_id = 0
|
||||
|
||||
supported = get_supported_execution_providers(has_gpu_selected=is_gpu_selected, device_id=dev_id)
|
||||
self.execution_provider_list = supported
|
||||
|
||||
if ep_option is not None:
|
||||
ep_option['values'] = self.execution_provider_list
|
||||
if not is_gpu_selected:
|
||||
# Rule 3: Lock the Execution Provider option (greyed out) until a GPU Device is chosen
|
||||
ep_option.configure(state='disabled')
|
||||
else:
|
||||
# Rule 2: Enable and allow selection
|
||||
ep_option.configure(state='readonly')
|
||||
|
||||
if ep_label is not None:
|
||||
if not is_gpu_selected:
|
||||
try:
|
||||
ep_label.configure(state='disabled', foreground='gray')
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
ep_label.configure(state='normal', foreground='#13849f')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Validate stored provider ONLY when a GPU Device is selected (Rule 2).
|
||||
# While locked (Rule 3), preserve the user's stored choice intact.
|
||||
if is_gpu_selected:
|
||||
current_ep = self.execution_provider_var.get()
|
||||
if current_ep not in self.execution_provider_list:
|
||||
default_ep = (
|
||||
'CUDAExecutionProvider' if 'CUDAExecutionProvider' in self.execution_provider_list
|
||||
else (self.execution_provider_list[0] if self.execution_provider_list else 'CUDAExecutionProvider')
|
||||
)
|
||||
self.execution_provider_var.set(default_ep)
|
||||
|
||||
def loop_gpu_list(self, option_menu:ComboBoxMenu, menu_name, option_list):
|
||||
option_menu['values'] = option_list
|
||||
@@ -3346,7 +3397,9 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk):
|
||||
#if not is_choose_arch:
|
||||
self.vocal_splitter_Button_opt(settings_menu, settings_menu_format_Frame, width=SETTINGS_BUT_WIDTH-2, pady=MENU_PADDING_4)
|
||||
|
||||
if not is_macos and self.is_gpu_available:
|
||||
from engine.tensorrt_rtx import is_ort_gpu_available
|
||||
has_ort_gpu = is_ort_gpu_available()
|
||||
if not is_macos and self.is_gpu_available and has_ort_gpu:
|
||||
gpu_list_options = lambda:self.loop_gpu_list(device_set_Option, 'gpudevice', self.cuda_device_list)#self.opencl_list if is_opencl_only or self.is_use_opencl_var.get() else self.cuda_device_list)
|
||||
device_set_Label = self.menu_title_LABEL_SET(settings_menu_format_Frame, CUDA_NUM_TEXT)
|
||||
device_set_Label.grid(pady=MENU_PADDING_2)
|
||||
@@ -3355,6 +3408,18 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk):
|
||||
device_set_Option.grid(padx=20,pady=MENU_PADDING_1)
|
||||
gpu_list_options()
|
||||
self.help_hints(device_set_Label, text=IS_CUDA_SELECT_HELP)
|
||||
|
||||
execution_provider_Label = self.menu_title_LABEL_SET(settings_menu_format_Frame, EXECUTION_PROVIDER_TEXT)
|
||||
execution_provider_Label.grid(pady=MENU_PADDING_2)
|
||||
|
||||
execution_provider_Option = ComboBoxMenu(settings_menu_format_Frame, textvariable=self.execution_provider_var, values=self.execution_provider_list, width=GEN_SETTINGS_WIDTH+1)
|
||||
execution_provider_Option.grid(padx=20, pady=MENU_PADDING_1)
|
||||
self.help_hints(execution_provider_Label, text=IS_EXECUTION_PROVIDER_HELP)
|
||||
|
||||
update_ep_callback = lambda *args: self.update_execution_provider_ui(execution_provider_Option, execution_provider_Label)
|
||||
device_set_Option.bind('<<ComboboxSelected>>', lambda e: update_ep_callback())
|
||||
update_ep_callback()
|
||||
|
||||
|
||||
model_precision_Label = self.menu_title_LABEL_SET(settings_menu_format_Frame, 'Precision Mode Preset')
|
||||
model_precision_Label.grid(pady=MENU_PADDING_2)
|
||||
@@ -6852,6 +6917,8 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk):
|
||||
self.save_format_var = tk.StringVar(value=data['save_format'])
|
||||
self.wav_type_set_var = tk.StringVar(value=data['wav_type_set'])#
|
||||
self.device_set_var = tk.StringVar(value=data['device_set'])#
|
||||
self.execution_provider_var = tk.StringVar(value=data.get('execution_provider', 'CUDAExecutionProvider'))
|
||||
self.execution_provider_list = ['CUDAExecutionProvider']
|
||||
self.user_code_var = tk.StringVar(value=data['user_code'])
|
||||
self.is_gpu_conversion_var = tk.BooleanVar(value=data['is_gpu_conversion'])
|
||||
self.model_precision_var = tk.StringVar(value=normalize_model_precision(data.get('model_precision')))
|
||||
@@ -6998,6 +7065,7 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk):
|
||||
self.save_format_var.set(loaded_setting['save_format'])
|
||||
self.wav_type_set_var.set(loaded_setting['wav_type_set'])#
|
||||
self.device_set_var.set(loaded_setting['device_set'])#
|
||||
self.execution_provider_var.set(loaded_setting.get('execution_provider', 'CUDAExecutionProvider'))
|
||||
self.user_code_var.set(loaded_setting['user_code'])
|
||||
self.phase_option_var.set(loaded_setting['phase_option'])#
|
||||
self.phase_shifts_var.set(loaded_setting['phase_shifts'])#
|
||||
@@ -7138,6 +7206,7 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk):
|
||||
'save_format': self.save_format_var.get(),
|
||||
'wav_type_set': self.wav_type_set_var.get(),#
|
||||
'device_set': self.device_set_var.get(),#
|
||||
'execution_provider': self.execution_provider_var.get(),
|
||||
'user_code': self.user_code_var.get(),
|
||||
'help_hints_var': self.help_hints_var.get(),
|
||||
'set_vocal_splitter': self.set_vocal_splitter_var.get(),
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
VERSION = 'v0.1.0-beta.1'
|
||||
BUILD_ID = 'RLUV_20261006_1949' # represent RLUV_yyyymmdd_HHMM (year, month, day, hour, minute)
|
||||
BUILD_ID = 'RLUV_20261009_1604' # represent RLUV_yyyymmdd_HHMM (year, month, day, hour, minute)
|
||||
PATCH = BUILD_ID
|
||||
PATCH_MAC = 'Not yet available'
|
||||
PATCH_LINUX = 'Not yet available'
|
||||
@@ -0,0 +1,39 @@
|
||||
"""TensorRT-RTX execution provider wrapper package for Ultimate Vocal Remover."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .environment import (
|
||||
PROVIDER_CPU,
|
||||
PROVIDER_CUDA,
|
||||
PROVIDER_TRT_PLUGIN,
|
||||
PROVIDER_TRT_RTX,
|
||||
get_supported_execution_providers,
|
||||
is_ort_gpu_available,
|
||||
is_trt_rtx_available,
|
||||
is_trt_rtx_installed,
|
||||
register_trt_rtx_libraries,
|
||||
)
|
||||
from .session import (
|
||||
DEFAULT_CUDA_OPTIONS,
|
||||
build_trt_options,
|
||||
build_trt_rtx_options,
|
||||
create_trt_rtx_session,
|
||||
get_default_cache_dir,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"PROVIDER_CPU",
|
||||
"PROVIDER_CUDA",
|
||||
"PROVIDER_TRT_PLUGIN",
|
||||
"PROVIDER_TRT_RTX",
|
||||
"DEFAULT_CUDA_OPTIONS",
|
||||
"is_trt_rtx_installed",
|
||||
"is_trt_rtx_available",
|
||||
"is_ort_gpu_available",
|
||||
"get_supported_execution_providers",
|
||||
"build_trt_options",
|
||||
"build_trt_rtx_options",
|
||||
"create_trt_rtx_session",
|
||||
"get_default_cache_dir",
|
||||
"register_trt_rtx_libraries",
|
||||
]
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Environment detection and runtime provider discovery for TensorRT-RTX."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
# Execution Provider names
|
||||
PROVIDER_TRT_RTX = "NvTensorRTRTXExecutionProvider"
|
||||
PROVIDER_TRT_PLUGIN = "nv_tensorrt_rtx"
|
||||
PROVIDER_CUDA = "CUDAExecutionProvider"
|
||||
PROVIDER_CPU = "CPUExecutionProvider"
|
||||
|
||||
_PLUGIN_REGISTERED = False
|
||||
|
||||
|
||||
def register_trt_rtx_libraries() -> None:
|
||||
"""Register TensorRT-RTX DLL directories and plugin library with ONNX Runtime."""
|
||||
global _PLUGIN_REGISTERED
|
||||
if _PLUGIN_REGISTERED:
|
||||
return
|
||||
|
||||
# 1. On Windows, ensure bundled pip plugin library DLL directory is in search path
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
import site
|
||||
for site_dir in site.getsitepackages():
|
||||
rtx_dir = os.path.join(site_dir, "onnxruntime_ep_nv_tensorrt_rtx")
|
||||
if os.path.isdir(rtx_dir):
|
||||
try:
|
||||
os.add_dll_directory(rtx_dir)
|
||||
except (AttributeError, OSError) as exc:
|
||||
print(f"[BACKEND] Warning: Failed to add DLL directory '{rtx_dir}': {exc}")
|
||||
os.environ["PATH"] = rtx_dir + os.pathsep + os.environ.get("PATH", "")
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Warning: Error configuring Windows DLL search paths: {exc}")
|
||||
|
||||
# 2. Register the standalone EP ABI plugin with ONNX Runtime
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
if hasattr(ort, "register_execution_provider_library"):
|
||||
try:
|
||||
import onnxruntime_ep_nv_tensorrt_rtx as trt_ep
|
||||
ep_name = trt_ep.get_ep_name()
|
||||
ep_lib = trt_ep.get_library_path()
|
||||
print(f"[BACKEND] Registering TensorRT-RTX execution provider library '{ep_name}' ({ep_lib})...")
|
||||
ort.register_execution_provider_library(ep_name, ep_lib)
|
||||
print(f"[BACKEND] Successfully registered TensorRT-RTX execution provider '{ep_name}'.")
|
||||
except ImportError:
|
||||
# Standalone plugin package is not installed; normal for standard/CPU installs
|
||||
pass
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Warning: Failed to register TensorRT-RTX library with ONNX Runtime: {exc}")
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Warning: Error checking ONNX Runtime plugin support: {exc}")
|
||||
|
||||
_PLUGIN_REGISTERED = True
|
||||
|
||||
|
||||
def is_trt_rtx_installed() -> bool:
|
||||
"""Check if TensorRT-RTX EP is genuinely registered and available in ONNX Runtime.
|
||||
|
||||
Standard TensorrtExecutionProvider is deliberately NOT treated as TensorRT-RTX.
|
||||
"""
|
||||
register_trt_rtx_libraries()
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
all_provs = ort.get_all_providers() if hasattr(ort, "get_all_providers") else []
|
||||
avail_provs = ort.get_available_providers()
|
||||
|
||||
# Check standalone EP ABI plugin via device enumeration
|
||||
if hasattr(ort, "get_ep_devices"):
|
||||
try:
|
||||
import onnxruntime_ep_nv_tensorrt_rtx as trt_ep
|
||||
plugin_name = trt_ep.get_ep_name()
|
||||
devices = [d for d in ort.get_ep_devices() if getattr(d, "ep_name", "") == plugin_name]
|
||||
if devices:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Error inspecting TensorRT-RTX EP devices: {exc}")
|
||||
|
||||
# Check built-in provider name (NvTensorRTRTXExecutionProvider)
|
||||
if PROVIDER_TRT_RTX in all_provs and PROVIDER_TRT_RTX in avail_provs:
|
||||
return True
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Error checking TensorRT-RTX installation: {exc}")
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def is_trt_rtx_available(device_id: int = 0) -> bool:
|
||||
"""Check whether TensorRT-RTX can genuinely execute on the specified GPU device."""
|
||||
if not torch.cuda.is_available():
|
||||
return False
|
||||
if device_id < 0 or device_id >= torch.cuda.device_count():
|
||||
return False
|
||||
return is_trt_rtx_installed()
|
||||
|
||||
|
||||
def is_ort_gpu_available() -> bool:
|
||||
"""Check whether ONNX Runtime supports any GPU execution provider."""
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
avail = ort.get_available_providers()
|
||||
gpu_providers = (
|
||||
PROVIDER_CUDA,
|
||||
"ROCMExecutionProvider",
|
||||
"MIGraphXExecutionProvider",
|
||||
"DmlExecutionProvider",
|
||||
)
|
||||
if any(p in avail for p in gpu_providers):
|
||||
return True
|
||||
if is_trt_rtx_installed():
|
||||
return True
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Warning: Failed to query ONNX Runtime GPU providers: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def get_supported_execution_providers(has_gpu_selected: bool = False, device_id: int = 0) -> List[str]:
|
||||
"""Return a list of execution providers supported by the current environment and selection.
|
||||
|
||||
Follows GUI Rules:
|
||||
- If no GPU is selected (has_gpu_selected=False): returns [CPUExecutionProvider].
|
||||
- If GPU is selected: auto-detects providers supported by that GPU (TRT-RTX if available, CUDA, CPU).
|
||||
"""
|
||||
providers: List[str] = []
|
||||
if not has_gpu_selected or not torch.cuda.is_available():
|
||||
return [PROVIDER_CPU]
|
||||
|
||||
# GPU is selected: check available providers
|
||||
register_trt_rtx_libraries()
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
avail = ort.get_available_providers()
|
||||
|
||||
# Check TensorRT-RTX capability
|
||||
if is_trt_rtx_available(device_id):
|
||||
providers.append(PROVIDER_TRT_RTX)
|
||||
|
||||
if PROVIDER_CUDA in avail:
|
||||
providers.append(PROVIDER_CUDA)
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] Error discovering supported execution providers: {exc}")
|
||||
|
||||
if not providers:
|
||||
providers.append(PROVIDER_CPU)
|
||||
|
||||
return providers
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Session creation and lifecycle management for TensorRT-RTX with graceful fallback."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from .environment import (
|
||||
PROVIDER_CPU,
|
||||
PROVIDER_CUDA,
|
||||
PROVIDER_TRT_PLUGIN,
|
||||
PROVIDER_TRT_RTX,
|
||||
register_trt_rtx_libraries,
|
||||
)
|
||||
|
||||
DEFAULT_CUDA_OPTIONS = {
|
||||
"arena_extend_strategy": "kSameAsRequested",
|
||||
"cudnn_conv_algo_search": "HEURISTIC",
|
||||
"do_copy_in_default_stream": "True",
|
||||
}
|
||||
|
||||
|
||||
def get_default_cache_dir() -> Path:
|
||||
"""Return default directory for cached TensorRT engine plans."""
|
||||
project_root = Path(__file__).resolve().parents[2]
|
||||
cache_dir = project_root / "models" / "MDX_Net_Models" / "trt_cache"
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
return cache_dir
|
||||
|
||||
|
||||
def build_trt_rtx_options(
|
||||
device_id: int = 0,
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
enable_cuda_graph: bool = False,
|
||||
) -> Dict[str, str]:
|
||||
"""Construct configuration options dictionary for TensorRT-RTX Execution Provider."""
|
||||
target_cache = Path(cache_dir) if cache_dir else get_default_cache_dir()
|
||||
target_cache.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
opts = {
|
||||
"device_id": str(int(device_id)),
|
||||
"nv_runtime_cache_path": str(target_cache),
|
||||
}
|
||||
if enable_cuda_graph:
|
||||
opts["enable_cuda_graph"] = "1"
|
||||
return opts
|
||||
|
||||
|
||||
# Backward-compatibility alias
|
||||
build_trt_options = build_trt_rtx_options
|
||||
|
||||
|
||||
def create_trt_rtx_session(
|
||||
model_input: Union[str, Path, bytes],
|
||||
device_id: int = 0,
|
||||
cuda_options: Optional[Dict[str, Any]] = None,
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
model_name: Optional[str] = None,
|
||||
enable_cuda_graph: bool = False,
|
||||
) -> Any:
|
||||
"""Create an ONNX Runtime InferenceSession preferring TensorRT-RTX with automatic fallback.
|
||||
|
||||
Fallback chain:
|
||||
1. TensorRT-RTX EP (via EP plugin or built-in provider)
|
||||
2. CUDAExecutionProvider
|
||||
3. CPUExecutionProvider
|
||||
"""
|
||||
import onnxruntime as ort
|
||||
|
||||
register_trt_rtx_libraries()
|
||||
cuda_opts = cuda_options or DEFAULT_CUDA_OPTIONS
|
||||
trt_opts = build_trt_rtx_options(
|
||||
device_id=device_id,
|
||||
cache_dir=cache_dir,
|
||||
enable_cuda_graph=enable_cuda_graph,
|
||||
)
|
||||
|
||||
if model_name:
|
||||
display_name = model_name
|
||||
elif isinstance(model_input, (str, Path)):
|
||||
display_name = os.path.basename(str(model_input))
|
||||
else:
|
||||
display_name = f"in-memory-model ({len(model_input) / (1024 ** 2):.2f} MiB)"
|
||||
|
||||
# Strategy 1: Standalone EP ABI plugin (onnxruntime-ep-nv-tensorrt-rtx)
|
||||
try:
|
||||
if hasattr(ort, "get_ep_devices"):
|
||||
import onnxruntime_ep_nv_tensorrt_rtx as trt_ep
|
||||
plugin_ep_name = trt_ep.get_ep_name()
|
||||
ep_devices = [
|
||||
d for d in ort.get_ep_devices()
|
||||
if getattr(d, "ep_name", "") == plugin_ep_name
|
||||
]
|
||||
if ep_devices:
|
||||
# Select the first matching TensorRT-RTX EP device
|
||||
target_dev = ep_devices[0]
|
||||
print(f"[BACKEND] Initializing TensorRT-RTX plugin session for {display_name}...")
|
||||
so = ort.SessionOptions()
|
||||
so.add_provider_for_devices([target_dev], trt_opts)
|
||||
session = ort.InferenceSession(model_input, sess_options=so)
|
||||
active = session.get_providers()[0] if hasattr(session, "get_providers") else PROVIDER_TRT_RTX
|
||||
print(f"[BACKEND] Active execution provider: {active}")
|
||||
return session
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] TensorRT-RTX plugin session creation failed ({type(exc).__name__}: {exc})")
|
||||
print("[BACKEND] Falling back to CUDAExecutionProvider...")
|
||||
|
||||
# Strategy 2: Built-in NvTensorRTRTXExecutionProvider (if present)
|
||||
avail_provs = ort.get_available_providers()
|
||||
if PROVIDER_TRT_RTX in avail_provs:
|
||||
try:
|
||||
print(f"[BACKEND] Initializing built-in {PROVIDER_TRT_RTX} session for {display_name}...")
|
||||
providers: List[Union[str, Tuple[str, Dict[str, Any]]]] = [
|
||||
(PROVIDER_TRT_RTX, trt_opts),
|
||||
(PROVIDER_CUDA, cuda_opts),
|
||||
PROVIDER_CPU,
|
||||
]
|
||||
session = ort.InferenceSession(model_input, providers=providers)
|
||||
active = session.get_providers()[0] if hasattr(session, "get_providers") else PROVIDER_TRT_RTX
|
||||
print(f"[BACKEND] Active execution provider: {active}")
|
||||
return session
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] {PROVIDER_TRT_RTX} session creation failed ({type(exc).__name__}: {exc})")
|
||||
print("[BACKEND] Falling back to CUDAExecutionProvider...")
|
||||
|
||||
# Strategy 3: CUDAExecutionProvider fallback
|
||||
if PROVIDER_CUDA in avail_provs:
|
||||
try:
|
||||
print(f"[BACKEND] Creating CUDAExecutionProvider session for {display_name}...")
|
||||
session = ort.InferenceSession(
|
||||
model_input,
|
||||
providers=[(PROVIDER_CUDA, cuda_opts), PROVIDER_CPU],
|
||||
)
|
||||
active = session.get_providers()[0] if hasattr(session, "get_providers") else PROVIDER_CUDA
|
||||
print(f"[BACKEND] Active execution provider: {active}")
|
||||
return session
|
||||
except Exception as exc:
|
||||
print(f"[BACKEND] CUDAExecutionProvider failed ({type(exc).__name__}: {exc})")
|
||||
print("[BACKEND] Falling back to CPUExecutionProvider...")
|
||||
|
||||
# Strategy 4: CPUExecutionProvider fallback
|
||||
print(f"[BACKEND] Creating CPUExecutionProvider session for {display_name}...")
|
||||
session = ort.InferenceSession(model_input, providers=[PROVIDER_CPU])
|
||||
print(f"[BACKEND] Active execution provider: {PROVIDER_CPU}")
|
||||
return session
|
||||
@@ -670,6 +670,7 @@ DEFAULT_DATA = {
|
||||
'save_format': WAV,
|
||||
'wav_type_set': 'PCM_16',
|
||||
'device_set': DEFAULT,
|
||||
'execution_provider': 'CUDAExecutionProvider',
|
||||
'user_code': '',
|
||||
'export_path': '',
|
||||
'input_paths': [],
|
||||
@@ -781,6 +782,7 @@ SETTING_CHECK = ('vr_model',
|
||||
'save_format',
|
||||
'wav_type_set',
|
||||
'device_set',
|
||||
'execution_provider',
|
||||
'user_code',
|
||||
'is_gpu_conversion',
|
||||
'model_precision',
|
||||
@@ -1050,6 +1052,10 @@ IS_TIME_CORRECTION_HELP = ('When checked, the output will retain the original BP
|
||||
SAVE_STEM_ONLY_HELP = 'Allows the user to save only the selected stem.'
|
||||
IS_NORMALIZATION_HELP = 'Normalizes output to prevent clipping.'
|
||||
IS_CUDA_SELECT_HELP = "If you have more than one GPU, you can pick which one to use for processing."
|
||||
EXECUTION_PROVIDER_TEXT = "Execution Provider"
|
||||
IS_EXECUTION_PROVIDER_HELP = ("Select the ONNX Runtime execution provider for accelerated inference:\n"
|
||||
"• CUDAExecutionProvider: Stable & proven baseline for all NVIDIA GPUs.\n"
|
||||
"• NvTensorRTRTXExecutionProvider / TensorrtExecutionProvider: Experimental fast TensorRT-RTX acceleration.")
|
||||
CROP_SIZE_HELP = '**Only compatible with select models only!**\n\n Setting should match training crop-size value. Leave as is if unsure.'
|
||||
IS_TTA_HELP = ('This option performs Test-Time-Augmentation to improve the separation quality.\n\n'
|
||||
'Note: Having this selected will increase the time it takes to complete a conversion')
|
||||
|
||||
@@ -48,6 +48,7 @@ if not exist "%REQ_FILE%" (
|
||||
|
||||
call :check_windows
|
||||
call :detect_gpu
|
||||
call :check_previous_install
|
||||
call :choose_mode
|
||||
call :ensure_uv
|
||||
if errorlevel 1 exit /b 1
|
||||
@@ -58,6 +59,8 @@ if errorlevel 1 exit /b 1
|
||||
|
||||
if /i "%INSTALL_MODE%"=="CPU" (
|
||||
call :install_cpu_stack
|
||||
) else if /i "%INSTALL_MODE%"=="TENSORRT_RTX" (
|
||||
call :install_trt_rtx_stack
|
||||
) else (
|
||||
call :install_nvidia_stack
|
||||
)
|
||||
@@ -73,6 +76,7 @@ if errorlevel 1 exit /b 1
|
||||
|
||||
call :smoke_test
|
||||
if errorlevel 1 exit /b 1
|
||||
call :record_install
|
||||
call :done
|
||||
exit /b 0
|
||||
|
||||
@@ -188,29 +192,45 @@ if "!DRIVER_SUPPORTED!"=="0" (
|
||||
)
|
||||
exit /b 0
|
||||
|
||||
:check_previous_install
|
||||
set "PREV_RECORD_EXISTS=0"
|
||||
set "PREV_MODE="
|
||||
set "PREV_OPTION="
|
||||
if exist "%~dp0install_record.json" if exist "%VENV_PYTHON%" (
|
||||
for /f "tokens=1,2 delims==" %%A in ('"%VENV_PYTHON%" "%~dp0install_record.py" read 2^>nul') do (
|
||||
if "%%A"=="MODE" set "PREV_MODE=%%B"
|
||||
if "%%A"=="OPTION" set "PREV_OPTION=%%B"
|
||||
set "PREV_RECORD_EXISTS=1"
|
||||
)
|
||||
)
|
||||
if "!PREV_RECORD_EXISTS!"=="1" (
|
||||
echo Previous installation detected : [!PREV_OPTION!] !PREV_MODE!
|
||||
call :log "Previous install detected: Option !PREV_OPTION!, Mode !PREV_MODE!"
|
||||
)
|
||||
exit /b 0
|
||||
|
||||
:choose_mode
|
||||
set "INSTALL_MODE=AUTO"
|
||||
set "INSTALL_MODE=NVIDIA"
|
||||
set "USER_FORCED_NVIDIA=0"
|
||||
echo.
|
||||
echo Choose installation mode:
|
||||
echo [A] Auto - choose NVIDIA for detected RTX GPUs; otherwise CPU
|
||||
echo [N] NVIDIA GPU - force PyTorch CUDA 12.8
|
||||
echo [C] CPU - safest fallback, including AMD systems because AMD is not supported yet
|
||||
echo [A] CUDA EP - Choose this for the stable and safe option
|
||||
echo [B] TensorRT-RTX EP - Choose this if you like experimental, fast, and optimized (CUDA EP will still exist as a stable fallback)
|
||||
echo [C] CPU - safest fallback, including AMD systems because AMD is not supported yet
|
||||
if "%HAS_NVIDIA%"=="0" if "%HAS_AMD%"=="1" echo Note: AMD GPU detected, but AMD acceleration is not supported yet. Auto will use CPU.
|
||||
echo.
|
||||
call :prompt_choice MODE_CHOICE "Mode [A/N/C, default A]: " "A N C" "A" "A, N, or C"
|
||||
if /i "!MODE_CHOICE!"=="N" (
|
||||
call :prompt_choice MODE_CHOICE "Mode [A/B/C, default A]: " "A B C" "A" "A, B, or C"
|
||||
if /i "!MODE_CHOICE!"=="A" (
|
||||
set "INSTALL_MODE=NVIDIA"
|
||||
set "USER_FORCED_NVIDIA=1"
|
||||
)
|
||||
if /i "!MODE_CHOICE!"=="B" (
|
||||
set "INSTALL_MODE=TENSORRT_RTX"
|
||||
set "USER_FORCED_NVIDIA=1"
|
||||
)
|
||||
if /i "!MODE_CHOICE!"=="C" set "INSTALL_MODE=CPU"
|
||||
|
||||
if /i "%INSTALL_MODE%"=="AUTO" (
|
||||
if "%HAS_NVIDIA%"=="1" if "%RTX_SUPPORTED%"=="1" set "INSTALL_MODE=NVIDIA"
|
||||
if /i "!INSTALL_MODE!"=="AUTO" set "INSTALL_MODE=CPU"
|
||||
)
|
||||
|
||||
if /i "!INSTALL_MODE!"=="NVIDIA" if "%USER_FORCED_NVIDIA%"=="1" (
|
||||
if not "!INSTALL_MODE!"=="CPU" if "%USER_FORCED_NVIDIA%"=="1" (
|
||||
if "%HAS_NVIDIA%"=="0" call :confirm_risky "NVIDIA was not detected. Force GPU stack installation anyway?"
|
||||
if /i "!INSTALL_MODE!"=="CPU" goto :choose_mode_done
|
||||
if "%RTX_SUPPORTED%"=="0" call :confirm_risky "The GPU is outside the RTX 20+ target. Force GPU stack installation anyway?"
|
||||
@@ -283,6 +303,9 @@ if /i "%INSTALL_MODE%"=="CPU" (
|
||||
if errorlevel 1 exit /b 1
|
||||
set "INSTALL_REQ=%CPU_REQ_FILE%"
|
||||
)
|
||||
if /i "%INSTALL_MODE%"=="TENSORRT_RTX" (
|
||||
set "INSTALL_REQ=requirements_trt_rtx.txt"
|
||||
)
|
||||
|
||||
if exist "%~dp0wheels\diffq-0.2.4-cp311-cp311-win_amd64.whl" (
|
||||
call :log "Pre-installing local binary wheel for diffq"
|
||||
@@ -325,7 +348,7 @@ exit /b 0
|
||||
|
||||
:install_nvidia_stack
|
||||
echo.
|
||||
echo Installing NVIDIA CUDA PyTorch stack...
|
||||
echo Installing NVIDIA CUDA PyTorch stack (Option A - CUDA EP)...
|
||||
call :log "Installing PyTorch CUDA from %TORCH_CU128%"
|
||||
uv pip install --python "%VENV_PYTHON%" %TORCH_PACKAGES% --index-url "%TORCH_CU128%"
|
||||
if errorlevel 1 (
|
||||
@@ -341,11 +364,32 @@ if errorlevel 1 (
|
||||
)
|
||||
|
||||
echo Installing ONNX Runtime GPU compatibility package...
|
||||
uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime >nul 2>&1
|
||||
uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime onnxruntime-ep-nv-tensorrt-rtx-cu12 >nul 2>&1
|
||||
uv pip install --python "%VENV_PYTHON%" onnxruntime-gpu==1.22.0
|
||||
if errorlevel 1 call :warn "Failed to install onnxruntime-gpu. UVR can still use PyTorch CUDA, but ONNX models may fall back or fail."
|
||||
exit /b 0
|
||||
|
||||
:install_trt_rtx_stack
|
||||
echo.
|
||||
echo Installing NVIDIA CUDA PyTorch stack (Option B - TensorRT-RTX EP)...
|
||||
call :log "Installing PyTorch CUDA from %TORCH_CU128%"
|
||||
uv pip install --python "%VENV_PYTHON%" %TORCH_PACKAGES% --index-url "%TORCH_CU128%"
|
||||
if errorlevel 1 (
|
||||
call :warn "PyTorch CUDA installation failed. Offering CPU fallback."
|
||||
call :prompt_choice CPU_FALLBACK "Install CPU stack instead? [Y/N, default Y]: " "Y N" "Y" "Y or N"
|
||||
if /i "!CPU_FALLBACK!"=="N" (
|
||||
call :fail "PyTorch CUDA installation failed and CPU fallback was cancelled."
|
||||
exit /b 1
|
||||
)
|
||||
set "INSTALL_MODE=CPU"
|
||||
call :install_cpu_stack
|
||||
exit /b 0
|
||||
)
|
||||
|
||||
echo Preparing TensorRT-RTX packages...
|
||||
uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime >nul 2>&1
|
||||
exit /b 0
|
||||
|
||||
:install_cpu_stack
|
||||
echo.
|
||||
echo Installing CPU fallback stack...
|
||||
@@ -356,7 +400,7 @@ if errorlevel 1 (
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime-gpu >nul 2>&1
|
||||
uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime-gpu onnxruntime-ep-nv-tensorrt-rtx-cu12 >nul 2>&1
|
||||
uv pip install --python "%VENV_PYTHON%" onnxruntime==1.22.0
|
||||
if errorlevel 1 call :warn "Failed to install ONNX Runtime CPU. PyTorch models can still run, but ONNX models may fail."
|
||||
exit /b 0
|
||||
@@ -578,15 +622,30 @@ if errorlevel 1 (
|
||||
"%VENV_PYTHON%" -c "import onnxruntime as ort; print('ONNX Runtime providers:', ort.get_available_providers())"
|
||||
if errorlevel 1 call :warn "ONNX Runtime smoke test failed. UVR can still run non-ONNX models."
|
||||
|
||||
if /i "%INSTALL_MODE%"=="NVIDIA" (
|
||||
if /i "%INSTALL_MODE%"=="TENSORRT_RTX" (
|
||||
"%VENV_PYTHON%" -c "import onnxruntime as ort; import onnxruntime_ep_nv_tensorrt_rtx as trt_ep; ort.register_execution_provider_library(trt_ep.get_ep_name(), trt_ep.get_library_path()); print('TensorRT-RTX plugin loaded successfully.')" 2>nul
|
||||
if errorlevel 1 call :warn "TensorRT-RTX EP plugin could not be initialized. CUDA EP will still be used as stable fallback."
|
||||
)
|
||||
|
||||
if not "%INSTALL_MODE%"=="CPU" (
|
||||
"%VENV_PYTHON%" -c "import torch, sys; sys.exit(0 if torch.cuda.is_available() else 2)"
|
||||
if errorlevel 1 (
|
||||
call :warn "NVIDIA mode is installed, but torch.cuda.is_available() is still False."
|
||||
call :warn "GPU mode is installed, but torch.cuda.is_available() is still False."
|
||||
call :warn "Likely causes: outdated driver, DLL conflict, or incompatible PyTorch wheel."
|
||||
)
|
||||
)
|
||||
exit /b 0
|
||||
|
||||
:record_install
|
||||
echo.
|
||||
echo Recording install state...
|
||||
"%VENV_PYTHON%" "%~dp0install_record.py" write --option "!MODE_CHOICE!" --mode "!INSTALL_MODE!" --reuse-venv "!REUSE_VENV!" --has-uv "1" --has-ffmpeg "!HAS_FFMPEG!" --has-rubberband "!HAS_RB!" --has-vc-redist "!VC_INSTALLED!" >nul 2>&1
|
||||
if not errorlevel 1 (
|
||||
call :log "Successfully wrote install record to install_record.json"
|
||||
)
|
||||
exit /b 0
|
||||
|
||||
|
||||
:done
|
||||
echo.
|
||||
echo ============================================================
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Persistent install record manager for UVR installation.
|
||||
|
||||
Tracks installer mode, detected dependencies, installed package versions,
|
||||
and user choices to enable seamless switching between modes (CUDA, TensorRT-RTX, CPU).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
INSTALL_DIR = Path(__file__).resolve().parent
|
||||
ROOT_DIR = INSTALL_DIR.parent
|
||||
DEFAULT_RECORD_PATH = INSTALL_DIR / "install_record.json"
|
||||
|
||||
|
||||
def get_app_version() -> Dict[str, str]:
|
||||
"""Extract app version and build ID from __version__.py."""
|
||||
version_file = ROOT_DIR / "__version__.py"
|
||||
version_info = {"version": "unknown", "build_id": "unknown"}
|
||||
if version_file.is_file():
|
||||
namespace: Dict[str, Any] = {}
|
||||
try:
|
||||
with open(version_file, "r", encoding="utf-8") as f:
|
||||
exec(f.read(), namespace)
|
||||
version_info["version"] = str(namespace.get("VERSION", "unknown"))
|
||||
version_info["build_id"] = str(namespace.get("BUILD_ID", "unknown"))
|
||||
except Exception:
|
||||
pass
|
||||
return version_info
|
||||
|
||||
|
||||
def get_package_version(pkg_name: str) -> Optional[str]:
|
||||
"""Retrieve installed version of a Python package safely."""
|
||||
try:
|
||||
return importlib.metadata.version(pkg_name)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def read_install_record(record_path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Read and validate the existing install record. Returns None if absent or invalid."""
|
||||
path = record_path or DEFAULT_RECORD_PATH
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and "install_mode" in data:
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def inspect_installed_packages() -> Dict[str, Any]:
|
||||
"""Inspect installed versions of key runtime dependencies."""
|
||||
return {
|
||||
"pytorch": {
|
||||
"torch": get_package_version("torch"),
|
||||
"torchvision": get_package_version("torchvision"),
|
||||
"torchaudio": get_package_version("torchaudio"),
|
||||
},
|
||||
"onnxruntime": {
|
||||
"onnxruntime": get_package_version("onnxruntime"),
|
||||
"onnxruntime_gpu": get_package_version("onnxruntime-gpu"),
|
||||
},
|
||||
"tensorrt_rtx": {
|
||||
"onnxruntime_ep_nv_tensorrt_rtx_cu12": get_package_version("onnxruntime-ep-nv-tensorrt-rtx-cu12"),
|
||||
"tensorrt_cu12_libs": get_package_version("tensorrt-cu12-libs"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_install_record(
|
||||
option: str,
|
||||
install_mode: str,
|
||||
reuse_venv: str = "Y",
|
||||
has_uv: bool = True,
|
||||
has_ffmpeg: bool = True,
|
||||
has_rubberband: bool = True,
|
||||
has_vc_redist: bool = True,
|
||||
record_path: Optional[Path] = None,
|
||||
) -> Path:
|
||||
"""Record a successful installation configuration to JSON."""
|
||||
path = record_path or DEFAULT_RECORD_PATH
|
||||
app_version = get_app_version()
|
||||
packages = inspect_installed_packages()
|
||||
|
||||
record: Dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"timestamp_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"app_version": app_version["version"],
|
||||
"app_build_id": app_version["build_id"],
|
||||
"selected_option": option.upper(),
|
||||
"install_mode": install_mode.upper(),
|
||||
"user_prompts": {
|
||||
"reuse_venv": reuse_venv.upper(),
|
||||
},
|
||||
"system_components": {
|
||||
"venv_path": str(ROOT_DIR / "venv"),
|
||||
"uv_detected": has_uv,
|
||||
"ffmpeg_present": has_ffmpeg,
|
||||
"rubberband_present": has_rubberband,
|
||||
"vc_redist_detected": has_vc_redist,
|
||||
},
|
||||
"stack_profile": (
|
||||
"tensorrt_rtx_ort124" if option.upper() == "B"
|
||||
else ("cpu_ort122" if option.upper() == "C" else "stable_cuda_ort122")
|
||||
),
|
||||
"ort_stack_note": (
|
||||
"Option B uses onnxruntime-gpu>=1.24.4 for TensorRT-RTX EP ABI plugin compatibility (requires ORT>=1.24); CUDA EP remains available as stable fallback."
|
||||
if option.upper() == "B"
|
||||
else "Option A/C uses stable benchmarked onnxruntime==1.22.0 stack."
|
||||
),
|
||||
"installed_packages": packages,
|
||||
}
|
||||
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(record, f, indent=2)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] Failed to write install record: {exc}", file=sys.stderr)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="UVR Install Record Utility")
|
||||
subparsers = parser.add_subparsers(dest="action")
|
||||
|
||||
read_p = subparsers.add_parser("read", help="Read existing record")
|
||||
read_p.add_argument("--json", action="store_true", help="Output raw JSON")
|
||||
|
||||
write_p = subparsers.add_parser("write", help="Write successful install record")
|
||||
write_p.add_argument("--option", required=True, help="Selected option letter (A, B, or C)")
|
||||
write_p.add_argument("--mode", required=True, help="Resulting install mode (NVIDIA, TENSORRT_RTX, CPU)")
|
||||
write_p.add_argument("--reuse-venv", default="Y", help="User answer for venv reuse (Y/N)")
|
||||
write_p.add_argument("--has-uv", default="1", help="Whether uv is present (1/0)")
|
||||
write_p.add_argument("--has-ffmpeg", default="1", help="Whether ffmpeg is present (1/0)")
|
||||
write_p.add_argument("--has-rubberband", default="1", help="Whether rubberband is present (1/0)")
|
||||
write_p.add_argument("--has-vc-redist", default="1", help="Whether VC redist is present (1/0)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.action == "read":
|
||||
record = read_install_record()
|
||||
if not record:
|
||||
print("NO_RECORD")
|
||||
return 0
|
||||
if args.json:
|
||||
print(json.dumps(record, indent=2))
|
||||
else:
|
||||
print(f"MODE={record.get('install_mode', 'UNKNOWN')}")
|
||||
print(f"OPTION={record.get('selected_option', 'UNKNOWN')}")
|
||||
print(f"VERSION={record.get('app_version', 'UNKNOWN')}")
|
||||
print(f"TIMESTAMP={record.get('timestamp_utc', 'UNKNOWN')}")
|
||||
return 0
|
||||
|
||||
elif args.action == "write":
|
||||
write_install_record(
|
||||
option=args.option,
|
||||
install_mode=args.mode,
|
||||
reuse_venv=args.reuse_venv,
|
||||
has_uv=args.has_uv == "1",
|
||||
has_ffmpeg=args.has_ffmpeg == "1",
|
||||
has_rubberband=args.has_rubberband == "1",
|
||||
has_vc_redist=args.has_vc_redist == "1",
|
||||
)
|
||||
print("RECORD_WRITTEN")
|
||||
return 0
|
||||
|
||||
parser.print_help()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,137 @@
|
||||
absl-py==2.4.0
|
||||
aiohappyeyeballs==2.6.2
|
||||
aiohttp==3.14.1
|
||||
aiosignal==1.4.0
|
||||
altgraph==0.17.3
|
||||
annotated-types==0.7.0
|
||||
antlr4-python3-runtime==4.9.3
|
||||
attrs==26.1.0
|
||||
audioread==3.1.0
|
||||
certifi==2026.7.22
|
||||
cffi==2.1.0
|
||||
charset-normalizer==3.4.9
|
||||
colorama==0.4.6
|
||||
coloredlogs==15.0.1
|
||||
contextlib2==21.6.0
|
||||
contourpy==1.3.3
|
||||
cppimport==26.4.17
|
||||
cryptography==3.4.6
|
||||
cuda-pathfinder==1.6.0
|
||||
cupy-cuda12x==14.1.1
|
||||
cycler==0.12.1
|
||||
cython==3.2.6
|
||||
decorator==5.3.1
|
||||
einops==0.6.0
|
||||
filelock==3.29.0
|
||||
flatbuffers==25.12.19
|
||||
fonttools==4.63.0
|
||||
frozenlist==1.8.0
|
||||
fsspec==2026.4.0
|
||||
future==0.18.3
|
||||
humanfriendly==10.0
|
||||
idna==3.18
|
||||
jinja2==3.1.6
|
||||
joblib==1.5.3
|
||||
jsonschema==4.26.0
|
||||
jsonschema-specifications==2025.9.1
|
||||
julius==0.2.7
|
||||
kiwisolver==1.5.0
|
||||
kthread==0.2.3
|
||||
lazy-loader==0.5
|
||||
librosa==0.11.0
|
||||
lief==1.0.0
|
||||
lightning-utilities==0.15.3
|
||||
llvmlite==0.48.0
|
||||
mako==1.3.12
|
||||
markdown-it-py==4.2.0
|
||||
markupsafe==3.0.3
|
||||
matchering==2.0.6
|
||||
matplotlib==3.11.0
|
||||
mdurl==0.1.2
|
||||
ml-collections==0.1.1
|
||||
ml-dtypes==0.5.4
|
||||
mpmath==1.3.0
|
||||
msgpack==1.2.1
|
||||
multidict==6.7.1
|
||||
narwhals==2.24.0
|
||||
natsort==8.2.0
|
||||
networkx==3.6.1
|
||||
ninja==1.13.0
|
||||
numba==0.66.0
|
||||
numpy==2.4.4
|
||||
# nvidia-cublas-cu12==12.9.2.10
|
||||
# nvidia-cuda-nvrtc-cu12==12.9.86
|
||||
# nvidia-cuda-runtime-cu12==12.9.79
|
||||
# nvidia-cudnn-cu12==9.23.2.1
|
||||
# nvidia-cufft-cu12==11.4.1.4
|
||||
# nvidia-curand-cu12==10.3.10.19
|
||||
nvidia-ml-py==13.610.43
|
||||
# nvidia-modelopt==0.45.0
|
||||
# nvidia-nvjitlink-cu12==12.9.86
|
||||
omegaconf==2.3.1
|
||||
onnx==1.21.0
|
||||
onnx-graphsurgeon==0.6.1
|
||||
onnx-ir==0.2.1
|
||||
onnx2pytorch==0.5.3
|
||||
onnxconverter-common==1.16.0
|
||||
onnxruntime-gpu==1.24.4
|
||||
onnxruntime-ep-nv-tensorrt-rtx-cu12==0.4.0
|
||||
onnxscript==0.7.1
|
||||
onnxslim==0.1.94
|
||||
opencv-python==4.6.0.66
|
||||
packaging==26.2
|
||||
pandas==2.3.3
|
||||
patsy==1.0.2
|
||||
pillow==12.2.0
|
||||
pip==24.0
|
||||
platformdirs==4.11.0
|
||||
playsound==1.2.2
|
||||
polygraphy==0.50.3
|
||||
pooch==1.9.0
|
||||
propcache==0.5.2
|
||||
protobuf==7.35.1
|
||||
psutil==5.9.4
|
||||
pulp==3.3.2
|
||||
pybind11==3.0.4
|
||||
pycparser==3.0
|
||||
pydantic==2.13.4
|
||||
pydantic-core==2.46.4
|
||||
pydub==0.25.1
|
||||
pyglet==1.5.23
|
||||
pygments==2.20.0
|
||||
pyparsing==3.3.2
|
||||
pyperclip==1.8.2
|
||||
pyreadline3==3.5.6
|
||||
pyrubberband==0.3.0
|
||||
pysoundfile==0.9.0.post1
|
||||
python-dateutil==2.9.0.post0
|
||||
pytorch-lightning==2.0.0
|
||||
pytz==2026.2
|
||||
pyyaml==6.0
|
||||
referencing==0.37.0
|
||||
regex==2026.7.19
|
||||
requests==2.34.2
|
||||
resampy==0.4.2
|
||||
rich==15.0.0
|
||||
rpds-py==2026.6.3
|
||||
safetensors==0.8.0
|
||||
samplerate==0.1.0
|
||||
scikit-learn==1.9.0
|
||||
scipy==1.17.1
|
||||
screeninfo==0.8.1
|
||||
setuptools==78.1.0
|
||||
six==1.17.0
|
||||
soundfile==0.14.0
|
||||
soundstretch==1.2
|
||||
soxr==1.1.0
|
||||
statsmodels==0.14.6
|
||||
sympy==1.14.0
|
||||
threadpoolctl==3.6.0
|
||||
torchmetrics==1.9.0
|
||||
tqdm==4.68.3
|
||||
typing-extensions==4.15.0
|
||||
typing-inspection==0.4.2
|
||||
tzdata==2026.2
|
||||
urllib3==2.7.0
|
||||
wget==3.2
|
||||
yarl==1.24.2
|
||||
+1
-1
@@ -13,7 +13,7 @@ if not exist "%UVR_PYTHON%" (
|
||||
)
|
||||
|
||||
echo Starting Ultimate Vocal Remover...
|
||||
powershell.exe -NoLogo -NoProfile -Command "& { & $env:UVR_PYTHON -u (Join-Path $env:UVR_ROOT 'UVR.py') 2>&1 | Tee-Object -FilePath $env:UVR_LOG; $uvrExitCode = $LASTEXITCODE; if ($null -eq $uvrExitCode) { $uvrExitCode = 1 }; exit $uvrExitCode }"
|
||||
"%UVR_PYTHON%" "%UVR_ROOT%UVR.py"
|
||||
set "UVR_EXIT_CODE=%ERRORLEVEL%"
|
||||
if not "%UVR_EXIT_CODE%"=="0" echo [ERROR] UVR exited with code %UVR_EXIT_CODE%. See uvr_run.log for details.
|
||||
pause
|
||||
|
||||
+78
-1
@@ -409,6 +409,78 @@ def create_onnx_fp16_session(model_path, providers):
|
||||
|
||||
|
||||
def create_onnx_session(model_path, providers, precision_policy):
|
||||
# Delegate to TensorRT-RTX wrapper if TensorRT-RTX was requested
|
||||
is_trt_requested = any(
|
||||
(isinstance(p, tuple) and "NvTensorRTRTX" in p[0]) or (isinstance(p, str) and "NvTensorRTRTX" in p)
|
||||
for p in providers
|
||||
) if isinstance(providers, (list, tuple)) else False
|
||||
|
||||
if is_trt_requested:
|
||||
from engine.tensorrt_rtx import create_trt_rtx_session
|
||||
dev_id = 0
|
||||
cuda_options = None
|
||||
if isinstance(providers, (list, tuple)):
|
||||
for p in providers:
|
||||
if isinstance(p, tuple):
|
||||
if p[0] == 'CUDAExecutionProvider' and isinstance(p[1], dict):
|
||||
cuda_options = p[1]
|
||||
if isinstance(p[1], dict) and "device_id" in p[1]:
|
||||
try:
|
||||
dev_id = int(p[1]["device_id"])
|
||||
except Exception:
|
||||
dev_id = 0
|
||||
|
||||
model_name = os.path.basename(model_path)
|
||||
if precision_policy.is_fp16:
|
||||
# Respect FP16 preset on TensorRT-RTX path
|
||||
import onnx
|
||||
from onnxconverter_common import float16
|
||||
|
||||
print(f"[PRECISION] Loading {model_name} into CPU RAM for in-memory FP16 conversion...")
|
||||
try:
|
||||
model_fp32 = onnx.load(model_path, load_external_data=True)
|
||||
try:
|
||||
model_fp16 = float16.convert_float_to_float16(model_fp32, keep_io_types=False)
|
||||
finally:
|
||||
del model_fp32
|
||||
gc.collect()
|
||||
|
||||
try:
|
||||
serialized_fp16 = model_fp16.SerializeToString()
|
||||
finally:
|
||||
del model_fp16
|
||||
gc.collect()
|
||||
|
||||
print(f"[PRECISION] Creating TensorRT-RTX FP16 session from RAM ({len(serialized_fp16) / (1024 ** 2):.2f} MiB)...")
|
||||
try:
|
||||
return create_trt_rtx_session(
|
||||
serialized_fp16,
|
||||
device_id=dev_id,
|
||||
cuda_options=cuda_options,
|
||||
model_name=model_name,
|
||||
)
|
||||
finally:
|
||||
del serialized_fp16
|
||||
gc.collect()
|
||||
except Exception as error:
|
||||
print(
|
||||
f"[PRECISION] In-memory FP16 conversion failed ({type(error).__name__}: {error}). "
|
||||
"Falling back to the original FP32 ONNX model without writing a temporary file."
|
||||
)
|
||||
return create_trt_rtx_session(
|
||||
model_path,
|
||||
device_id=dev_id,
|
||||
cuda_options=cuda_options,
|
||||
model_name=model_name,
|
||||
)
|
||||
else:
|
||||
return create_trt_rtx_session(
|
||||
model_path,
|
||||
device_id=dev_id,
|
||||
cuda_options=cuda_options,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
if not precision_policy.is_fp16:
|
||||
return ort.InferenceSession(model_path, providers=providers)
|
||||
|
||||
@@ -421,6 +493,7 @@ def create_onnx_session(model_path, providers, precision_policy):
|
||||
)
|
||||
return ort.InferenceSession(model_path, providers=providers)
|
||||
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from UVR import ModelData
|
||||
@@ -585,7 +658,11 @@ class SeperateAttributes:
|
||||
'cudnn_conv_algo_search': 'HEURISTIC',
|
||||
'do_copy_in_default_stream': 'True'
|
||||
}
|
||||
self.run_type = [('CUDAExecutionProvider', cuda_options), 'CPUExecutionProvider']
|
||||
chosen_ep = getattr(model_data, 'execution_provider', 'CUDAExecutionProvider')
|
||||
if chosen_ep == 'NvTensorRTRTXExecutionProvider':
|
||||
self.run_type = ['NvTensorRTRTXExecutionProvider', ('CUDAExecutionProvider', cuda_options), 'CPUExecutionProvider']
|
||||
else:
|
||||
self.run_type = [('CUDAExecutionProvider', cuda_options), 'CPUExecutionProvider']
|
||||
|
||||
if model_data.process_method == MDX_ARCH_TYPE:
|
||||
self.is_mdx_ckpt = model_data.is_mdx_ckpt
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Unit tests for TensorRT-RTX wrapper, provider resolution, precision presets, and install record."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import onnxruntime as ort
|
||||
import torch
|
||||
|
||||
from engine.tensorrt_rtx import (
|
||||
PROVIDER_CPU,
|
||||
PROVIDER_CUDA,
|
||||
PROVIDER_TRT_RTX,
|
||||
build_trt_rtx_options,
|
||||
create_trt_rtx_session,
|
||||
get_supported_execution_providers,
|
||||
is_ort_gpu_available,
|
||||
is_trt_rtx_available,
|
||||
is_trt_rtx_installed,
|
||||
)
|
||||
from install.install_record import read_install_record, write_install_record
|
||||
from separate import SmartPrecisionPolicy, create_onnx_session
|
||||
|
||||
|
||||
class TensorRTRTXEngineTests(unittest.TestCase):
|
||||
"""Test TensorRT-RTX options, provider discovery, honest separation, and session fallback."""
|
||||
|
||||
def test_build_trt_rtx_options_keys(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
options = build_trt_rtx_options(device_id=0, cache_dir=tmp_dir, enable_cuda_graph=True)
|
||||
self.assertEqual(options["device_id"], "0")
|
||||
self.assertEqual(options["nv_runtime_cache_path"], str(Path(tmp_dir)))
|
||||
self.assertEqual(options["enable_cuda_graph"], "1")
|
||||
# Must NOT contain standard TensorRT keys
|
||||
for key in options:
|
||||
self.assertFalse(key.startswith("trt_"), f"Unexpected standard TRT option: {key}")
|
||||
|
||||
def test_standard_tensorrt_is_not_treated_as_trt_rtx(self):
|
||||
# Even if standard TensorrtExecutionProvider is present in ORT, it must NOT be offered as TensorRT-RTX
|
||||
with mock.patch("onnxruntime.get_available_providers", return_value=["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]):
|
||||
with mock.patch("onnxruntime.get_all_providers", return_value=["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]):
|
||||
# Ensure plugin is not detected
|
||||
with mock.patch("onnxruntime.get_ep_devices", create=True, return_value=[]):
|
||||
self.assertFalse(is_trt_rtx_installed())
|
||||
providers = get_supported_execution_providers(has_gpu_selected=True, device_id=0)
|
||||
self.assertNotIn(PROVIDER_TRT_RTX, providers)
|
||||
self.assertIn(PROVIDER_CUDA, providers)
|
||||
|
||||
def test_provider_resolution_rules(self):
|
||||
# Rule 3: GPU not selected -> returns CPUExecutionProvider
|
||||
cpu_providers = get_supported_execution_providers(has_gpu_selected=False)
|
||||
self.assertEqual(cpu_providers, [PROVIDER_CPU])
|
||||
|
||||
# Rule 2: GPU selected -> returns GPU providers with CUDA fallback
|
||||
gpu_providers = get_supported_execution_providers(has_gpu_selected=True, device_id=0)
|
||||
self.assertIsInstance(gpu_providers, list)
|
||||
self.assertGreater(len(gpu_providers), 0)
|
||||
self.assertTrue(PROVIDER_CUDA in gpu_providers or PROVIDER_CPU in gpu_providers)
|
||||
|
||||
def test_saved_provider_preserved_while_locked(self):
|
||||
# Simulate GUI state behavior
|
||||
is_gpu_selected = False # DEFAULT GPU device
|
||||
stored_ep = PROVIDER_TRT_RTX
|
||||
|
||||
# While locked, stored provider must NOT be overwritten
|
||||
if is_gpu_selected:
|
||||
stored_ep = "OVERWRITTEN"
|
||||
self.assertEqual(stored_ep, PROVIDER_TRT_RTX)
|
||||
|
||||
# When GPU is selected, validate against detected providers
|
||||
is_gpu_selected = True
|
||||
supported = [PROVIDER_CUDA, PROVIDER_CPU]
|
||||
if is_gpu_selected and stored_ep not in supported:
|
||||
stored_ep = PROVIDER_CUDA
|
||||
self.assertEqual(stored_ep, PROVIDER_CUDA)
|
||||
|
||||
@mock.patch("onnxruntime.InferenceSession")
|
||||
def test_session_fallback_chain(self, mock_session):
|
||||
sentinel_session = mock.MagicMock()
|
||||
mock_session.return_value = sentinel_session
|
||||
|
||||
# Case 1: Plugin is present -> uses sess_options or providers
|
||||
session = create_trt_rtx_session("test_model.onnx", device_id=0)
|
||||
self.assertIsNotNone(session)
|
||||
mock_session.assert_called()
|
||||
|
||||
# Case 2: When TRT-RTX plugin session creation fails, it falls back to CUDA
|
||||
mock_session.reset_mock()
|
||||
mock_session.side_effect = [
|
||||
RuntimeError("Plugin creation failure"),
|
||||
sentinel_session,
|
||||
]
|
||||
session_fb = create_trt_rtx_session("test_model.onnx", device_id=0)
|
||||
self.assertIsNotNone(session_fb)
|
||||
self.assertGreaterEqual(mock_session.call_count, 2)
|
||||
second_call_kwargs = mock_session.call_args_list[1][1]
|
||||
providers_used = second_call_kwargs.get("providers", [])
|
||||
self.assertTrue(
|
||||
any("CUDAExecutionProvider" in (p[0] if isinstance(p, tuple) else p) for p in providers_used)
|
||||
or any("CPUExecutionProvider" in (p[0] if isinstance(p, tuple) else p) for p in providers_used)
|
||||
)
|
||||
|
||||
def test_precision_policy_diagnose_consistency(self):
|
||||
from gui_data.constants import MODEL_PRECISION_FP16, MODEL_PRECISION_FP32
|
||||
policy_fp16 = SmartPrecisionPolicy(MODEL_PRECISION_FP16)
|
||||
self.assertTrue(policy_fp16.is_fp16)
|
||||
self.assertEqual(policy_fp16.effective_mode_for("ONNX"), "FP16")
|
||||
|
||||
policy_fp32 = SmartPrecisionPolicy(MODEL_PRECISION_FP32)
|
||||
self.assertTrue(policy_fp32.is_fp32)
|
||||
self.assertEqual(policy_fp32.effective_mode_for("ONNX"), "FP32")
|
||||
|
||||
|
||||
class InstallRecordTests(unittest.TestCase):
|
||||
"""Test persistent install record write, read, stack notes, and recovery."""
|
||||
|
||||
def test_write_and_read_install_record_with_stack_info(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
record_path = Path(tmp_dir) / "test_install_record.json"
|
||||
|
||||
# Write record for Option B
|
||||
write_install_record(
|
||||
option="B",
|
||||
install_mode="TENSORRT_RTX",
|
||||
reuse_venv="Y",
|
||||
has_uv=True,
|
||||
has_ffmpeg=True,
|
||||
has_rubberband=True,
|
||||
has_vc_redist=True,
|
||||
record_path=record_path,
|
||||
)
|
||||
|
||||
self.assertTrue(record_path.is_file())
|
||||
|
||||
# Read record back
|
||||
data = read_install_record(record_path)
|
||||
self.assertIsNotNone(data)
|
||||
self.assertEqual(data["selected_option"], "B")
|
||||
self.assertEqual(data["install_mode"], "TENSORRT_RTX")
|
||||
self.assertEqual(data["stack_profile"], "tensorrt_rtx_ort124")
|
||||
self.assertIn("ort_stack_note", data)
|
||||
self.assertIn("installed_packages", data)
|
||||
|
||||
def test_read_missing_or_corrupt_record_returns_none(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
missing_path = Path(tmp_dir) / "non_existent.json"
|
||||
self.assertIsNone(read_install_record(missing_path))
|
||||
|
||||
corrupt_path = Path(tmp_dir) / "corrupt.json"
|
||||
corrupt_path.write_text("{bad json: 123", encoding="utf-8")
|
||||
self.assertIsNone(read_install_record(corrupt_path))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -16,6 +16,7 @@ from uvr_cli.model_resolver import (
|
||||
HeadlessModelData,
|
||||
build_headless_model_data,
|
||||
list_available_models,
|
||||
parse_and_validate_device,
|
||||
resolve_model_file,
|
||||
)
|
||||
from uvr_cli.progress import InferenceProgressBar
|
||||
@@ -70,6 +71,31 @@ class BenchmarkCLITests(unittest.TestCase):
|
||||
self.assertEqual(model_data_custom.batch_size, 2)
|
||||
self.assertEqual(model_data_custom.device_name, "cuda:0")
|
||||
|
||||
def test_device_selection_cuda_and_cpu(self):
|
||||
is_gpu, idx, ep = parse_and_validate_device("cuda:0")
|
||||
self.assertTrue(is_gpu)
|
||||
self.assertEqual(idx, "0")
|
||||
self.assertEqual(ep, "CUDAExecutionProvider")
|
||||
|
||||
is_gpu, idx, ep = parse_and_validate_device("cpu")
|
||||
self.assertFalse(is_gpu)
|
||||
self.assertEqual(idx, "0")
|
||||
self.assertEqual(ep, "CPUExecutionProvider")
|
||||
|
||||
def test_device_selection_trt_rtx(self):
|
||||
is_gpu, idx, ep = parse_and_validate_device("trt_rtx:0")
|
||||
self.assertTrue(is_gpu)
|
||||
self.assertEqual(idx, "0")
|
||||
self.assertEqual(ep, "NvTensorRTRTXExecutionProvider")
|
||||
|
||||
def test_device_selection_invalid_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
parse_and_validate_device("vulkan:0")
|
||||
with self.assertRaises(ValueError):
|
||||
parse_and_validate_device("cuda:invalid")
|
||||
with self.assertRaises(ValueError):
|
||||
parse_and_validate_device("invalid")
|
||||
|
||||
def test_telemetry_sampler_lifecycle(self):
|
||||
sampler = TelemetrySampler(device_index=0, interval_sec=0.05)
|
||||
sampler.start()
|
||||
|
||||
+3
-3
@@ -49,7 +49,7 @@ venv\Scripts\python.exe -m uvr_cli run `
|
||||
--audio "path/to/song.wav" `
|
||||
--model "UVR-MDX-NET-Inst_HQ_4.onnx" `
|
||||
--precision fp16 `
|
||||
--device cuda:0 `
|
||||
--device trt_rtx:0 `
|
||||
--output-dir "separated_outputs"
|
||||
```
|
||||
|
||||
@@ -60,7 +60,7 @@ venv\Scripts\python.exe -m uvr_cli run `
|
||||
| `--audio` | `-a` | `Path` | *Required* | Path to the input audio file to separate (WAV, FLAC, MP3, etc.). |
|
||||
| `--model` | `-m` | `String` | *Required* | Model filename (e.g. `UVR-MDX-NET-Inst_HQ_4.onnx`), friendly name, or absolute path. |
|
||||
| `--precision` | `-p` | `Choice` | `fp16` | Inference precision policy: `fp16` (default, in-memory downcast), `fp32`, or `bf16` (experimental). |
|
||||
| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda:0`, `cuda:1`, or `cpu`). |
|
||||
| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda[:N]`, `trt_rtx[:N]`, or `cpu`). |
|
||||
| `--output-dir` | `-o` | `Path` | `separated_outputs` | Output directory where separated stem files will be saved. |
|
||||
| `--segment-size` | | `Integer` | *Model Default* | MDX chunk size in samples (e.g. `256`, `3000`, `4000`, `6000`). |
|
||||
| `--overlap` | | `Float` | *Model Default* | Overlap ratio between adjacent chunks (e.g. `0.25`, `0.50`, `0.75`). |
|
||||
@@ -103,7 +103,7 @@ venv\Scripts\python.exe -m uvr_cli bench `
|
||||
| `--audio` | `-a` | `Path` | *Required* | Path to input audio file for benchmarking (uncompressed WAV recommended). |
|
||||
| `--model` | `-m` | `String` | *Required* | Model filename, friendly name, or absolute path to benchmark. |
|
||||
| `--precision` | `-p` | `Choice` | `fp16` | Inference precision policy: `fp16`, `fp32`, or `bf16` (experimental). |
|
||||
| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda:0`, `cpu`). |
|
||||
| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda[:N]`, `trt_rtx[:N]`, or `cpu`). |
|
||||
| `--rounds` | `-r` | `Integer` | `3` | Number of measured benchmark rounds to run after warmup. |
|
||||
| `--warmup` | `-w` | `Integer` | `1` | Number of untimed warmup rounds to prime model sessions, ONNX memory, and CUDA allocations. |
|
||||
| `--export-dir` | `-e` | `Path` | `benchmark_results` | Directory where JSON telemetry reports and `summary.csv` will be saved. |
|
||||
|
||||
+39
-27
@@ -106,15 +106,19 @@ def handle_run(args: argparse.Namespace) -> int:
|
||||
precision = normalize_cli_precision(args.precision)
|
||||
|
||||
print(f"Resolving model: {model_path.name} ({arch})")
|
||||
model_data = build_headless_model_data(
|
||||
model_path=model_path,
|
||||
architecture=arch,
|
||||
precision=precision,
|
||||
device=args.device,
|
||||
segment_size=args.segment_size,
|
||||
overlap=args.overlap,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
try:
|
||||
model_data = build_headless_model_data(
|
||||
model_path=model_path,
|
||||
architecture=arch,
|
||||
precision=precision,
|
||||
device=args.device,
|
||||
segment_size=args.segment_size,
|
||||
overlap=args.overlap,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
except ValueError as val_err:
|
||||
print(f"[ERROR] {val_err}")
|
||||
return 1
|
||||
|
||||
audio_path = Path(args.audio).resolve()
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
@@ -192,15 +196,19 @@ def handle_bench(args: argparse.Namespace) -> int:
|
||||
model_path, arch = resolve_model_file(args.model)
|
||||
precision = normalize_cli_precision(args.precision)
|
||||
|
||||
model_data = build_headless_model_data(
|
||||
model_path=model_path,
|
||||
architecture=arch,
|
||||
precision=precision,
|
||||
device=args.device,
|
||||
segment_size=args.segment_size,
|
||||
overlap=args.overlap,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
try:
|
||||
model_data = build_headless_model_data(
|
||||
model_path=model_path,
|
||||
architecture=arch,
|
||||
precision=precision,
|
||||
device=args.device,
|
||||
segment_size=args.segment_size,
|
||||
overlap=args.overlap,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
except ValueError as val_err:
|
||||
print(f"[ERROR] {val_err}")
|
||||
return 1
|
||||
|
||||
device_idx = int(args.device.split(":")[-1]) if ":" in args.device else 0
|
||||
|
||||
@@ -245,12 +253,16 @@ def handle_smoke_test(args: argparse.Namespace) -> int:
|
||||
|
||||
print(f"Selected smoke test model: {model_path.name} ({arch})")
|
||||
|
||||
model_data = build_headless_model_data(
|
||||
model_path=model_path,
|
||||
architecture=arch,
|
||||
precision=precision,
|
||||
device=args.device,
|
||||
)
|
||||
try:
|
||||
model_data = build_headless_model_data(
|
||||
model_path=model_path,
|
||||
architecture=arch,
|
||||
precision=precision,
|
||||
device=args.device,
|
||||
)
|
||||
except ValueError as val_err:
|
||||
print(f"[ERROR] {val_err}")
|
||||
return 1
|
||||
|
||||
device_idx = int(args.device.split(":")[-1]) if ":" in args.device else 0
|
||||
|
||||
@@ -303,7 +315,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
default="fp16",
|
||||
help="Inference precision: fp32, fp16, bf16 (default: fp16)",
|
||||
)
|
||||
run_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)")
|
||||
run_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda[:N], trt_rtx[:N], cpu (default: cuda:0)")
|
||||
run_parser.add_argument("--output-dir", "-o", default="separated_outputs", help="Output directory for stems")
|
||||
run_parser.add_argument("--segment-size", type=int, default=None, help="Segment size (MDX chunk size)")
|
||||
run_parser.add_argument("--overlap", type=float, default=None, help="Overlap ratio (e.g. 0.25)")
|
||||
@@ -325,7 +337,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
default="fp16",
|
||||
help="Inference precision: fp32, fp16, bf16 (default: fp16)",
|
||||
)
|
||||
bench_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)")
|
||||
bench_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda[:N], trt_rtx[:N], cpu (default: cuda:0)")
|
||||
bench_parser.add_argument("--rounds", "-r", type=int, default=3, help="Number of measurement rounds (default: 3)")
|
||||
bench_parser.add_argument("--warmup", "-w", type=int, default=1, help="Number of warmup rounds (default: 1)")
|
||||
bench_parser.add_argument(
|
||||
@@ -342,7 +354,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
smoke_parser = subparsers.add_parser("smoke-test", help="Run automated smoke test using synthetic audio")
|
||||
smoke_parser.add_argument("--model", "-m", default=None, help="Optional model name (defaults to first available)")
|
||||
smoke_parser.add_argument("--precision", "-p", default="fp16", help="Inference precision (default: fp16)")
|
||||
smoke_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)")
|
||||
smoke_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda[:N], trt_rtx[:N], cpu (default: cuda:0)")
|
||||
smoke_parser.add_argument(
|
||||
"--export-dir",
|
||||
"-e",
|
||||
|
||||
@@ -100,6 +100,7 @@ class HeadlessModelData:
|
||||
self.model_hash: Optional[str] = None
|
||||
self.model_status: bool = True
|
||||
self.device_set: str = "0"
|
||||
self.execution_provider: str = "CUDAExecutionProvider"
|
||||
self.is_gpu_conversion: int = 0
|
||||
self.is_use_opencl: bool = False
|
||||
self.model_precision: str = MODEL_PRECISION_DEFAULT
|
||||
@@ -276,6 +277,43 @@ def resolve_model_file(model_identifier: str) -> Tuple[Path, str]:
|
||||
)
|
||||
|
||||
|
||||
def parse_and_validate_device(device_str: str) -> Tuple[bool, str, str]:
|
||||
"""Parse and validate CLI device string.
|
||||
|
||||
Returns:
|
||||
(is_gpu, device_index, execution_provider)
|
||||
|
||||
Raises:
|
||||
ValueError if the device format or name is invalid.
|
||||
"""
|
||||
dev = device_str.strip().lower()
|
||||
if dev == "cpu":
|
||||
return False, "0", "CPUExecutionProvider"
|
||||
|
||||
parts = dev.split(":")
|
||||
prefix = parts[0]
|
||||
index = parts[1] if len(parts) > 1 else "0"
|
||||
|
||||
if prefix not in ("cuda", "trt_rtx", "directml"):
|
||||
raise ValueError(
|
||||
f"Invalid device '{device_str}'. Valid choices are: 'cuda[:N]', 'trt_rtx[:N]', or 'cpu' (e.g. 'cuda:0', 'trt_rtx:0', 'cpu')."
|
||||
)
|
||||
|
||||
if not index.isdigit():
|
||||
raise ValueError(
|
||||
f"Invalid device index in '{device_str}'. Device index must be an integer (e.g. 'cuda:0', 'trt_rtx:0')."
|
||||
)
|
||||
|
||||
if prefix == "trt_rtx":
|
||||
ep = "NvTensorRTRTXExecutionProvider"
|
||||
elif prefix == "cuda":
|
||||
ep = "CUDAExecutionProvider"
|
||||
else:
|
||||
ep = "CUDAExecutionProvider"
|
||||
|
||||
return True, index, ep
|
||||
|
||||
|
||||
def build_headless_model_data(
|
||||
model_path: Path,
|
||||
architecture: str,
|
||||
@@ -291,8 +329,26 @@ def build_headless_model_data(
|
||||
model_basename = model_path.stem
|
||||
model_hash = compute_uvr_hash(model_path)
|
||||
|
||||
is_gpu = "cuda" in device.lower() or "directml" in device.lower()
|
||||
device_set = device.split(":")[-1] if ":" in device else "0"
|
||||
is_gpu, device_set, chosen_ep = parse_and_validate_device(device)
|
||||
|
||||
# Provider availability and fallback
|
||||
if is_gpu:
|
||||
import torch
|
||||
if not torch.cuda.is_available():
|
||||
print(f"[WARN] GPU acceleration requested ('{device}') but CUDA is not available. Falling back to CPU.")
|
||||
is_gpu = False
|
||||
device_set = "0"
|
||||
chosen_ep = "CPUExecutionProvider"
|
||||
elif chosen_ep == "NvTensorRTRTXExecutionProvider":
|
||||
try:
|
||||
from engine.tensorrt_rtx import is_trt_rtx_available
|
||||
dev_idx = int(device_set)
|
||||
if not is_trt_rtx_available(dev_idx):
|
||||
print(f"[BACKEND] TensorRT-RTX EP is not available on device {dev_idx}. Falling back to CUDAExecutionProvider.")
|
||||
chosen_ep = "CUDAExecutionProvider"
|
||||
except Exception:
|
||||
print("[BACKEND] TensorRT-RTX EP check failed. Falling back to CUDAExecutionProvider.")
|
||||
chosen_ep = "CUDAExecutionProvider"
|
||||
|
||||
clean_prec = precision.strip().lower()
|
||||
if clean_prec in ("fp32", "32"):
|
||||
@@ -315,6 +371,8 @@ def build_headless_model_data(
|
||||
device_name=device,
|
||||
is_gpu_conversion=0 if is_gpu else -1,
|
||||
)
|
||||
data.execution_provider = chosen_ep
|
||||
|
||||
|
||||
if segment_size is not None:
|
||||
data.segment_size = segment_size
|
||||
|
||||
Reference in new issue
Block a user