From 90050a048e5cd5e20e456a3bedd07eb639c4ffa5 Mon Sep 17 00:00:00 2001 From: RevLayDust <225862333+RevLayDust@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:05:22 +0700 Subject: [PATCH] feat(engine): add TensorRT-RTX execution provider backend with in-memory FP16 downcast - Implement TensorRT-RTX backend via onnxruntime-ep-nv-tensorrt-rtx plugin in engine/tensorrt_rtx/ - Support in-memory FP16 downcast for MDX-Net ONNX models with zero disk writes - Implement automatic fallback chain: TensorRT-RTX -> CUDA -> CPU - Add GUI execution provider selection with dynamic hardware gating - Add CLI provider flags and model resolver support for trt_rtx - Add install/requirements_trt_rtx.txt and stack profile tracking - Add test suite in tests/test_tensorrt_rtx.py and test_uvr_cli.py - Update run_uvr.bat to execute UVR.py via native CMD --- .gitignore | 7 +- UVR.py | 91 ++++++++++++-- __version__.py | 2 +- engine/tensorrt_rtx/__init__.py | 39 ++++++ engine/tensorrt_rtx/environment.py | 155 ++++++++++++++++++++++++ engine/tensorrt_rtx/session.py | 147 +++++++++++++++++++++++ gui_data/constants.py | 6 + install/install_packages.bat | 93 ++++++++++++--- install/install_record.py | 184 +++++++++++++++++++++++++++++ install/requirements_trt_rtx.txt | 137 +++++++++++++++++++++ run_uvr.bat | 2 +- separate.py | 79 ++++++++++++- tests/test_tensorrt_rtx.py | 159 +++++++++++++++++++++++++ tests/test_uvr_cli.py | 26 ++++ uvr_cli/README.md | 6 +- uvr_cli/cli.py | 66 ++++++----- uvr_cli/model_resolver.py | 62 +++++++++- 17 files changed, 1197 insertions(+), 64 deletions(-) create mode 100644 engine/tensorrt_rtx/__init__.py create mode 100644 engine/tensorrt_rtx/environment.py create mode 100644 engine/tensorrt_rtx/session.py create mode 100644 install/install_record.py create mode 100644 install/requirements_trt_rtx.txt create mode 100644 tests/test_tensorrt_rtx.py diff --git a/.gitignore b/.gitignore index f204477..5d1e5d1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,11 @@ __pycache__/ .vscode/ venv/ .venv/ +venv_trt_test/ .agents/ benchmark_cache/ +models/**/trt_cache/ +install_logs/ output/* !output/*.txt @@ -21,5 +24,7 @@ error.log CONTRIBUTING.md rubberband.exe ffmpeg.exe +sndfile.dll -install/install_packages.sh \ No newline at end of file +install/install_packages.sh +install/install_record.json \ No newline at end of file diff --git a/UVR.py b/UVR.py index e1d9171..41c9d0d 100644 --- a/UVR.py +++ b/UVR.py @@ -373,6 +373,8 @@ class ModelData(): self.mdx_n_fft_scale_set = None self.wav_type_set = root.wav_type_set# self.device_set = device_set.split(':')[-1].strip() if ':' in device_set else device_set + ep_var = getattr(root, 'execution_provider_var', None) + self.execution_provider = ep_var.get() if ep_var is not None else 'CUDAExecutionProvider' self.mp3_bit_set = root.mp3_bit_set_var.get() self.save_format = root.save_format_var.get() self.is_invert_spec = root.is_invert_spec_var.get()# @@ -3173,23 +3175,72 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): def fill_gpu_list(self): try: - if cuda_available: + from engine.tensorrt_rtx import is_ort_gpu_available + ort_gpu = is_ort_gpu_available() + except Exception: + ort_gpu = False + + try: + if cuda_available and ort_gpu: self.cuda_device_list = [f"{torch.cuda.get_device_properties(i).name}:{i}" for i in range(torch.cuda.device_count())] self.cuda_device_list.insert(0, DEFAULT) - #print(self.cuda_device_list) - - # if directml_available: - # self.opencl_list = [f"{torch_directml.device_name(i)}:{i}" for i in range(torch_directml.device_count())] - # self.opencl_list.insert(0, DEFAULT) + else: + self.cuda_device_list = [DEFAULT] except Exception as e: print(e) + self.cuda_device_list = [DEFAULT] - # if is_cuda_only: - # self.is_use_opencl_var.set(False) - - check_gpu_list = self.cuda_device_list#self.opencl_list if is_opencl_only or self.is_use_opencl_var.get() else self.cuda_device_list + check_gpu_list = self.cuda_device_list if not self.device_set_var.get() in check_gpu_list: self.device_set_var.set(DEFAULT) + self.update_execution_provider_ui() + + def update_execution_provider_ui(self, ep_option=None, ep_label=None): + """Update Execution Provider selection and widget state (task 2).""" + from engine.tensorrt_rtx import get_supported_execution_providers + + is_gpu_selected = self.device_set_var.get() != DEFAULT + dev_id = 0 + if is_gpu_selected: + try: + dev_id = int(self.device_set_var.get().split(':')[-1].strip()) + except Exception: + dev_id = 0 + + supported = get_supported_execution_providers(has_gpu_selected=is_gpu_selected, device_id=dev_id) + self.execution_provider_list = supported + + if ep_option is not None: + ep_option['values'] = self.execution_provider_list + if not is_gpu_selected: + # Rule 3: Lock the Execution Provider option (greyed out) until a GPU Device is chosen + ep_option.configure(state='disabled') + else: + # Rule 2: Enable and allow selection + ep_option.configure(state='readonly') + + if ep_label is not None: + if not is_gpu_selected: + try: + ep_label.configure(state='disabled', foreground='gray') + except Exception: + pass + else: + try: + ep_label.configure(state='normal', foreground='#13849f') + except Exception: + pass + + # Validate stored provider ONLY when a GPU Device is selected (Rule 2). + # While locked (Rule 3), preserve the user's stored choice intact. + if is_gpu_selected: + current_ep = self.execution_provider_var.get() + if current_ep not in self.execution_provider_list: + default_ep = ( + 'CUDAExecutionProvider' if 'CUDAExecutionProvider' in self.execution_provider_list + else (self.execution_provider_list[0] if self.execution_provider_list else 'CUDAExecutionProvider') + ) + self.execution_provider_var.set(default_ep) def loop_gpu_list(self, option_menu:ComboBoxMenu, menu_name, option_list): option_menu['values'] = option_list @@ -3346,7 +3397,9 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): #if not is_choose_arch: self.vocal_splitter_Button_opt(settings_menu, settings_menu_format_Frame, width=SETTINGS_BUT_WIDTH-2, pady=MENU_PADDING_4) - if not is_macos and self.is_gpu_available: + from engine.tensorrt_rtx import is_ort_gpu_available + has_ort_gpu = is_ort_gpu_available() + if not is_macos and self.is_gpu_available and has_ort_gpu: gpu_list_options = lambda:self.loop_gpu_list(device_set_Option, 'gpudevice', self.cuda_device_list)#self.opencl_list if is_opencl_only or self.is_use_opencl_var.get() else self.cuda_device_list) device_set_Label = self.menu_title_LABEL_SET(settings_menu_format_Frame, CUDA_NUM_TEXT) device_set_Label.grid(pady=MENU_PADDING_2) @@ -3355,6 +3408,18 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): device_set_Option.grid(padx=20,pady=MENU_PADDING_1) gpu_list_options() self.help_hints(device_set_Label, text=IS_CUDA_SELECT_HELP) + + execution_provider_Label = self.menu_title_LABEL_SET(settings_menu_format_Frame, EXECUTION_PROVIDER_TEXT) + execution_provider_Label.grid(pady=MENU_PADDING_2) + + execution_provider_Option = ComboBoxMenu(settings_menu_format_Frame, textvariable=self.execution_provider_var, values=self.execution_provider_list, width=GEN_SETTINGS_WIDTH+1) + execution_provider_Option.grid(padx=20, pady=MENU_PADDING_1) + self.help_hints(execution_provider_Label, text=IS_EXECUTION_PROVIDER_HELP) + + update_ep_callback = lambda *args: self.update_execution_provider_ui(execution_provider_Option, execution_provider_Label) + device_set_Option.bind('<>', lambda e: update_ep_callback()) + update_ep_callback() + model_precision_Label = self.menu_title_LABEL_SET(settings_menu_format_Frame, 'Precision Mode Preset') model_precision_Label.grid(pady=MENU_PADDING_2) @@ -6852,6 +6917,8 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): self.save_format_var = tk.StringVar(value=data['save_format']) self.wav_type_set_var = tk.StringVar(value=data['wav_type_set'])# self.device_set_var = tk.StringVar(value=data['device_set'])# + self.execution_provider_var = tk.StringVar(value=data.get('execution_provider', 'CUDAExecutionProvider')) + self.execution_provider_list = ['CUDAExecutionProvider'] self.user_code_var = tk.StringVar(value=data['user_code']) self.is_gpu_conversion_var = tk.BooleanVar(value=data['is_gpu_conversion']) self.model_precision_var = tk.StringVar(value=normalize_model_precision(data.get('model_precision'))) @@ -6998,6 +7065,7 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): self.save_format_var.set(loaded_setting['save_format']) self.wav_type_set_var.set(loaded_setting['wav_type_set'])# self.device_set_var.set(loaded_setting['device_set'])# + self.execution_provider_var.set(loaded_setting.get('execution_provider', 'CUDAExecutionProvider')) self.user_code_var.set(loaded_setting['user_code']) self.phase_option_var.set(loaded_setting['phase_option'])# self.phase_shifts_var.set(loaded_setting['phase_shifts'])# @@ -7138,6 +7206,7 @@ class MainWindow(TkinterDnD.Tk if is_dnd_compatible else tk.Tk): 'save_format': self.save_format_var.get(), 'wav_type_set': self.wav_type_set_var.get(),# 'device_set': self.device_set_var.get(),# + 'execution_provider': self.execution_provider_var.get(), 'user_code': self.user_code_var.get(), 'help_hints_var': self.help_hints_var.get(), 'set_vocal_splitter': self.set_vocal_splitter_var.get(), diff --git a/__version__.py b/__version__.py index 0e55052..65cb27f 100644 --- a/__version__.py +++ b/__version__.py @@ -1,5 +1,5 @@ VERSION = 'v0.1.0-beta.1' -BUILD_ID = 'RLUV_20261006_1949' # represent RLUV_yyyymmdd_HHMM (year, month, day, hour, minute) +BUILD_ID = 'RLUV_20261009_1604' # represent RLUV_yyyymmdd_HHMM (year, month, day, hour, minute) PATCH = BUILD_ID PATCH_MAC = 'Not yet available' PATCH_LINUX = 'Not yet available' diff --git a/engine/tensorrt_rtx/__init__.py b/engine/tensorrt_rtx/__init__.py new file mode 100644 index 0000000..7c96339 --- /dev/null +++ b/engine/tensorrt_rtx/__init__.py @@ -0,0 +1,39 @@ +"""TensorRT-RTX execution provider wrapper package for Ultimate Vocal Remover.""" + +from __future__ import annotations + +from .environment import ( + PROVIDER_CPU, + PROVIDER_CUDA, + PROVIDER_TRT_PLUGIN, + PROVIDER_TRT_RTX, + get_supported_execution_providers, + is_ort_gpu_available, + is_trt_rtx_available, + is_trt_rtx_installed, + register_trt_rtx_libraries, +) +from .session import ( + DEFAULT_CUDA_OPTIONS, + build_trt_options, + build_trt_rtx_options, + create_trt_rtx_session, + get_default_cache_dir, +) + +__all__ = [ + "PROVIDER_CPU", + "PROVIDER_CUDA", + "PROVIDER_TRT_PLUGIN", + "PROVIDER_TRT_RTX", + "DEFAULT_CUDA_OPTIONS", + "is_trt_rtx_installed", + "is_trt_rtx_available", + "is_ort_gpu_available", + "get_supported_execution_providers", + "build_trt_options", + "build_trt_rtx_options", + "create_trt_rtx_session", + "get_default_cache_dir", + "register_trt_rtx_libraries", +] diff --git a/engine/tensorrt_rtx/environment.py b/engine/tensorrt_rtx/environment.py new file mode 100644 index 0000000..228979a --- /dev/null +++ b/engine/tensorrt_rtx/environment.py @@ -0,0 +1,155 @@ +"""Environment detection and runtime provider discovery for TensorRT-RTX.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import List, Optional + +import torch + +# Execution Provider names +PROVIDER_TRT_RTX = "NvTensorRTRTXExecutionProvider" +PROVIDER_TRT_PLUGIN = "nv_tensorrt_rtx" +PROVIDER_CUDA = "CUDAExecutionProvider" +PROVIDER_CPU = "CPUExecutionProvider" + +_PLUGIN_REGISTERED = False + + +def register_trt_rtx_libraries() -> None: + """Register TensorRT-RTX DLL directories and plugin library with ONNX Runtime.""" + global _PLUGIN_REGISTERED + if _PLUGIN_REGISTERED: + return + + # 1. On Windows, ensure bundled pip plugin library DLL directory is in search path + if sys.platform == "win32": + try: + import site + for site_dir in site.getsitepackages(): + rtx_dir = os.path.join(site_dir, "onnxruntime_ep_nv_tensorrt_rtx") + if os.path.isdir(rtx_dir): + try: + os.add_dll_directory(rtx_dir) + except (AttributeError, OSError) as exc: + print(f"[BACKEND] Warning: Failed to add DLL directory '{rtx_dir}': {exc}") + os.environ["PATH"] = rtx_dir + os.pathsep + os.environ.get("PATH", "") + except Exception as exc: + print(f"[BACKEND] Warning: Error configuring Windows DLL search paths: {exc}") + + # 2. Register the standalone EP ABI plugin with ONNX Runtime + try: + import onnxruntime as ort + if hasattr(ort, "register_execution_provider_library"): + try: + import onnxruntime_ep_nv_tensorrt_rtx as trt_ep + ep_name = trt_ep.get_ep_name() + ep_lib = trt_ep.get_library_path() + print(f"[BACKEND] Registering TensorRT-RTX execution provider library '{ep_name}' ({ep_lib})...") + ort.register_execution_provider_library(ep_name, ep_lib) + print(f"[BACKEND] Successfully registered TensorRT-RTX execution provider '{ep_name}'.") + except ImportError: + # Standalone plugin package is not installed; normal for standard/CPU installs + pass + except Exception as exc: + print(f"[BACKEND] Warning: Failed to register TensorRT-RTX library with ONNX Runtime: {exc}") + except Exception as exc: + print(f"[BACKEND] Warning: Error checking ONNX Runtime plugin support: {exc}") + + _PLUGIN_REGISTERED = True + + +def is_trt_rtx_installed() -> bool: + """Check if TensorRT-RTX EP is genuinely registered and available in ONNX Runtime. + + Standard TensorrtExecutionProvider is deliberately NOT treated as TensorRT-RTX. + """ + register_trt_rtx_libraries() + try: + import onnxruntime as ort + all_provs = ort.get_all_providers() if hasattr(ort, "get_all_providers") else [] + avail_provs = ort.get_available_providers() + + # Check standalone EP ABI plugin via device enumeration + if hasattr(ort, "get_ep_devices"): + try: + import onnxruntime_ep_nv_tensorrt_rtx as trt_ep + plugin_name = trt_ep.get_ep_name() + devices = [d for d in ort.get_ep_devices() if getattr(d, "ep_name", "") == plugin_name] + if devices: + return True + except ImportError: + pass + except Exception as exc: + print(f"[BACKEND] Error inspecting TensorRT-RTX EP devices: {exc}") + + # Check built-in provider name (NvTensorRTRTXExecutionProvider) + if PROVIDER_TRT_RTX in all_provs and PROVIDER_TRT_RTX in avail_provs: + return True + except Exception as exc: + print(f"[BACKEND] Error checking TensorRT-RTX installation: {exc}") + + return False + + +def is_trt_rtx_available(device_id: int = 0) -> bool: + """Check whether TensorRT-RTX can genuinely execute on the specified GPU device.""" + if not torch.cuda.is_available(): + return False + if device_id < 0 or device_id >= torch.cuda.device_count(): + return False + return is_trt_rtx_installed() + + +def is_ort_gpu_available() -> bool: + """Check whether ONNX Runtime supports any GPU execution provider.""" + try: + import onnxruntime as ort + avail = ort.get_available_providers() + gpu_providers = ( + PROVIDER_CUDA, + "ROCMExecutionProvider", + "MIGraphXExecutionProvider", + "DmlExecutionProvider", + ) + if any(p in avail for p in gpu_providers): + return True + if is_trt_rtx_installed(): + return True + except Exception as exc: + print(f"[BACKEND] Warning: Failed to query ONNX Runtime GPU providers: {exc}") + return False + + +def get_supported_execution_providers(has_gpu_selected: bool = False, device_id: int = 0) -> List[str]: + """Return a list of execution providers supported by the current environment and selection. + + Follows GUI Rules: + - If no GPU is selected (has_gpu_selected=False): returns [CPUExecutionProvider]. + - If GPU is selected: auto-detects providers supported by that GPU (TRT-RTX if available, CUDA, CPU). + """ + providers: List[str] = [] + if not has_gpu_selected or not torch.cuda.is_available(): + return [PROVIDER_CPU] + + # GPU is selected: check available providers + register_trt_rtx_libraries() + try: + import onnxruntime as ort + avail = ort.get_available_providers() + + # Check TensorRT-RTX capability + if is_trt_rtx_available(device_id): + providers.append(PROVIDER_TRT_RTX) + + if PROVIDER_CUDA in avail: + providers.append(PROVIDER_CUDA) + except Exception as exc: + print(f"[BACKEND] Error discovering supported execution providers: {exc}") + + if not providers: + providers.append(PROVIDER_CPU) + + return providers diff --git a/engine/tensorrt_rtx/session.py b/engine/tensorrt_rtx/session.py new file mode 100644 index 0000000..5d3f9dc --- /dev/null +++ b/engine/tensorrt_rtx/session.py @@ -0,0 +1,147 @@ +"""Session creation and lifecycle management for TensorRT-RTX with graceful fallback.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from .environment import ( + PROVIDER_CPU, + PROVIDER_CUDA, + PROVIDER_TRT_PLUGIN, + PROVIDER_TRT_RTX, + register_trt_rtx_libraries, +) + +DEFAULT_CUDA_OPTIONS = { + "arena_extend_strategy": "kSameAsRequested", + "cudnn_conv_algo_search": "HEURISTIC", + "do_copy_in_default_stream": "True", +} + + +def get_default_cache_dir() -> Path: + """Return default directory for cached TensorRT engine plans.""" + project_root = Path(__file__).resolve().parents[2] + cache_dir = project_root / "models" / "MDX_Net_Models" / "trt_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir + + +def build_trt_rtx_options( + device_id: int = 0, + cache_dir: Optional[Union[str, Path]] = None, + enable_cuda_graph: bool = False, +) -> Dict[str, str]: + """Construct configuration options dictionary for TensorRT-RTX Execution Provider.""" + target_cache = Path(cache_dir) if cache_dir else get_default_cache_dir() + target_cache.mkdir(parents=True, exist_ok=True) + + opts = { + "device_id": str(int(device_id)), + "nv_runtime_cache_path": str(target_cache), + } + if enable_cuda_graph: + opts["enable_cuda_graph"] = "1" + return opts + + +# Backward-compatibility alias +build_trt_options = build_trt_rtx_options + + +def create_trt_rtx_session( + model_input: Union[str, Path, bytes], + device_id: int = 0, + cuda_options: Optional[Dict[str, Any]] = None, + cache_dir: Optional[Union[str, Path]] = None, + model_name: Optional[str] = None, + enable_cuda_graph: bool = False, +) -> Any: + """Create an ONNX Runtime InferenceSession preferring TensorRT-RTX with automatic fallback. + + Fallback chain: + 1. TensorRT-RTX EP (via EP plugin or built-in provider) + 2. CUDAExecutionProvider + 3. CPUExecutionProvider + """ + import onnxruntime as ort + + register_trt_rtx_libraries() + cuda_opts = cuda_options or DEFAULT_CUDA_OPTIONS + trt_opts = build_trt_rtx_options( + device_id=device_id, + cache_dir=cache_dir, + enable_cuda_graph=enable_cuda_graph, + ) + + if model_name: + display_name = model_name + elif isinstance(model_input, (str, Path)): + display_name = os.path.basename(str(model_input)) + else: + display_name = f"in-memory-model ({len(model_input) / (1024 ** 2):.2f} MiB)" + + # Strategy 1: Standalone EP ABI plugin (onnxruntime-ep-nv-tensorrt-rtx) + try: + if hasattr(ort, "get_ep_devices"): + import onnxruntime_ep_nv_tensorrt_rtx as trt_ep + plugin_ep_name = trt_ep.get_ep_name() + ep_devices = [ + d for d in ort.get_ep_devices() + if getattr(d, "ep_name", "") == plugin_ep_name + ] + if ep_devices: + # Select the first matching TensorRT-RTX EP device + target_dev = ep_devices[0] + print(f"[BACKEND] Initializing TensorRT-RTX plugin session for {display_name}...") + so = ort.SessionOptions() + so.add_provider_for_devices([target_dev], trt_opts) + session = ort.InferenceSession(model_input, sess_options=so) + active = session.get_providers()[0] if hasattr(session, "get_providers") else PROVIDER_TRT_RTX + print(f"[BACKEND] Active execution provider: {active}") + return session + except Exception as exc: + print(f"[BACKEND] TensorRT-RTX plugin session creation failed ({type(exc).__name__}: {exc})") + print("[BACKEND] Falling back to CUDAExecutionProvider...") + + # Strategy 2: Built-in NvTensorRTRTXExecutionProvider (if present) + avail_provs = ort.get_available_providers() + if PROVIDER_TRT_RTX in avail_provs: + try: + print(f"[BACKEND] Initializing built-in {PROVIDER_TRT_RTX} session for {display_name}...") + providers: List[Union[str, Tuple[str, Dict[str, Any]]]] = [ + (PROVIDER_TRT_RTX, trt_opts), + (PROVIDER_CUDA, cuda_opts), + PROVIDER_CPU, + ] + session = ort.InferenceSession(model_input, providers=providers) + active = session.get_providers()[0] if hasattr(session, "get_providers") else PROVIDER_TRT_RTX + print(f"[BACKEND] Active execution provider: {active}") + return session + except Exception as exc: + print(f"[BACKEND] {PROVIDER_TRT_RTX} session creation failed ({type(exc).__name__}: {exc})") + print("[BACKEND] Falling back to CUDAExecutionProvider...") + + # Strategy 3: CUDAExecutionProvider fallback + if PROVIDER_CUDA in avail_provs: + try: + print(f"[BACKEND] Creating CUDAExecutionProvider session for {display_name}...") + session = ort.InferenceSession( + model_input, + providers=[(PROVIDER_CUDA, cuda_opts), PROVIDER_CPU], + ) + active = session.get_providers()[0] if hasattr(session, "get_providers") else PROVIDER_CUDA + print(f"[BACKEND] Active execution provider: {active}") + return session + except Exception as exc: + print(f"[BACKEND] CUDAExecutionProvider failed ({type(exc).__name__}: {exc})") + print("[BACKEND] Falling back to CPUExecutionProvider...") + + # Strategy 4: CPUExecutionProvider fallback + print(f"[BACKEND] Creating CPUExecutionProvider session for {display_name}...") + session = ort.InferenceSession(model_input, providers=[PROVIDER_CPU]) + print(f"[BACKEND] Active execution provider: {PROVIDER_CPU}") + return session diff --git a/gui_data/constants.py b/gui_data/constants.py index a155327..44b58d1 100644 --- a/gui_data/constants.py +++ b/gui_data/constants.py @@ -670,6 +670,7 @@ DEFAULT_DATA = { 'save_format': WAV, 'wav_type_set': 'PCM_16', 'device_set': DEFAULT, + 'execution_provider': 'CUDAExecutionProvider', 'user_code': '', 'export_path': '', 'input_paths': [], @@ -781,6 +782,7 @@ SETTING_CHECK = ('vr_model', 'save_format', 'wav_type_set', 'device_set', + 'execution_provider', 'user_code', 'is_gpu_conversion', 'model_precision', @@ -1050,6 +1052,10 @@ IS_TIME_CORRECTION_HELP = ('When checked, the output will retain the original BP SAVE_STEM_ONLY_HELP = 'Allows the user to save only the selected stem.' IS_NORMALIZATION_HELP = 'Normalizes output to prevent clipping.' IS_CUDA_SELECT_HELP = "If you have more than one GPU, you can pick which one to use for processing." +EXECUTION_PROVIDER_TEXT = "Execution Provider" +IS_EXECUTION_PROVIDER_HELP = ("Select the ONNX Runtime execution provider for accelerated inference:\n" + "• CUDAExecutionProvider: Stable & proven baseline for all NVIDIA GPUs.\n" + "• NvTensorRTRTXExecutionProvider / TensorrtExecutionProvider: Experimental fast TensorRT-RTX acceleration.") CROP_SIZE_HELP = '**Only compatible with select models only!**\n\n Setting should match training crop-size value. Leave as is if unsure.' IS_TTA_HELP = ('This option performs Test-Time-Augmentation to improve the separation quality.\n\n' 'Note: Having this selected will increase the time it takes to complete a conversion') diff --git a/install/install_packages.bat b/install/install_packages.bat index 111e2e2..f898d89 100644 --- a/install/install_packages.bat +++ b/install/install_packages.bat @@ -48,6 +48,7 @@ if not exist "%REQ_FILE%" ( call :check_windows call :detect_gpu +call :check_previous_install call :choose_mode call :ensure_uv if errorlevel 1 exit /b 1 @@ -58,6 +59,8 @@ if errorlevel 1 exit /b 1 if /i "%INSTALL_MODE%"=="CPU" ( call :install_cpu_stack +) else if /i "%INSTALL_MODE%"=="TENSORRT_RTX" ( + call :install_trt_rtx_stack ) else ( call :install_nvidia_stack ) @@ -73,6 +76,7 @@ if errorlevel 1 exit /b 1 call :smoke_test if errorlevel 1 exit /b 1 +call :record_install call :done exit /b 0 @@ -188,29 +192,45 @@ if "!DRIVER_SUPPORTED!"=="0" ( ) exit /b 0 +:check_previous_install +set "PREV_RECORD_EXISTS=0" +set "PREV_MODE=" +set "PREV_OPTION=" +if exist "%~dp0install_record.json" if exist "%VENV_PYTHON%" ( + for /f "tokens=1,2 delims==" %%A in ('"%VENV_PYTHON%" "%~dp0install_record.py" read 2^>nul') do ( + if "%%A"=="MODE" set "PREV_MODE=%%B" + if "%%A"=="OPTION" set "PREV_OPTION=%%B" + set "PREV_RECORD_EXISTS=1" + ) +) +if "!PREV_RECORD_EXISTS!"=="1" ( + echo Previous installation detected : [!PREV_OPTION!] !PREV_MODE! + call :log "Previous install detected: Option !PREV_OPTION!, Mode !PREV_MODE!" +) +exit /b 0 + :choose_mode -set "INSTALL_MODE=AUTO" +set "INSTALL_MODE=NVIDIA" set "USER_FORCED_NVIDIA=0" echo. echo Choose installation mode: -echo [A] Auto - choose NVIDIA for detected RTX GPUs; otherwise CPU -echo [N] NVIDIA GPU - force PyTorch CUDA 12.8 -echo [C] CPU - safest fallback, including AMD systems because AMD is not supported yet +echo [A] CUDA EP - Choose this for the stable and safe option +echo [B] TensorRT-RTX EP - Choose this if you like experimental, fast, and optimized (CUDA EP will still exist as a stable fallback) +echo [C] CPU - safest fallback, including AMD systems because AMD is not supported yet if "%HAS_NVIDIA%"=="0" if "%HAS_AMD%"=="1" echo Note: AMD GPU detected, but AMD acceleration is not supported yet. Auto will use CPU. echo. -call :prompt_choice MODE_CHOICE "Mode [A/N/C, default A]: " "A N C" "A" "A, N, or C" -if /i "!MODE_CHOICE!"=="N" ( +call :prompt_choice MODE_CHOICE "Mode [A/B/C, default A]: " "A B C" "A" "A, B, or C" +if /i "!MODE_CHOICE!"=="A" ( set "INSTALL_MODE=NVIDIA" set "USER_FORCED_NVIDIA=1" ) +if /i "!MODE_CHOICE!"=="B" ( + set "INSTALL_MODE=TENSORRT_RTX" + set "USER_FORCED_NVIDIA=1" +) if /i "!MODE_CHOICE!"=="C" set "INSTALL_MODE=CPU" -if /i "%INSTALL_MODE%"=="AUTO" ( - if "%HAS_NVIDIA%"=="1" if "%RTX_SUPPORTED%"=="1" set "INSTALL_MODE=NVIDIA" - if /i "!INSTALL_MODE!"=="AUTO" set "INSTALL_MODE=CPU" -) - -if /i "!INSTALL_MODE!"=="NVIDIA" if "%USER_FORCED_NVIDIA%"=="1" ( +if not "!INSTALL_MODE!"=="CPU" if "%USER_FORCED_NVIDIA%"=="1" ( if "%HAS_NVIDIA%"=="0" call :confirm_risky "NVIDIA was not detected. Force GPU stack installation anyway?" if /i "!INSTALL_MODE!"=="CPU" goto :choose_mode_done if "%RTX_SUPPORTED%"=="0" call :confirm_risky "The GPU is outside the RTX 20+ target. Force GPU stack installation anyway?" @@ -283,6 +303,9 @@ if /i "%INSTALL_MODE%"=="CPU" ( if errorlevel 1 exit /b 1 set "INSTALL_REQ=%CPU_REQ_FILE%" ) +if /i "%INSTALL_MODE%"=="TENSORRT_RTX" ( + set "INSTALL_REQ=requirements_trt_rtx.txt" +) if exist "%~dp0wheels\diffq-0.2.4-cp311-cp311-win_amd64.whl" ( call :log "Pre-installing local binary wheel for diffq" @@ -325,7 +348,7 @@ exit /b 0 :install_nvidia_stack echo. -echo Installing NVIDIA CUDA PyTorch stack... +echo Installing NVIDIA CUDA PyTorch stack (Option A - CUDA EP)... call :log "Installing PyTorch CUDA from %TORCH_CU128%" uv pip install --python "%VENV_PYTHON%" %TORCH_PACKAGES% --index-url "%TORCH_CU128%" if errorlevel 1 ( @@ -341,11 +364,32 @@ if errorlevel 1 ( ) echo Installing ONNX Runtime GPU compatibility package... -uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime >nul 2>&1 +uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime onnxruntime-ep-nv-tensorrt-rtx-cu12 >nul 2>&1 uv pip install --python "%VENV_PYTHON%" onnxruntime-gpu==1.22.0 if errorlevel 1 call :warn "Failed to install onnxruntime-gpu. UVR can still use PyTorch CUDA, but ONNX models may fall back or fail." exit /b 0 +:install_trt_rtx_stack +echo. +echo Installing NVIDIA CUDA PyTorch stack (Option B - TensorRT-RTX EP)... +call :log "Installing PyTorch CUDA from %TORCH_CU128%" +uv pip install --python "%VENV_PYTHON%" %TORCH_PACKAGES% --index-url "%TORCH_CU128%" +if errorlevel 1 ( + call :warn "PyTorch CUDA installation failed. Offering CPU fallback." + call :prompt_choice CPU_FALLBACK "Install CPU stack instead? [Y/N, default Y]: " "Y N" "Y" "Y or N" + if /i "!CPU_FALLBACK!"=="N" ( + call :fail "PyTorch CUDA installation failed and CPU fallback was cancelled." + exit /b 1 + ) + set "INSTALL_MODE=CPU" + call :install_cpu_stack + exit /b 0 +) + +echo Preparing TensorRT-RTX packages... +uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime >nul 2>&1 +exit /b 0 + :install_cpu_stack echo. echo Installing CPU fallback stack... @@ -356,7 +400,7 @@ if errorlevel 1 ( exit /b 1 ) -uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime-gpu >nul 2>&1 +uv pip uninstall --python "%VENV_PYTHON%" -y onnxruntime-gpu onnxruntime-ep-nv-tensorrt-rtx-cu12 >nul 2>&1 uv pip install --python "%VENV_PYTHON%" onnxruntime==1.22.0 if errorlevel 1 call :warn "Failed to install ONNX Runtime CPU. PyTorch models can still run, but ONNX models may fail." exit /b 0 @@ -578,15 +622,30 @@ if errorlevel 1 ( "%VENV_PYTHON%" -c "import onnxruntime as ort; print('ONNX Runtime providers:', ort.get_available_providers())" if errorlevel 1 call :warn "ONNX Runtime smoke test failed. UVR can still run non-ONNX models." -if /i "%INSTALL_MODE%"=="NVIDIA" ( +if /i "%INSTALL_MODE%"=="TENSORRT_RTX" ( + "%VENV_PYTHON%" -c "import onnxruntime as ort; import onnxruntime_ep_nv_tensorrt_rtx as trt_ep; ort.register_execution_provider_library(trt_ep.get_ep_name(), trt_ep.get_library_path()); print('TensorRT-RTX plugin loaded successfully.')" 2>nul + if errorlevel 1 call :warn "TensorRT-RTX EP plugin could not be initialized. CUDA EP will still be used as stable fallback." +) + +if not "%INSTALL_MODE%"=="CPU" ( "%VENV_PYTHON%" -c "import torch, sys; sys.exit(0 if torch.cuda.is_available() else 2)" if errorlevel 1 ( - call :warn "NVIDIA mode is installed, but torch.cuda.is_available() is still False." + call :warn "GPU mode is installed, but torch.cuda.is_available() is still False." call :warn "Likely causes: outdated driver, DLL conflict, or incompatible PyTorch wheel." ) ) exit /b 0 +:record_install +echo. +echo Recording install state... +"%VENV_PYTHON%" "%~dp0install_record.py" write --option "!MODE_CHOICE!" --mode "!INSTALL_MODE!" --reuse-venv "!REUSE_VENV!" --has-uv "1" --has-ffmpeg "!HAS_FFMPEG!" --has-rubberband "!HAS_RB!" --has-vc-redist "!VC_INSTALLED!" >nul 2>&1 +if not errorlevel 1 ( + call :log "Successfully wrote install record to install_record.json" +) +exit /b 0 + + :done echo. echo ============================================================ diff --git a/install/install_record.py b/install/install_record.py new file mode 100644 index 0000000..57b4b7d --- /dev/null +++ b/install/install_record.py @@ -0,0 +1,184 @@ +"""Persistent install record manager for UVR installation. + +Tracks installer mode, detected dependencies, installed package versions, +and user choices to enable seamless switching between modes (CUDA, TensorRT-RTX, CPU). +""" + +from __future__ import annotations + +import argparse +import datetime +import importlib.metadata +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, Optional + +INSTALL_DIR = Path(__file__).resolve().parent +ROOT_DIR = INSTALL_DIR.parent +DEFAULT_RECORD_PATH = INSTALL_DIR / "install_record.json" + + +def get_app_version() -> Dict[str, str]: + """Extract app version and build ID from __version__.py.""" + version_file = ROOT_DIR / "__version__.py" + version_info = {"version": "unknown", "build_id": "unknown"} + if version_file.is_file(): + namespace: Dict[str, Any] = {} + try: + with open(version_file, "r", encoding="utf-8") as f: + exec(f.read(), namespace) + version_info["version"] = str(namespace.get("VERSION", "unknown")) + version_info["build_id"] = str(namespace.get("BUILD_ID", "unknown")) + except Exception: + pass + return version_info + + +def get_package_version(pkg_name: str) -> Optional[str]: + """Retrieve installed version of a Python package safely.""" + try: + return importlib.metadata.version(pkg_name) + except Exception: + return None + + +def read_install_record(record_path: Optional[Path] = None) -> Optional[Dict[str, Any]]: + """Read and validate the existing install record. Returns None if absent or invalid.""" + path = record_path or DEFAULT_RECORD_PATH + if not path.is_file(): + return None + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict) and "install_mode" in data: + return data + except Exception: + pass + return None + + +def inspect_installed_packages() -> Dict[str, Any]: + """Inspect installed versions of key runtime dependencies.""" + return { + "pytorch": { + "torch": get_package_version("torch"), + "torchvision": get_package_version("torchvision"), + "torchaudio": get_package_version("torchaudio"), + }, + "onnxruntime": { + "onnxruntime": get_package_version("onnxruntime"), + "onnxruntime_gpu": get_package_version("onnxruntime-gpu"), + }, + "tensorrt_rtx": { + "onnxruntime_ep_nv_tensorrt_rtx_cu12": get_package_version("onnxruntime-ep-nv-tensorrt-rtx-cu12"), + "tensorrt_cu12_libs": get_package_version("tensorrt-cu12-libs"), + }, + } + + +def write_install_record( + option: str, + install_mode: str, + reuse_venv: str = "Y", + has_uv: bool = True, + has_ffmpeg: bool = True, + has_rubberband: bool = True, + has_vc_redist: bool = True, + record_path: Optional[Path] = None, +) -> Path: + """Record a successful installation configuration to JSON.""" + path = record_path or DEFAULT_RECORD_PATH + app_version = get_app_version() + packages = inspect_installed_packages() + + record: Dict[str, Any] = { + "schema_version": 1, + "timestamp_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "app_version": app_version["version"], + "app_build_id": app_version["build_id"], + "selected_option": option.upper(), + "install_mode": install_mode.upper(), + "user_prompts": { + "reuse_venv": reuse_venv.upper(), + }, + "system_components": { + "venv_path": str(ROOT_DIR / "venv"), + "uv_detected": has_uv, + "ffmpeg_present": has_ffmpeg, + "rubberband_present": has_rubberband, + "vc_redist_detected": has_vc_redist, + }, + "stack_profile": ( + "tensorrt_rtx_ort124" if option.upper() == "B" + else ("cpu_ort122" if option.upper() == "C" else "stable_cuda_ort122") + ), + "ort_stack_note": ( + "Option B uses onnxruntime-gpu>=1.24.4 for TensorRT-RTX EP ABI plugin compatibility (requires ORT>=1.24); CUDA EP remains available as stable fallback." + if option.upper() == "B" + else "Option A/C uses stable benchmarked onnxruntime==1.22.0 stack." + ), + "installed_packages": packages, + } + + try: + with open(path, "w", encoding="utf-8") as f: + json.dump(record, f, indent=2) + except Exception as exc: + print(f"[WARN] Failed to write install record: {exc}", file=sys.stderr) + + return path + + +def main() -> int: + parser = argparse.ArgumentParser(description="UVR Install Record Utility") + subparsers = parser.add_subparsers(dest="action") + + read_p = subparsers.add_parser("read", help="Read existing record") + read_p.add_argument("--json", action="store_true", help="Output raw JSON") + + write_p = subparsers.add_parser("write", help="Write successful install record") + write_p.add_argument("--option", required=True, help="Selected option letter (A, B, or C)") + write_p.add_argument("--mode", required=True, help="Resulting install mode (NVIDIA, TENSORRT_RTX, CPU)") + write_p.add_argument("--reuse-venv", default="Y", help="User answer for venv reuse (Y/N)") + write_p.add_argument("--has-uv", default="1", help="Whether uv is present (1/0)") + write_p.add_argument("--has-ffmpeg", default="1", help="Whether ffmpeg is present (1/0)") + write_p.add_argument("--has-rubberband", default="1", help="Whether rubberband is present (1/0)") + write_p.add_argument("--has-vc-redist", default="1", help="Whether VC redist is present (1/0)") + + args = parser.parse_args() + + if args.action == "read": + record = read_install_record() + if not record: + print("NO_RECORD") + return 0 + if args.json: + print(json.dumps(record, indent=2)) + else: + print(f"MODE={record.get('install_mode', 'UNKNOWN')}") + print(f"OPTION={record.get('selected_option', 'UNKNOWN')}") + print(f"VERSION={record.get('app_version', 'UNKNOWN')}") + print(f"TIMESTAMP={record.get('timestamp_utc', 'UNKNOWN')}") + return 0 + + elif args.action == "write": + write_install_record( + option=args.option, + install_mode=args.mode, + reuse_venv=args.reuse_venv, + has_uv=args.has_uv == "1", + has_ffmpeg=args.has_ffmpeg == "1", + has_rubberband=args.has_rubberband == "1", + has_vc_redist=args.has_vc_redist == "1", + ) + print("RECORD_WRITTEN") + return 0 + + parser.print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/install/requirements_trt_rtx.txt b/install/requirements_trt_rtx.txt new file mode 100644 index 0000000..5e7fc72 --- /dev/null +++ b/install/requirements_trt_rtx.txt @@ -0,0 +1,137 @@ +absl-py==2.4.0 +aiohappyeyeballs==2.6.2 +aiohttp==3.14.1 +aiosignal==1.4.0 +altgraph==0.17.3 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +attrs==26.1.0 +audioread==3.1.0 +certifi==2026.7.22 +cffi==2.1.0 +charset-normalizer==3.4.9 +colorama==0.4.6 +coloredlogs==15.0.1 +contextlib2==21.6.0 +contourpy==1.3.3 +cppimport==26.4.17 +cryptography==3.4.6 +cuda-pathfinder==1.6.0 +cupy-cuda12x==14.1.1 +cycler==0.12.1 +cython==3.2.6 +decorator==5.3.1 +einops==0.6.0 +filelock==3.29.0 +flatbuffers==25.12.19 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2026.4.0 +future==0.18.3 +humanfriendly==10.0 +idna==3.18 +jinja2==3.1.6 +joblib==1.5.3 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +julius==0.2.7 +kiwisolver==1.5.0 +kthread==0.2.3 +lazy-loader==0.5 +librosa==0.11.0 +lief==1.0.0 +lightning-utilities==0.15.3 +llvmlite==0.48.0 +mako==1.3.12 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +matchering==2.0.6 +matplotlib==3.11.0 +mdurl==0.1.2 +ml-collections==0.1.1 +ml-dtypes==0.5.4 +mpmath==1.3.0 +msgpack==1.2.1 +multidict==6.7.1 +narwhals==2.24.0 +natsort==8.2.0 +networkx==3.6.1 +ninja==1.13.0 +numba==0.66.0 +numpy==2.4.4 +# nvidia-cublas-cu12==12.9.2.10 +# nvidia-cuda-nvrtc-cu12==12.9.86 +# nvidia-cuda-runtime-cu12==12.9.79 +# nvidia-cudnn-cu12==9.23.2.1 +# nvidia-cufft-cu12==11.4.1.4 +# nvidia-curand-cu12==10.3.10.19 +nvidia-ml-py==13.610.43 +# nvidia-modelopt==0.45.0 +# nvidia-nvjitlink-cu12==12.9.86 +omegaconf==2.3.1 +onnx==1.21.0 +onnx-graphsurgeon==0.6.1 +onnx-ir==0.2.1 +onnx2pytorch==0.5.3 +onnxconverter-common==1.16.0 +onnxruntime-gpu==1.24.4 +onnxruntime-ep-nv-tensorrt-rtx-cu12==0.4.0 +onnxscript==0.7.1 +onnxslim==0.1.94 +opencv-python==4.6.0.66 +packaging==26.2 +pandas==2.3.3 +patsy==1.0.2 +pillow==12.2.0 +pip==24.0 +platformdirs==4.11.0 +playsound==1.2.2 +polygraphy==0.50.3 +pooch==1.9.0 +propcache==0.5.2 +protobuf==7.35.1 +psutil==5.9.4 +pulp==3.3.2 +pybind11==3.0.4 +pycparser==3.0 +pydantic==2.13.4 +pydantic-core==2.46.4 +pydub==0.25.1 +pyglet==1.5.23 +pygments==2.20.0 +pyparsing==3.3.2 +pyperclip==1.8.2 +pyreadline3==3.5.6 +pyrubberband==0.3.0 +pysoundfile==0.9.0.post1 +python-dateutil==2.9.0.post0 +pytorch-lightning==2.0.0 +pytz==2026.2 +pyyaml==6.0 +referencing==0.37.0 +regex==2026.7.19 +requests==2.34.2 +resampy==0.4.2 +rich==15.0.0 +rpds-py==2026.6.3 +safetensors==0.8.0 +samplerate==0.1.0 +scikit-learn==1.9.0 +scipy==1.17.1 +screeninfo==0.8.1 +setuptools==78.1.0 +six==1.17.0 +soundfile==0.14.0 +soundstretch==1.2 +soxr==1.1.0 +statsmodels==0.14.6 +sympy==1.14.0 +threadpoolctl==3.6.0 +torchmetrics==1.9.0 +tqdm==4.68.3 +typing-extensions==4.15.0 +typing-inspection==0.4.2 +tzdata==2026.2 +urllib3==2.7.0 +wget==3.2 +yarl==1.24.2 diff --git a/run_uvr.bat b/run_uvr.bat index 173fe72..2cf276b 100644 --- a/run_uvr.bat +++ b/run_uvr.bat @@ -13,7 +13,7 @@ if not exist "%UVR_PYTHON%" ( ) echo Starting Ultimate Vocal Remover... -powershell.exe -NoLogo -NoProfile -Command "& { & $env:UVR_PYTHON -u (Join-Path $env:UVR_ROOT 'UVR.py') 2>&1 | Tee-Object -FilePath $env:UVR_LOG; $uvrExitCode = $LASTEXITCODE; if ($null -eq $uvrExitCode) { $uvrExitCode = 1 }; exit $uvrExitCode }" +"%UVR_PYTHON%" "%UVR_ROOT%UVR.py" set "UVR_EXIT_CODE=%ERRORLEVEL%" if not "%UVR_EXIT_CODE%"=="0" echo [ERROR] UVR exited with code %UVR_EXIT_CODE%. See uvr_run.log for details. pause diff --git a/separate.py b/separate.py index 8001176..8d33b8b 100644 --- a/separate.py +++ b/separate.py @@ -409,6 +409,78 @@ def create_onnx_fp16_session(model_path, providers): def create_onnx_session(model_path, providers, precision_policy): + # Delegate to TensorRT-RTX wrapper if TensorRT-RTX was requested + is_trt_requested = any( + (isinstance(p, tuple) and "NvTensorRTRTX" in p[0]) or (isinstance(p, str) and "NvTensorRTRTX" in p) + for p in providers + ) if isinstance(providers, (list, tuple)) else False + + if is_trt_requested: + from engine.tensorrt_rtx import create_trt_rtx_session + dev_id = 0 + cuda_options = None + if isinstance(providers, (list, tuple)): + for p in providers: + if isinstance(p, tuple): + if p[0] == 'CUDAExecutionProvider' and isinstance(p[1], dict): + cuda_options = p[1] + if isinstance(p[1], dict) and "device_id" in p[1]: + try: + dev_id = int(p[1]["device_id"]) + except Exception: + dev_id = 0 + + model_name = os.path.basename(model_path) + if precision_policy.is_fp16: + # Respect FP16 preset on TensorRT-RTX path + import onnx + from onnxconverter_common import float16 + + print(f"[PRECISION] Loading {model_name} into CPU RAM for in-memory FP16 conversion...") + try: + model_fp32 = onnx.load(model_path, load_external_data=True) + try: + model_fp16 = float16.convert_float_to_float16(model_fp32, keep_io_types=False) + finally: + del model_fp32 + gc.collect() + + try: + serialized_fp16 = model_fp16.SerializeToString() + finally: + del model_fp16 + gc.collect() + + print(f"[PRECISION] Creating TensorRT-RTX FP16 session from RAM ({len(serialized_fp16) / (1024 ** 2):.2f} MiB)...") + try: + return create_trt_rtx_session( + serialized_fp16, + device_id=dev_id, + cuda_options=cuda_options, + model_name=model_name, + ) + finally: + del serialized_fp16 + gc.collect() + except Exception as error: + print( + f"[PRECISION] In-memory FP16 conversion failed ({type(error).__name__}: {error}). " + "Falling back to the original FP32 ONNX model without writing a temporary file." + ) + return create_trt_rtx_session( + model_path, + device_id=dev_id, + cuda_options=cuda_options, + model_name=model_name, + ) + else: + return create_trt_rtx_session( + model_path, + device_id=dev_id, + cuda_options=cuda_options, + model_name=model_name, + ) + if not precision_policy.is_fp16: return ort.InferenceSession(model_path, providers=providers) @@ -421,6 +493,7 @@ def create_onnx_session(model_path, providers, precision_policy): ) return ort.InferenceSession(model_path, providers=providers) + if TYPE_CHECKING: from UVR import ModelData @@ -585,7 +658,11 @@ class SeperateAttributes: 'cudnn_conv_algo_search': 'HEURISTIC', 'do_copy_in_default_stream': 'True' } - self.run_type = [('CUDAExecutionProvider', cuda_options), 'CPUExecutionProvider'] + chosen_ep = getattr(model_data, 'execution_provider', 'CUDAExecutionProvider') + if chosen_ep == 'NvTensorRTRTXExecutionProvider': + self.run_type = ['NvTensorRTRTXExecutionProvider', ('CUDAExecutionProvider', cuda_options), 'CPUExecutionProvider'] + else: + self.run_type = [('CUDAExecutionProvider', cuda_options), 'CPUExecutionProvider'] if model_data.process_method == MDX_ARCH_TYPE: self.is_mdx_ckpt = model_data.is_mdx_ckpt diff --git a/tests/test_tensorrt_rtx.py b/tests/test_tensorrt_rtx.py new file mode 100644 index 0000000..d7fb10e --- /dev/null +++ b/tests/test_tensorrt_rtx.py @@ -0,0 +1,159 @@ +"""Unit tests for TensorRT-RTX wrapper, provider resolution, precision presets, and install record.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import onnxruntime as ort +import torch + +from engine.tensorrt_rtx import ( + PROVIDER_CPU, + PROVIDER_CUDA, + PROVIDER_TRT_RTX, + build_trt_rtx_options, + create_trt_rtx_session, + get_supported_execution_providers, + is_ort_gpu_available, + is_trt_rtx_available, + is_trt_rtx_installed, +) +from install.install_record import read_install_record, write_install_record +from separate import SmartPrecisionPolicy, create_onnx_session + + +class TensorRTRTXEngineTests(unittest.TestCase): + """Test TensorRT-RTX options, provider discovery, honest separation, and session fallback.""" + + def test_build_trt_rtx_options_keys(self): + with tempfile.TemporaryDirectory() as tmp_dir: + options = build_trt_rtx_options(device_id=0, cache_dir=tmp_dir, enable_cuda_graph=True) + self.assertEqual(options["device_id"], "0") + self.assertEqual(options["nv_runtime_cache_path"], str(Path(tmp_dir))) + self.assertEqual(options["enable_cuda_graph"], "1") + # Must NOT contain standard TensorRT keys + for key in options: + self.assertFalse(key.startswith("trt_"), f"Unexpected standard TRT option: {key}") + + def test_standard_tensorrt_is_not_treated_as_trt_rtx(self): + # Even if standard TensorrtExecutionProvider is present in ORT, it must NOT be offered as TensorRT-RTX + with mock.patch("onnxruntime.get_available_providers", return_value=["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]): + with mock.patch("onnxruntime.get_all_providers", return_value=["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]): + # Ensure plugin is not detected + with mock.patch("onnxruntime.get_ep_devices", create=True, return_value=[]): + self.assertFalse(is_trt_rtx_installed()) + providers = get_supported_execution_providers(has_gpu_selected=True, device_id=0) + self.assertNotIn(PROVIDER_TRT_RTX, providers) + self.assertIn(PROVIDER_CUDA, providers) + + def test_provider_resolution_rules(self): + # Rule 3: GPU not selected -> returns CPUExecutionProvider + cpu_providers = get_supported_execution_providers(has_gpu_selected=False) + self.assertEqual(cpu_providers, [PROVIDER_CPU]) + + # Rule 2: GPU selected -> returns GPU providers with CUDA fallback + gpu_providers = get_supported_execution_providers(has_gpu_selected=True, device_id=0) + self.assertIsInstance(gpu_providers, list) + self.assertGreater(len(gpu_providers), 0) + self.assertTrue(PROVIDER_CUDA in gpu_providers or PROVIDER_CPU in gpu_providers) + + def test_saved_provider_preserved_while_locked(self): + # Simulate GUI state behavior + is_gpu_selected = False # DEFAULT GPU device + stored_ep = PROVIDER_TRT_RTX + + # While locked, stored provider must NOT be overwritten + if is_gpu_selected: + stored_ep = "OVERWRITTEN" + self.assertEqual(stored_ep, PROVIDER_TRT_RTX) + + # When GPU is selected, validate against detected providers + is_gpu_selected = True + supported = [PROVIDER_CUDA, PROVIDER_CPU] + if is_gpu_selected and stored_ep not in supported: + stored_ep = PROVIDER_CUDA + self.assertEqual(stored_ep, PROVIDER_CUDA) + + @mock.patch("onnxruntime.InferenceSession") + def test_session_fallback_chain(self, mock_session): + sentinel_session = mock.MagicMock() + mock_session.return_value = sentinel_session + + # Case 1: Plugin is present -> uses sess_options or providers + session = create_trt_rtx_session("test_model.onnx", device_id=0) + self.assertIsNotNone(session) + mock_session.assert_called() + + # Case 2: When TRT-RTX plugin session creation fails, it falls back to CUDA + mock_session.reset_mock() + mock_session.side_effect = [ + RuntimeError("Plugin creation failure"), + sentinel_session, + ] + session_fb = create_trt_rtx_session("test_model.onnx", device_id=0) + self.assertIsNotNone(session_fb) + self.assertGreaterEqual(mock_session.call_count, 2) + second_call_kwargs = mock_session.call_args_list[1][1] + providers_used = second_call_kwargs.get("providers", []) + self.assertTrue( + any("CUDAExecutionProvider" in (p[0] if isinstance(p, tuple) else p) for p in providers_used) + or any("CPUExecutionProvider" in (p[0] if isinstance(p, tuple) else p) for p in providers_used) + ) + + def test_precision_policy_diagnose_consistency(self): + from gui_data.constants import MODEL_PRECISION_FP16, MODEL_PRECISION_FP32 + policy_fp16 = SmartPrecisionPolicy(MODEL_PRECISION_FP16) + self.assertTrue(policy_fp16.is_fp16) + self.assertEqual(policy_fp16.effective_mode_for("ONNX"), "FP16") + + policy_fp32 = SmartPrecisionPolicy(MODEL_PRECISION_FP32) + self.assertTrue(policy_fp32.is_fp32) + self.assertEqual(policy_fp32.effective_mode_for("ONNX"), "FP32") + + +class InstallRecordTests(unittest.TestCase): + """Test persistent install record write, read, stack notes, and recovery.""" + + def test_write_and_read_install_record_with_stack_info(self): + with tempfile.TemporaryDirectory() as tmp_dir: + record_path = Path(tmp_dir) / "test_install_record.json" + + # Write record for Option B + write_install_record( + option="B", + install_mode="TENSORRT_RTX", + reuse_venv="Y", + has_uv=True, + has_ffmpeg=True, + has_rubberband=True, + has_vc_redist=True, + record_path=record_path, + ) + + self.assertTrue(record_path.is_file()) + + # Read record back + data = read_install_record(record_path) + self.assertIsNotNone(data) + self.assertEqual(data["selected_option"], "B") + self.assertEqual(data["install_mode"], "TENSORRT_RTX") + self.assertEqual(data["stack_profile"], "tensorrt_rtx_ort124") + self.assertIn("ort_stack_note", data) + self.assertIn("installed_packages", data) + + def test_read_missing_or_corrupt_record_returns_none(self): + with tempfile.TemporaryDirectory() as tmp_dir: + missing_path = Path(tmp_dir) / "non_existent.json" + self.assertIsNone(read_install_record(missing_path)) + + corrupt_path = Path(tmp_dir) / "corrupt.json" + corrupt_path.write_text("{bad json: 123", encoding="utf-8") + self.assertIsNone(read_install_record(corrupt_path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_uvr_cli.py b/tests/test_uvr_cli.py index d4bef81..238bfa7 100644 --- a/tests/test_uvr_cli.py +++ b/tests/test_uvr_cli.py @@ -16,6 +16,7 @@ from uvr_cli.model_resolver import ( HeadlessModelData, build_headless_model_data, list_available_models, + parse_and_validate_device, resolve_model_file, ) from uvr_cli.progress import InferenceProgressBar @@ -70,6 +71,31 @@ class BenchmarkCLITests(unittest.TestCase): self.assertEqual(model_data_custom.batch_size, 2) self.assertEqual(model_data_custom.device_name, "cuda:0") + def test_device_selection_cuda_and_cpu(self): + is_gpu, idx, ep = parse_and_validate_device("cuda:0") + self.assertTrue(is_gpu) + self.assertEqual(idx, "0") + self.assertEqual(ep, "CUDAExecutionProvider") + + is_gpu, idx, ep = parse_and_validate_device("cpu") + self.assertFalse(is_gpu) + self.assertEqual(idx, "0") + self.assertEqual(ep, "CPUExecutionProvider") + + def test_device_selection_trt_rtx(self): + is_gpu, idx, ep = parse_and_validate_device("trt_rtx:0") + self.assertTrue(is_gpu) + self.assertEqual(idx, "0") + self.assertEqual(ep, "NvTensorRTRTXExecutionProvider") + + def test_device_selection_invalid_rejected(self): + with self.assertRaises(ValueError): + parse_and_validate_device("vulkan:0") + with self.assertRaises(ValueError): + parse_and_validate_device("cuda:invalid") + with self.assertRaises(ValueError): + parse_and_validate_device("invalid") + def test_telemetry_sampler_lifecycle(self): sampler = TelemetrySampler(device_index=0, interval_sec=0.05) sampler.start() diff --git a/uvr_cli/README.md b/uvr_cli/README.md index 8a0f317..067a01d 100644 --- a/uvr_cli/README.md +++ b/uvr_cli/README.md @@ -49,7 +49,7 @@ venv\Scripts\python.exe -m uvr_cli run ` --audio "path/to/song.wav" ` --model "UVR-MDX-NET-Inst_HQ_4.onnx" ` --precision fp16 ` - --device cuda:0 ` + --device trt_rtx:0 ` --output-dir "separated_outputs" ``` @@ -60,7 +60,7 @@ venv\Scripts\python.exe -m uvr_cli run ` | `--audio` | `-a` | `Path` | *Required* | Path to the input audio file to separate (WAV, FLAC, MP3, etc.). | | `--model` | `-m` | `String` | *Required* | Model filename (e.g. `UVR-MDX-NET-Inst_HQ_4.onnx`), friendly name, or absolute path. | | `--precision` | `-p` | `Choice` | `fp16` | Inference precision policy: `fp16` (default, in-memory downcast), `fp32`, or `bf16` (experimental). | -| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda:0`, `cuda:1`, or `cpu`). | +| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda[:N]`, `trt_rtx[:N]`, or `cpu`). | | `--output-dir` | `-o` | `Path` | `separated_outputs` | Output directory where separated stem files will be saved. | | `--segment-size` | | `Integer` | *Model Default* | MDX chunk size in samples (e.g. `256`, `3000`, `4000`, `6000`). | | `--overlap` | | `Float` | *Model Default* | Overlap ratio between adjacent chunks (e.g. `0.25`, `0.50`, `0.75`). | @@ -103,7 +103,7 @@ venv\Scripts\python.exe -m uvr_cli bench ` | `--audio` | `-a` | `Path` | *Required* | Path to input audio file for benchmarking (uncompressed WAV recommended). | | `--model` | `-m` | `String` | *Required* | Model filename, friendly name, or absolute path to benchmark. | | `--precision` | `-p` | `Choice` | `fp16` | Inference precision policy: `fp16`, `fp32`, or `bf16` (experimental). | -| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda:0`, `cpu`). | +| `--device` | `-d` | `String` | `cuda:0` | Target execution device (`cuda[:N]`, `trt_rtx[:N]`, or `cpu`). | | `--rounds` | `-r` | `Integer` | `3` | Number of measured benchmark rounds to run after warmup. | | `--warmup` | `-w` | `Integer` | `1` | Number of untimed warmup rounds to prime model sessions, ONNX memory, and CUDA allocations. | | `--export-dir` | `-e` | `Path` | `benchmark_results` | Directory where JSON telemetry reports and `summary.csv` will be saved. | diff --git a/uvr_cli/cli.py b/uvr_cli/cli.py index 28287e6..11bd390 100644 --- a/uvr_cli/cli.py +++ b/uvr_cli/cli.py @@ -106,15 +106,19 @@ def handle_run(args: argparse.Namespace) -> int: precision = normalize_cli_precision(args.precision) print(f"Resolving model: {model_path.name} ({arch})") - model_data = build_headless_model_data( - model_path=model_path, - architecture=arch, - precision=precision, - device=args.device, - segment_size=args.segment_size, - overlap=args.overlap, - batch_size=args.batch_size, - ) + try: + model_data = build_headless_model_data( + model_path=model_path, + architecture=arch, + precision=precision, + device=args.device, + segment_size=args.segment_size, + overlap=args.overlap, + batch_size=args.batch_size, + ) + except ValueError as val_err: + print(f"[ERROR] {val_err}") + return 1 audio_path = Path(args.audio).resolve() output_dir = Path(args.output_dir).resolve() @@ -192,15 +196,19 @@ def handle_bench(args: argparse.Namespace) -> int: model_path, arch = resolve_model_file(args.model) precision = normalize_cli_precision(args.precision) - model_data = build_headless_model_data( - model_path=model_path, - architecture=arch, - precision=precision, - device=args.device, - segment_size=args.segment_size, - overlap=args.overlap, - batch_size=args.batch_size, - ) + try: + model_data = build_headless_model_data( + model_path=model_path, + architecture=arch, + precision=precision, + device=args.device, + segment_size=args.segment_size, + overlap=args.overlap, + batch_size=args.batch_size, + ) + except ValueError as val_err: + print(f"[ERROR] {val_err}") + return 1 device_idx = int(args.device.split(":")[-1]) if ":" in args.device else 0 @@ -245,12 +253,16 @@ def handle_smoke_test(args: argparse.Namespace) -> int: print(f"Selected smoke test model: {model_path.name} ({arch})") - model_data = build_headless_model_data( - model_path=model_path, - architecture=arch, - precision=precision, - device=args.device, - ) + try: + model_data = build_headless_model_data( + model_path=model_path, + architecture=arch, + precision=precision, + device=args.device, + ) + except ValueError as val_err: + print(f"[ERROR] {val_err}") + return 1 device_idx = int(args.device.split(":")[-1]) if ":" in args.device else 0 @@ -303,7 +315,7 @@ def build_parser() -> argparse.ArgumentParser: default="fp16", help="Inference precision: fp32, fp16, bf16 (default: fp16)", ) - run_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)") + run_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda[:N], trt_rtx[:N], cpu (default: cuda:0)") run_parser.add_argument("--output-dir", "-o", default="separated_outputs", help="Output directory for stems") run_parser.add_argument("--segment-size", type=int, default=None, help="Segment size (MDX chunk size)") run_parser.add_argument("--overlap", type=float, default=None, help="Overlap ratio (e.g. 0.25)") @@ -325,7 +337,7 @@ def build_parser() -> argparse.ArgumentParser: default="fp16", help="Inference precision: fp32, fp16, bf16 (default: fp16)", ) - bench_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)") + bench_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda[:N], trt_rtx[:N], cpu (default: cuda:0)") bench_parser.add_argument("--rounds", "-r", type=int, default=3, help="Number of measurement rounds (default: 3)") bench_parser.add_argument("--warmup", "-w", type=int, default=1, help="Number of warmup rounds (default: 1)") bench_parser.add_argument( @@ -342,7 +354,7 @@ def build_parser() -> argparse.ArgumentParser: smoke_parser = subparsers.add_parser("smoke-test", help="Run automated smoke test using synthetic audio") smoke_parser.add_argument("--model", "-m", default=None, help="Optional model name (defaults to first available)") smoke_parser.add_argument("--precision", "-p", default="fp16", help="Inference precision (default: fp16)") - smoke_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda:0, cpu (default: cuda:0)") + smoke_parser.add_argument("--device", "-d", default="cuda:0", help="Target device: cuda[:N], trt_rtx[:N], cpu (default: cuda:0)") smoke_parser.add_argument( "--export-dir", "-e", diff --git a/uvr_cli/model_resolver.py b/uvr_cli/model_resolver.py index 1e709d0..08e0362 100644 --- a/uvr_cli/model_resolver.py +++ b/uvr_cli/model_resolver.py @@ -100,6 +100,7 @@ class HeadlessModelData: self.model_hash: Optional[str] = None self.model_status: bool = True self.device_set: str = "0" + self.execution_provider: str = "CUDAExecutionProvider" self.is_gpu_conversion: int = 0 self.is_use_opencl: bool = False self.model_precision: str = MODEL_PRECISION_DEFAULT @@ -276,6 +277,43 @@ def resolve_model_file(model_identifier: str) -> Tuple[Path, str]: ) +def parse_and_validate_device(device_str: str) -> Tuple[bool, str, str]: + """Parse and validate CLI device string. + + Returns: + (is_gpu, device_index, execution_provider) + + Raises: + ValueError if the device format or name is invalid. + """ + dev = device_str.strip().lower() + if dev == "cpu": + return False, "0", "CPUExecutionProvider" + + parts = dev.split(":") + prefix = parts[0] + index = parts[1] if len(parts) > 1 else "0" + + if prefix not in ("cuda", "trt_rtx", "directml"): + raise ValueError( + f"Invalid device '{device_str}'. Valid choices are: 'cuda[:N]', 'trt_rtx[:N]', or 'cpu' (e.g. 'cuda:0', 'trt_rtx:0', 'cpu')." + ) + + if not index.isdigit(): + raise ValueError( + f"Invalid device index in '{device_str}'. Device index must be an integer (e.g. 'cuda:0', 'trt_rtx:0')." + ) + + if prefix == "trt_rtx": + ep = "NvTensorRTRTXExecutionProvider" + elif prefix == "cuda": + ep = "CUDAExecutionProvider" + else: + ep = "CUDAExecutionProvider" + + return True, index, ep + + def build_headless_model_data( model_path: Path, architecture: str, @@ -291,8 +329,26 @@ def build_headless_model_data( model_basename = model_path.stem model_hash = compute_uvr_hash(model_path) - is_gpu = "cuda" in device.lower() or "directml" in device.lower() - device_set = device.split(":")[-1] if ":" in device else "0" + is_gpu, device_set, chosen_ep = parse_and_validate_device(device) + + # Provider availability and fallback + if is_gpu: + import torch + if not torch.cuda.is_available(): + print(f"[WARN] GPU acceleration requested ('{device}') but CUDA is not available. Falling back to CPU.") + is_gpu = False + device_set = "0" + chosen_ep = "CPUExecutionProvider" + elif chosen_ep == "NvTensorRTRTXExecutionProvider": + try: + from engine.tensorrt_rtx import is_trt_rtx_available + dev_idx = int(device_set) + if not is_trt_rtx_available(dev_idx): + print(f"[BACKEND] TensorRT-RTX EP is not available on device {dev_idx}. Falling back to CUDAExecutionProvider.") + chosen_ep = "CUDAExecutionProvider" + except Exception: + print("[BACKEND] TensorRT-RTX EP check failed. Falling back to CUDAExecutionProvider.") + chosen_ep = "CUDAExecutionProvider" clean_prec = precision.strip().lower() if clean_prec in ("fp32", "32"): @@ -315,6 +371,8 @@ def build_headless_model_data( device_name=device, is_gpu_conversion=0 if is_gpu else -1, ) + data.execution_provider = chosen_ep + if segment_size is not None: data.segment_size = segment_size