From 8a027666efd94628f98ea0c09599b3e0bde3914f Mon Sep 17 00:00:00 2001 From: Vadim Date: Sat, 20 May 2023 23:37:21 +0200 Subject: [PATCH 1/8] - Added GPU support for demucs and Whisper --- README.md | 16 +++++++++++++++- UltraSinger.py | 2 +- moduls/Audio/separation.py | 5 +++-- moduls/DeviceDetection/device_detection.py | 19 +++++++------------ requirements.txt | 7 +++++-- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4759e73..290bede 100644 --- a/README.md +++ b/README.md @@ -169,4 +169,18 @@ After that you need to install the requirements from `requirements.txt`. Also install ffmpeg separately. -Then just use the `How to use` commands, described above. \ No newline at end of file +Then just use the `How to use` commands, described above. + +### Use GPU + +You need a cuda device. +For best use you need to install the cuda driver for your gpu see [here](https://developer.nvidia.com/cuda-downloads). +Install torch with cuda separately in your `venv`. See this [link](https://pytorch.org/get-started/locally/). + +Command: +``` +pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 +``` + +If program crash because of low VRAM than use a smaller model. +Whisper needs more than 8GB VRAM in the `large` model! \ No newline at end of file diff --git a/UltraSinger.py b/UltraSinger.py index b89c7e7..161502b 100644 --- a/UltraSinger.py +++ b/UltraSinger.py @@ -283,7 +283,7 @@ def transcribe_audio(transcribed_data): def separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_input_path): audio_separation_path = os.path.join(cache_path, "separated", "htdemucs", basename_without_ext) if settings.use_separated_vocal or settings.create_karaoke: - separate_audio(ultrastar_audio_input_path, cache_path) + separate_audio(ultrastar_audio_input_path, cache_path, settings.device) if settings.use_separated_vocal: vocals_path = os.path.join(audio_separation_path, "vocals.wav") convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path) diff --git a/moduls/Audio/separation.py b/moduls/Audio/separation.py index 20f4da3..1bc8b5c 100644 --- a/moduls/Audio/separation.py +++ b/moduls/Audio/separation.py @@ -8,7 +8,8 @@ def separate_audio(input_file_path, output_file, device="cpu"): # Model selection? # -n mdx_q # -n htdemucs_ft - # todo: -d cpu otherwise it automatically uses gpu - subprocess.run(["demucs", "--two-stems=vocals", input_file_path]) + device = "-d cpu" if device == "cpu" else "cuda" + subprocess.run(["demucs", "-d", device, "--two-stems=vocals", input_file_path]) + separated_folder = path_join(current_executor_path(), "separated") move(separated_folder, output_file) diff --git a/moduls/DeviceDetection/device_detection.py b/moduls/DeviceDetection/device_detection.py index 120138d..bd0b3c2 100644 --- a/moduls/DeviceDetection/device_detection.py +++ b/moduls/DeviceDetection/device_detection.py @@ -1,20 +1,15 @@ import tensorflow as tf from moduls.Log import PRINT_ULTRASTAR, print_red_highlighted_text - +import torch def get_available_device(): - gpus = tf.config.list_physical_devices('GPU') + isCuda = torch.cuda.is_available() - if not gpus: - print("{} There are no GPUs available. Using {}.".format(PRINT_ULTRASTAR, print_red_highlighted_text("cpu"))) + if not isCuda: + print("{} There are no {} devices available. -> Using {}.".format(PRINT_ULTRASTAR, print_red_highlighted_text("cuda"), print_red_highlighted_text("cpu"))) return "cpu" + else: + print("{} Using {} GPU.".format(PRINT_ULTRASTAR, print_red_highlighted_text("cuda"))) - print("{} Found available GPUs:".format(PRINT_ULTRASTAR)) - for gpu in gpus: - print("Name:", gpu.name, " Type:", gpu.device_type) - - # Todo: Finish this - print("{} GPU usage are currently in development. Using {}".format(PRINT_ULTRASTAR, - print_red_highlighted_text("cpu"))) - return "cpu" + return "cuda" diff --git a/requirements.txt b/requirements.txt index 1a02274..f78f00e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,11 +5,14 @@ pydub==0.25.1 Levenshtein==0.20.9 scipy==1.10.0 vosk==0.3.45 -tensorflow~=2.11.0 +tensorflow==2.11.0 +tensorflow-directml-plugin~=0.4.0.dev230202 +#cudatoolkit==11.8.0 matplotlib~=3.7.1 yt_dlp~=2023.3.4 numpy~=1.23.5 git+https://github.com/linto-ai/whisper-timestamped PyHyphen~=4.0.3 demucs~=4.0.0 -tqdm~=4.65.0 \ No newline at end of file +tqdm~=4.65.0 +torchaudio~=2.0.2 \ No newline at end of file From ccf17e5d264d127d549c9fedc207db778d2b2e3e Mon Sep 17 00:00:00 2001 From: Vadim Date: Sun, 21 May 2023 00:13:43 +0200 Subject: [PATCH 2/8] - Update README.md --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 290bede..8823a0c 100644 --- a/README.md +++ b/README.md @@ -173,14 +173,20 @@ Then just use the `How to use` commands, described above. ### Use GPU -You need a cuda device. +With an GPU you can speed up the process and also the quality of the transcription and pitching is better. + +You need a cuda device for this to work. For best use you need to install the cuda driver for your gpu see [here](https://developer.nvidia.com/cuda-downloads). Install torch with cuda separately in your `venv`. See this [link](https://pytorch.org/get-started/locally/). -Command: +Command for `pip`: ``` pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 ``` -If program crash because of low VRAM than use a smaller model. +When you want to use `conda` instead you need a different installation command. See this [link](https://pytorch.org/get-started/locally/). + +#### Info + +If somthing crash because of low VRAM than use a smaller model. Whisper needs more than 8GB VRAM in the `large` model! \ No newline at end of file From b8ca18dee7b1c92eb87462ede287e1d323a6d6ec Mon Sep 17 00:00:00 2001 From: Vadim Date: Sun, 21 May 2023 21:57:13 +0200 Subject: [PATCH 3/8] - Changed/Updated requirements --- moduls/DeviceDetection/device_detection.py | 2 -- requirements.txt | 13 ++++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/moduls/DeviceDetection/device_detection.py b/moduls/DeviceDetection/device_detection.py index bd0b3c2..6475505 100644 --- a/moduls/DeviceDetection/device_detection.py +++ b/moduls/DeviceDetection/device_detection.py @@ -1,5 +1,3 @@ -import tensorflow as tf - from moduls.Log import PRINT_ULTRASTAR, print_red_highlighted_text import torch diff --git a/requirements.txt b/requirements.txt index f78f00e..eb80b88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ crepe==0.0.13 librosa==0.9.2 -pretty_midi==0.2.9 +pretty_midi~=0.2.10 pydub==0.25.1 -Levenshtein==0.20.9 -scipy==1.10.0 +Levenshtein~=0.21.0 +scipy~=1.10.1 vosk==0.3.45 -tensorflow==2.11.0 -tensorflow-directml-plugin~=0.4.0.dev230202 +tensorflow~=2.12.0 +#tensorflow-directml-plugin~=0.4.0.dev230202 #cudatoolkit==11.8.0 matplotlib~=3.7.1 yt_dlp~=2023.3.4 @@ -14,5 +14,4 @@ numpy~=1.23.5 git+https://github.com/linto-ai/whisper-timestamped PyHyphen~=4.0.3 demucs~=4.0.0 -tqdm~=4.65.0 -torchaudio~=2.0.2 \ No newline at end of file +tqdm~=4.65.0 \ No newline at end of file From 13477dde06214a68f2e3c3f5e71cd2298af4a458 Mon Sep 17 00:00:00 2001 From: Vadim Date: Sun, 21 May 2023 22:25:47 +0200 Subject: [PATCH 4/8] - fix cpu argument for separation --- moduls/Audio/separation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moduls/Audio/separation.py b/moduls/Audio/separation.py index 1bc8b5c..6ef3d38 100644 --- a/moduls/Audio/separation.py +++ b/moduls/Audio/separation.py @@ -8,7 +8,7 @@ def separate_audio(input_file_path, output_file, device="cpu"): # Model selection? # -n mdx_q # -n htdemucs_ft - device = "-d cpu" if device == "cpu" else "cuda" + device = "cpu" if device != "cuda" else "cuda" subprocess.run(["demucs", "-d", device, "--two-stems=vocals", input_file_path]) separated_folder = path_join(current_executor_path(), "separated") From b3c9bb235e7a51aa18cea47535cbc1263c39b3c1 Mon Sep 17 00:00:00 2001 From: Vadim Rangnau <36919102+rakuri255@users.noreply.github.com> Date: Mon, 22 May 2023 10:51:35 +0200 Subject: [PATCH 5/8] Update GPU usage README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 12d8806..fbd3963 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Install torch with cuda separately in your `venv`. See this [link](https://pytor Command for `pip`: ``` -pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 +pip3 install torch~=2.0.1+cu117 torchvision~=0.15.2+cu117 torchaudio~=2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117 ``` When you want to use `conda` instead you need a different installation command. See this [link](https://pytorch.org/get-started/locally/). @@ -189,4 +189,4 @@ When you want to use `conda` instead you need a different installation command. #### Info If somthing crash because of low VRAM than use a smaller model. -Whisper needs more than 8GB VRAM in the `large` model! \ No newline at end of file +Whisper needs more than 8GB VRAM in the `large` model! From 7e978a281470d95dd0d441d47100fd435139e930 Mon Sep 17 00:00:00 2001 From: Vadim Date: Mon, 22 May 2023 21:40:06 +0200 Subject: [PATCH 6/8] - added option to force cpu for separation and whisper --- README.md | 4 ++++ Settings.py | 6 ++++-- UltraSinger.py | 12 +++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fbd3963..0d58cc8 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ _Not all options working now!_ --disable_separation True|False >> ((default) is False) --disable_karaoke True|False >> ((default) is False) --create_audio_chunks True|False >> ((default) is False) + --force_whisper_cpu True|False >> ((default) is False) + --force_separation_cpu True|False >> ((default) is False) [pitcher] # Default is crepe full @@ -190,3 +192,5 @@ When you want to use `conda` instead you need a different installation command. If somthing crash because of low VRAM than use a smaller model. Whisper needs more than 8GB VRAM in the `large` model! + +But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`. diff --git a/Settings.py b/Settings.py index f455436..e60f15a 100644 --- a/Settings.py +++ b/Settings.py @@ -15,7 +15,7 @@ class Settings: # Whisper transcriber = 'whisper' # whisper|vosk - whisper_model = 'large' # Multilingual model tiny|base|small|medium|large + whisper_model = 'large' # Multilingual model tiny|base|small|medium|large # English-only model tiny.en|base.en|small.en|medium.en # Vosk @@ -26,5 +26,7 @@ class Settings: crepe_step_size = 10 # Device - device = 'cpu' # cpu|cuda|gpu + device = 'cpu' # cpu|cuda + force_whisper_cpu = False + force_separation_cpu = False diff --git a/UltraSinger.py b/UltraSinger.py index a85e685..b345af9 100644 --- a/UltraSinger.py +++ b/UltraSinger.py @@ -280,8 +280,9 @@ def run(): def transcribe_audio(transcribed_data): if settings.transcriber == "whisper": + device = "cpu" if settings.force_whisper_cpu else settings.device transcribed_data, language = transcribe_with_whisper(settings.mono_audio_path, settings.whisper_model, - settings.device) + device) else: # vosk transcribed_data = transcribe_with_vosk(settings.mono_audio_path, settings.vosk_model_path) # todo: make language selectable @@ -291,8 +292,9 @@ def transcribe_audio(transcribed_data): def separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_input_path): audio_separation_path = os.path.join(cache_path, "separated", "htdemucs", basename_without_ext) + device = "cpu" if settings.force_separation_cpu else settings.device if settings.use_separated_vocal or settings.create_karaoke: - separate_audio(ultrastar_audio_input_path, cache_path, settings.device) + separate_audio(ultrastar_audio_input_path, cache_path, device) if settings.use_separated_vocal: vocals_path = os.path.join(audio_separation_path, "vocals.wav") convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path) @@ -433,7 +435,7 @@ def main(argv): def init_settings(argv): short = "hi:o:amv:" long = ["ifile=", "ofile=", "crepe=", "vosk=", "whisper=", "hyphenation=", "disable_separation=", - "disable_karaoke=", "create_audio_chunks="] + "disable_karaoke=", "create_audio_chunks=", "force_whisper_cpu=", "force_separation_cpu="] opts, args = getopt.getopt(argv, short, long) if len(opts) == 0: print_help() @@ -462,6 +464,10 @@ def init_settings(argv): settings.create_karaoke = not arg elif opt in ("--create_audio_chunks"): settings.create_audio_chunks = arg + elif opt in ("--force_whisper_cpu"): + settings.force_whisper_cpu = arg + elif opt in ("--force_separation_cpu"): + settings.force_separation_cpu = arg if settings.output_file_path == '': if settings.input_file_path.startswith('https:'): dirname = os.getcwd() From dcd18cab7919a6ed5e7823f8c8ee44bba15a8bdd Mon Sep 17 00:00:00 2001 From: Vadim Date: Mon, 29 May 2023 13:55:56 +0200 Subject: [PATCH 7/8] - Added f strings --- moduls/DeviceDetection/device_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moduls/DeviceDetection/device_detection.py b/moduls/DeviceDetection/device_detection.py index 6475505..788ce0f 100644 --- a/moduls/DeviceDetection/device_detection.py +++ b/moduls/DeviceDetection/device_detection.py @@ -5,9 +5,9 @@ def get_available_device(): isCuda = torch.cuda.is_available() if not isCuda: - print("{} There are no {} devices available. -> Using {}.".format(PRINT_ULTRASTAR, print_red_highlighted_text("cuda"), print_red_highlighted_text("cpu"))) + print(f"{PRINT_ULTRASTAR} There are no {print_red_highlighted_text('cuda')} devices available. -> Using {print_red_highlighted_text('cpu')}.") return "cpu" else: - print("{} Using {} GPU.".format(PRINT_ULTRASTAR, print_red_highlighted_text("cuda"))) + print(f"{PRINT_ULTRASTAR} Using {print_red_highlighted_text('cuda')} GPU.") return "cuda" From 2ba180140c26a250ec0f3c5394342fa4dab59daa Mon Sep 17 00:00:00 2001 From: Vadim Date: Thu, 1 Jun 2023 16:40:15 +0200 Subject: [PATCH 8/8] - Changed readme --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 1b9eb70..73f60fc 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ This will help me alot to keep this project alive and improve it. * Type `.venv\Scripts\activate` and press enter. * You should see now a `(.venv)` in front of your console line. * Install the requirements with `pip install -r requirements.txt`. +* Install gpu requirements `pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117` * Now you can use the UltraSinger source code with `py UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]`. See [How to use](#how-to-use) for more information. For more information about Python environments look [here](https://code.visualstudio.com/docs/python/environments#_global-virtual-or-conda-environments). @@ -51,6 +52,7 @@ As copy: py -m venv .venv .venv\Scripts\activate pip install -r requirements.txt +pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117 ``` @@ -214,3 +216,16 @@ If somthing crash because of low VRAM than use a smaller model. Whisper needs more than 8GB VRAM in the `large` model! But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`. + +### Use GPU + +With an GPU you can speed up the process and also the quality of the transcription and pitching is better. +You need a cuda device for this to work. +If you use MAC than sorry, there is no cuda device for MAC machinges. + +#### Info + +If somthing crash because of low VRAM than use a smaller model. +Whisper needs more than 8GB VRAM in the `large` model! + +But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`.