diff --git a/.gitignore b/.gitignore index 9c544e3..acdad70 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.env +.vscode/ # virtual environment venv .venv @@ -5,5 +7,9 @@ venv # python **__pycache__ +# test data +test_input +test_output + # default output folder -output +output \ No newline at end of file diff --git a/README.md b/README.md index 9f8f6b4..6cb9d76 100644 --- a/README.md +++ b/README.md @@ -112,15 +112,18 @@ _Not all options working now!_ [transcription] # Default is whisper - --whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2 - English-only model > tiny.en|base.en|small.en|medium.en - --align_model Use other languages model for Whisper provided from huggingface.co - - --vosk Needs model + --whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2 + English-only model > tiny.en|base.en|small.en|medium.en + --whisper_align_model Use other languages model for Whisper provided from huggingface.co + --language Override the language detected by whisper, does not affect transcription but steps after transcription + --whisper_batch_size Reduce if low on GPU mem >> ((default) is 16) + --whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu) + --vosk Needs model [pitcher] # Default is crepe - --crepe tiny|small|medium|large|full >> ((default) is full) + --crepe tiny|full >> ((default) is full) + --crepe_step_size unit is miliseconds >> ((default) is 10) [extra] --hyphenation True|False >> ((default) is True) @@ -128,6 +131,7 @@ _Not all options working now!_ --disable_karaoke True|False >> ((default) is False) --create_audio_chunks True|False >> ((default) is False) --plot True|False >> ((default) is False) + --force_cpu True|False >> ((default) is False) ``` For standard use, you only need to use [opt]. All other options are optional. @@ -205,7 +209,7 @@ starts at the place or is heard. ### Pitcher -Pitching is done with the `crepe` model. +Pitching is done with the `crepe` model. Also consider that a bigger model is more accurate, but also takes longer to pitch. For just testing you should use `tiny`. If you want solid accurate, then use the `full` model. @@ -252,9 +256,23 @@ pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu11 When you want to use `conda` instead you need a different installation command. See this [link](https://pytorch.org/get-started/locally/). +#### Considerations for Windows users + +The pitch tracker used by UltraSinger (crepe), uses TensorFlow as it's backend. +TensorFlow dropped GPU support for Windows for versions >2.10 as you can see in this [release note](https://github.com/tensorflow/tensorflow/releases/tag/v2.11.1) and their [installation instructions](https://www.tensorflow.org/install/pip#windows-native). + +For now UltraSinger runs the latest version available that still supports GPUs on windows. + +For running later versions of TensorFlow on windows while still taking advantage of GPU support the suggested solution is: + +* [install WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) +* within the Ubuntu WSL2 installation + * run `sudo apt update && sudo apt install nvidia-cuda-toolkit` + * follow the setup instructions for UltraSinger at the top of this document + #### Info -If somthing crash because of low VRAM than use a smaller model. +If something crashes because of low VRAM than use a smaller model. Whisper needs more than 8GB VRAM in the `large` model! -But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`. +You can also force cpu usage with the extra option `--force_cpu`. diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index e71128e..97475e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,5 @@ +[tool.isort] +profile = "black" [tool.pytest.ini_options] pythonpath = [ "src", diff --git a/pytest/modules/Audio/test_denoise.py b/pytest/modules/Audio/test_denoise.py new file mode 100644 index 0000000..f4eba4c --- /dev/null +++ b/pytest/modules/Audio/test_denoise.py @@ -0,0 +1,24 @@ +"""Tests for whisper.py""" + +import os +import unittest +from src.modules.Audio.denoise import ffmpeg_reduce_noise +import pytest + + +class DenoiseTest(unittest.TestCase): + @pytest.mark.skip(reason="Skipping this FUNCTION level test, can be used for manual tests") + def test_ffmpeg_reduce_noise(self): + # Arrange + test_dir = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.abspath(test_dir + "/../../..") + test_file_abs_path = os.path.abspath(root_dir + "/test_input/vocals.wav") + test_file_name = os.path.basename(test_file_abs_path) + test_output = test_dir + "/test_output" + + # Act + ffmpeg_reduce_noise(test_file_abs_path, test_output + "/output_" + test_file_name) + + +if __name__ == "__main__": + unittest.main() diff --git a/pytest/modules/Pitcher/test_pitcher.py b/pytest/modules/Pitcher/test_pitcher.py new file mode 100644 index 0000000..e623986 --- /dev/null +++ b/pytest/modules/Pitcher/test_pitcher.py @@ -0,0 +1,27 @@ +"""Tests for pitcher.py""" + +import os +import unittest +import src.modules.Pitcher.pitcher as test_subject +import pytest +from src.modules.plot import plot + + +class PitcherTest(unittest.TestCase): + @pytest.mark.skip(reason="Skipping this FUNCTION level test, can be used for manual tests") + def test_get_pitch_with_crepe_file(self): + # Arrange + test_dir = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.abspath(test_dir + "/../../..") + # test_file_abs_path = os.path.abspath(root_dir + "/test_input/audio_denoised.wav") + test_file_abs_path = os.path.abspath(root_dir + "/test_input/test_denoised.wav") + test_output = root_dir + "/test_output" + + # Act + pitched_data = test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', device="cuda") + # test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', 'cpu', batch_size=1024) + plot(pitched_data, test_output, title="pitching test") + print("done") + +if __name__ == "__main__": + unittest.main() diff --git a/requirements.txt b/requirements.txt index fde325c..40bace8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,25 +1,27 @@ -crepe==0.0.13 -librosa==0.9.2 -pretty_midi~=0.2.10 -pydub==0.25.1 -Levenshtein~=0.21.0 -scipy~=1.10.1 -vosk==0.3.44 -tensorflow~=2.12.0 -#tensorflow-directml-plugin~=0.4.0.dev230202 -#cudatoolkit==11.8.0 -matplotlib~=3.7.1 -yt_dlp~=2023.7.6 -numpy~=1.23.5 -git+https://github.com/m-bain/whisperx.git -PyHyphen~=4.0.3 +crepe~=0.0.13 demucs~=4.0.0 -tqdm~=4.65.0 +ffmpeg_python~=0.2.0 +git+https://github.com/m-bain/whisperx.git langcodes~=3.3.0 language-data~=1.1 -Pillow~=9.5.0 -isort==5.12 -black==23.3 -pylint==2.17 +librosa~=0.9.2 +matplotlib~=3.7.2 +musicbrainzngs~=0.7.1 +numpy~=1.23.5 +Pillow~=10.0.0 +pretty_midi~=0.2.10 +pydub~=0.25.1 +PyHyphen~=4.0.3 +python_Levenshtein~=0.21.1 +scipy~=1.11.2 +tensorflow<2.11 +tqdm~=4.65.2 +vosk~=0.3.44 +whisperx~=3.1.1 +yt_dlp~=2023.7.6 + +isort~=5.12 +black~=23.3 +pylint~=2.17 pytest~=7.3.1 -musicbrainzngs~=0.7.1 \ No newline at end of file +protobuf==3.20.* \ No newline at end of file diff --git a/src/Settings.py b/src/Settings.py index 93d5b25..3a11327 100644 --- a/src/Settings.py +++ b/src/Settings.py @@ -10,6 +10,8 @@ class Settings: output_file_path = "" mono_audio_path = "" + language = None + # Transcribe audio_chunk_folder_name = "audio-chunks" @@ -18,15 +20,17 @@ class Settings: whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2 # English-only model tiny.en|base.en|small.en|medium.en whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2" + whisper_batch_size = 16 # reduce if low on GPU mem + whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy) # Vosk vosk_model_path = "" # "models/vosk-model-small-en-us-0.15" # Pitch crepe_model_capacity = "full" # tiny|small|medium|large|full - crepe_step_size = 10 + crepe_step_size = 10 # in miliseconds # Device - device = 'cpu' # cpu|cuda - force_whisper_cpu = False - force_separation_cpu = False + pytorch_device = 'cpu' # cpu|cuda + tensorflow_device = 'cpu' # cpu|cuda + force_cpu = False diff --git a/src/UltraSinger.py b/src/UltraSinger.py index 4a21e7f..b1cb8a4 100644 --- a/src/UltraSinger.py +++ b/src/UltraSinger.py @@ -4,20 +4,19 @@ import copy import getopt import os import sys -from time import sleep import Levenshtein import librosa from tqdm import tqdm -from modules import os_helper +from modules import os_helper, timer from modules.Audio.denoise import ffmpeg_reduce_noise from modules.Audio.separation import separate_audio from modules.Audio.vocal_chunks import ( export_chunks_from_transcribed_data, export_chunks_from_ultrastar_data, ) -from modules.Audio.silence_processing import remove_silence_from_transcribtion_data +from modules.Audio.silence_processing import remove_silence_from_transcription_data from modules.csv_handler import export_transcribed_data_to_csv from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3 from modules.Audio.youtube import ( @@ -26,7 +25,7 @@ from modules.Audio.youtube import ( download_youtube_video, get_youtube_title, ) -from modules.DeviceDetection.device_detection import get_available_device +from modules.DeviceDetection.device_detection import check_gpu_support from modules.console_colors import ( ULTRASINGER_HEAD, blue_highlighted, @@ -41,7 +40,7 @@ from modules.Midi.midi_creator import ( most_frequent, ) from modules.Pitcher.pitcher import ( - get_frequency_with_high_confidence, + get_frequencies_with_high_confidence, get_pitch_with_crepe_file, ) from modules.Pitcher.pitched_data import PitchedData @@ -91,9 +90,12 @@ def pitch_each_chunk_with_crepe(directory: str) -> list[str]: filepath = os.path.join(directory, filename) # todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes pitched_data = get_pitch_with_crepe_file( - filepath, settings.crepe_step_size, settings.crepe_model_capacity + filepath, + settings.crepe_model_capacity, + settings.crepe_step_size, + settings.tensorflow_device, ) - conf_f = get_frequency_with_high_confidence( + conf_f = get_frequencies_with_high_confidence( pitched_data.frequencies, pitched_data.confidence ) @@ -230,7 +232,6 @@ def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData]) ) return None - sleep(0.1) hyphenator = create_hyphenator(lang_region) for i in tqdm(enumerate(transcribed_data)): pos = i[0] @@ -309,20 +310,21 @@ def run() -> None: # Audio transcription transcribed_data = None - language = None + language = settings.language if is_audio: - language, transcribed_data = transcribe_audio() + detected_language, transcribed_data = transcribe_audio() + if language is None: + language = detected_language + remove_unecessary_punctuations(transcribed_data) - transcribed_data = remove_silence_from_transcribtion_data( + transcribed_data = remove_silence_from_transcription_data( settings.mono_audio_path, transcribed_data ) if settings.hyphenation: hyphen_words = hyphenate_each_word(language, transcribed_data) if hyphen_words is not None: - transcribed_data = add_hyphen_to_data( - transcribed_data, hyphen_words - ) + transcribed_data = add_hyphen_to_data(transcribed_data, hyphen_words) # todo: do we need to correct words? # lyric = 'input/faber_lyric.txt' @@ -345,7 +347,7 @@ def run() -> None: # Create plot if settings.create_plot: - plot(pitched_data, transcribed_data, midi_notes, song_output) + plot(pitched_data, song_output, transcribed_data, midi_notes) # Write Ultrastar txt if is_audio: @@ -408,16 +410,21 @@ def get_unused_song_output_dir(path: str) -> str: def transcribe_audio() -> (str, list[TranscribedData]): """Transcribe audio with AI""" if settings.transcriber == "whisper": - device = "cpu" if settings.force_whisper_cpu else settings.device - transcribed_data, language = transcribe_with_whisper( - settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model) + transcribed_data, detected_language = transcribe_with_whisper( + settings.mono_audio_path, + settings.whisper_model, + settings.pytorch_device, + settings.whisper_align_model, + settings.whisper_batch_size, + settings.whisper_compute_type, + settings.language, + ) else: # vosk transcribed_data = transcribe_with_vosk( settings.mono_audio_path, settings.vosk_model_path ) - # todo: make language selectable - language = "en" - return language, transcribed_data + detected_language = "en" + return detected_language, transcribed_data def separate_vocal_from_audio( @@ -427,16 +434,16 @@ def separate_vocal_from_audio( audio_separation_path = os.path.join( cache_path, "separated", "htdemucs", basename_without_ext ) - device = "cpu" if settings.force_separation_cpu else settings.device + if settings.use_separated_vocal or settings.create_karaoke: - separate_audio(ultrastar_audio_input_path, cache_path, device) + separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device) + if settings.use_separated_vocal: - vocals_path = os.path.join(audio_separation_path, "vocals.wav") - convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path) + input_path = os.path.join(audio_separation_path, "vocals.wav") else: - convert_audio_to_mono_wav( - ultrastar_audio_input_path, settings.mono_audio_path - ) + input_path = ultrastar_audio_input_path + + convert_audio_to_mono_wav(input_path, settings.mono_audio_path) return audio_separation_path @@ -697,8 +704,9 @@ def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrast # midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name) pitched_data = get_pitch_with_crepe_file( settings.mono_audio_path, - settings.crepe_step_size, settings.crepe_model_capacity, + settings.crepe_step_size, + settings.tensorflow_device, ) if is_audio: start_times = [] @@ -777,15 +785,25 @@ def init_settings(argv: list[str]) -> None: elif opt in ("--whisper"): settings.transcriber = "whisper" settings.whisper_model = arg - elif opt in ("--align_model"): + elif opt in ("--whisper_align_model"): settings.whisper_align_model = arg + elif opt in ("--whisper_batch_size"): + settings.whisper_batch_size = int(arg) + elif opt in ("--whisper_compute_type"): + settings.whisper_compute_type = arg + elif opt in ("--language"): + settings.language = arg elif opt in ("--vosk"): settings.transcriber = "vosk" settings.vosk_model_path = arg elif opt in ("--crepe"): settings.crepe_model_capacity = arg + elif opt in ("--crepe_step_size"): + settings.crepe_step_size = int(arg) elif opt in ("--plot"): - settings.create_plot = arg + settings.create_plot = arg in ["True", "true"] + elif opt in ("--midi"): + settings.create_midi = arg in ["True", "true"] elif opt in ("--hyphenation"): settings.hyphenation = arg elif opt in ("--disable_separation"): @@ -794,17 +812,20 @@ def init_settings(argv: list[str]) -> None: settings.create_karaoke = not arg elif opt in ("--create_audio_chunks"): settings.create_audio_chunks = arg - elif opt in ("--force_whisper_cpu"): - settings.force_whisper_cpu = arg - elif opt in ("--force_separation_cpu"): - settings.force_separation_cpu = arg + elif opt in ("--force_cpu"): + settings.force_cpu = arg + if settings.force_cpu: + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + if settings.output_file_path == "": if settings.input_file_path.startswith("https:"): dirname = os.getcwd() else: dirname = os.path.dirname(settings.input_file_path) settings.output_file_path = os.path.join(dirname, "output") - settings.device = get_available_device() + + if not settings.force_cpu: + settings.tensorflow_device, settings.pytorch_device = check_gpu_support() def arg_options(): @@ -813,16 +834,20 @@ def arg_options(): "ifile=", "ofile=", "crepe=", + "crepe_step_size=", "vosk=", "whisper=", - "align_model=", + "whisper_align_model=", + "whisper_batch_size=", + "whisper_compute_type=", + "language=", "plot=", + "midi=", "hyphenation=", "disable_separation=", "disable_karaoke=", "create_audio_chunks=", - "force_whisper_cpu=", - "force_separation_cpu=" + "force_cpu=", ] return long, short diff --git a/src/modules/Audio/denoise.py b/src/modules/Audio/denoise.py index 5987739..ff32d02 100644 --- a/src/modules/Audio/denoise.py +++ b/src/modules/Audio/denoise.py @@ -22,9 +22,14 @@ def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None: print( f"{ULTRASINGER_HEAD} Reduce noise from vocal audio with {blue_highlighted('ffmpeg')}." ) - ( - ffmpeg.input(input_file_path) - .output(output_file, af="afftdn=nr=70:nf=-50:tn=1") - .overwrite_output() - .run() - ) + try: + ( + ffmpeg.input(input_file_path) + .output(output_file, af="afftdn=nr=70:nf=-50:tn=1") + .overwrite_output() + .run(capture_stdout=True, capture_stderr=True) + ) + except ffmpeg.Error as ffmpeg_exception: + print("ffmpeg stdout:", ffmpeg_exception.stdout.decode("utf8")) + print("ffmpeg stderr:", ffmpeg_exception.stderr.decode("utf8")) + raise ffmpeg_exception diff --git a/src/modules/Audio/separation.py b/src/modules/Audio/separation.py index d34241e..42885c4 100644 --- a/src/modules/Audio/separation.py +++ b/src/modules/Audio/separation.py @@ -19,7 +19,6 @@ def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None # Model selection? # -n mdx_q # -n htdemucs_ft - device = "cpu" if device != "cuda" else "cuda" subprocess.run( ["demucs", "-d", device, "--two-stems=vocals", input_file_path] ) diff --git a/src/modules/Audio/silence_processing.py b/src/modules/Audio/silence_processing.py index e139547..7c3a559 100644 --- a/src/modules/Audio/silence_processing.py +++ b/src/modules/Audio/silence_processing.py @@ -6,8 +6,8 @@ from modules.console_colors import ULTRASINGER_HEAD from modules.Speech_Recognition.TranscribedData import TranscribedData -def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]: - """Remove silence from given transcribtion data""" +def remove_silence_from_transcription_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]: + """Remove silence from given transcription data""" print( f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data" diff --git a/src/modules/DeviceDetection/device_detection.py b/src/modules/DeviceDetection/device_detection.py index 5912438..092f295 100644 --- a/src/modules/DeviceDetection/device_detection.py +++ b/src/modules/DeviceDetection/device_detection.py @@ -1,19 +1,37 @@ """Device detection module.""" import torch +import os +import tensorflow as tf -from modules.console_colors import ULTRASINGER_HEAD, red_highlighted +from modules.console_colors import ULTRASINGER_HEAD, red_highlighted, blue_highlighted +tensorflow_gpu_supported = False +pytorch_gpu_supported = False -def get_available_device() -> str: - """Get available worker device e.g cuda or cpu""" - is_cuda = torch.cuda.is_available() +def check_gpu_support() -> tuple[bool, bool]: + """Check worker device (e.g cuda or cpu) supported by tensorflow and pytorch""" - if not is_cuda: + print(f"{ULTRASINGER_HEAD} Checking GPU support for {blue_highlighted('tensorflow')} and {blue_highlighted('pytorch')}.") + + tensorflow_gpu_supported = False + pytorch_gpu_supported = False + + gpus = tf.config.list_physical_devices('GPU') + if gpus: + tensorflow_gpu_supported = True + print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - using {red_highlighted('cuda')} gpu.") + else: + print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}.") + if os.name == 'nt': + print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - versions above 2.10 dropped GPU support for Windows, refer to the readme for possible solutions.") + + pytorch_gpu_supported = torch.cuda.is_available() + if not pytorch_gpu_supported: print( - f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}." + f"{ULTRASINGER_HEAD} {blue_highlighted('pytorch')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}." ) - return "cpu" - print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.") + else: + print(f"{ULTRASINGER_HEAD} {blue_highlighted('pytorch')} - using {red_highlighted('cuda')} gpu.") - return "cuda" + return 'cuda' if tensorflow_gpu_supported else 'cpu', 'cuda' if pytorch_gpu_supported else 'cpu' diff --git a/src/modules/Midi/midi_creator.py b/src/modules/Midi/midi_creator.py index b91b279..b045841 100644 --- a/src/modules/Midi/midi_creator.py +++ b/src/modules/Midi/midi_creator.py @@ -7,7 +7,7 @@ import librosa import numpy as np import pretty_midi -from modules.Pitcher.pitcher import get_frequency_with_high_confidence +from modules.Pitcher.pitcher import get_frequencies_with_high_confidence from modules.Ultrastar.ultrastar_converter import ( get_end_time_from_ultrastar, get_start_time_from_ultrastar, @@ -115,7 +115,7 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch freqs = pitched_data.frequencies[start:end] confs = pitched_data.confidence[start:end] - conf_f = get_frequency_with_high_confidence(freqs, confs) + conf_f = get_frequencies_with_high_confidence(freqs, confs) notes = convert_frequencies_to_notes(conf_f) diff --git a/src/modules/Pitcher/pitched_data.py b/src/modules/Pitcher/pitched_data.py index 574192f..13d828c 100644 --- a/src/modules/Pitcher/pitched_data.py +++ b/src/modules/Pitcher/pitched_data.py @@ -1,9 +1,11 @@ """Pitched data""" +from dataclasses import dataclass +@dataclass class PitchedData: """Pitched data from crepe""" - times = list[float] - frequencies = list[float] - confidence = list[float] + times: list[float] + frequencies: list[float] + confidence: list[float] diff --git a/src/modules/Pitcher/pitcher.py b/src/modules/Pitcher/pitcher.py index 2d17c67..5bf9f77 100644 --- a/src/modules/Pitcher/pitcher.py +++ b/src/modules/Pitcher/pitcher.py @@ -3,49 +3,50 @@ import crepe from scipy.io import wavfile -from modules.console_colors import ( - ULTRASINGER_HEAD, - blue_highlighted, - red_highlighted, -) +from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted from modules.Pitcher.pitched_data import PitchedData -def get_pitch_with_crepe_file(filename: str, step_size: int, model_capacity: str) -> PitchedData: +def get_pitch_with_crepe_file( + filename: str, model_capacity: str, step_size: int = 10, device: str = "cpu" +) -> PitchedData: """Pitch with crepe""" + print( - f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker" + f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted(device)} as worker" ) - # Todo: add GPU support by using torchcrepe sample_rate, audio = wavfile.read(filename) - pitched_data = PitchedData() - ( - pitched_data.times, - pitched_data.frequencies, - pitched_data.confidence, - activation, - ) = crepe.predict( - audio, sample_rate, model_capacity, step_size=step_size, viterbi=True - ) - return pitched_data + return get_pitch_with_crepe(audio, sample_rate, model_capacity, step_size) -def get_pitch_with_crepe(audio, sample_rate: int, step_size: int, model_capacity: str) -> PitchedData: +def get_pitch_with_crepe( + audio, sample_rate: int, model_capacity: str, step_size: int = 10 +) -> PitchedData: """Pitch with crepe""" - pitched_data = PitchedData() - ( - pitched_data.times, - pitched_data.frequencies, - pitched_data.confidence, - activation, - ) = crepe.predict( + times, frequencies, confidence, activation = crepe.predict( audio, sample_rate, model_capacity, step_size=step_size, viterbi=True ) - return pitched_data + return PitchedData(times, frequencies, confidence) -def get_frequency_with_high_confidence(frequencies: list[float], confidences: list[float], threshold=0.4) -> list[float]: +def get_pitched_data_with_high_confidence( + pitched_data: PitchedData, threshold=0.4 +) -> PitchedData: + """Get frequency with high confidence""" + new_pitched_data = PitchedData([], [], []) + for i, conf in enumerate(pitched_data.confidence): + if conf > threshold: + new_pitched_data.times.append(pitched_data.times[i]) + new_pitched_data.frequencies.append(pitched_data.frequencies[i]) + new_pitched_data.confidence.append(pitched_data.confidence[i]) + + return new_pitched_data + + +def get_frequencies_with_high_confidence( + frequencies: list[float], confidences: list[float], threshold=0.4 +) -> list[float]: """Get frequency with high confidence""" conf_f = [] for i, conf in enumerate(confidences): diff --git a/src/modules/Speech_Recognition/Whisper.py b/src/modules/Speech_Recognition/Whisper.py index 12bb7ee..0545ff7 100644 --- a/src/modules/Speech_Recognition/Whisper.py +++ b/src/modules/Speech_Recognition/Whisper.py @@ -1,18 +1,23 @@ """Whisper Speech Recognition Module""" -import whisperx import sys -from modules.console_colors import ( - ULTRASINGER_HEAD, - blue_highlighted, - red_highlighted, -) +import whisperx +from torch.cuda import OutOfMemoryError + +from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted from modules.Speech_Recognition.TranscribedData import TranscribedData -def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> ( -list[TranscribedData], str): +def transcribe_with_whisper( + audio_path: str, + model: str, + device="cpu", + model_name: str = None, + batch_size: int = 16, + compute_type: str = None, + language: str = None, +) -> (list[TranscribedData], str): """Transcribe with whisper""" print( @@ -21,31 +26,57 @@ list[TranscribedData], str): if model_name is not None: print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}") - batch_size = 16 # reduce if low on GPU mem - compute_type = ( - "float16" if device == "cuda" else "int8" - ) # change to "int8" if low on GPU mem (may reduce accuracy) + if compute_type is None: + compute_type = "float16" if device == "cuda" else "int8" - loaded_whisper_model = whisperx.load_model( - model, device=device, compute_type=compute_type - ) + try: + loaded_whisper_model = whisperx.load_model( + model, language=language, device=device, compute_type=compute_type + ) + except ValueError as value_error: + if ( + "Requested float16 compute type, but the target device or backend do not support efficient float16 computation." + in str(value_error.args[0]) + ): + print(value_error) + print( + f"{ULTRASINGER_HEAD} Your GPU does not support efficient float16 computation; run UltraSinger with '--whisper_compute_type int8'" + ) + sys.exit(1) + + raise value_error + except OutOfMemoryError as oom_exception: + print(oom_exception) + print( + f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu" + ) + sys.exit(1) audio = whisperx.load_audio(audio_path) print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}") - result = loaded_whisper_model.transcribe(audio, batch_size=batch_size) - language = result["language"] + result = loaded_whisper_model.transcribe( + audio, batch_size=batch_size, language=language + ) + + detected_language = result["language"] + if language is None: + language = detected_language # load alignment model and metadata try: - model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name) + model_a, metadata = whisperx.load_align_model( + language_code=language, device=device, model_name=model_name + ) except ValueError as ve: - print(f"{red_highlighted(f'{ve}')}" - f"\n" - f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. " - f"Try add it with --align_model [hugingface].") - sys.exit(0) + print( + f"{red_highlighted(f'{ve}')}" + f"\n" + f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. " + f"Try add it with --align_model [hugingface]." + ) + sys.exit(1) # align whisper output result_aligned = whisperx.align( @@ -59,7 +90,7 @@ list[TranscribedData], str): transcribed_data = convert_to_transcribed_data(result_aligned) - return transcribed_data, language + return transcribed_data, detected_language def convert_to_transcribed_data(result_aligned): diff --git a/src/modules/Speech_Recognition/hyphenation.py b/src/modules/Speech_Recognition/hyphenation.py index a166c56..c117059 100644 --- a/src/modules/Speech_Recognition/hyphenation.py +++ b/src/modules/Speech_Recognition/hyphenation.py @@ -88,6 +88,11 @@ def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None: """Hyphenate word""" cleaned_string, removed_indices, removed_symbols = clean_word(word) + + # Hyphenation of word longer than 100 characters throws exception + if len(cleaned_string) > 100: + return None + syllabus = hyphenator.syllables(cleaned_string) length = len(syllabus) diff --git a/src/modules/plot.py b/src/modules/plot.py index 13bf9c7..881a453 100644 --- a/src/modules/plot.py +++ b/src/modules/plot.py @@ -1,71 +1,248 @@ """Plot transcribed data""" - import os +from dataclasses import dataclass +from re import sub + import librosa +import numpy from matplotlib import pyplot as plt from matplotlib.patches import Rectangle -from modules.Pitcher.pitched_data import PitchedData -from modules.Speech_Recognition.TranscribedData import TranscribedData + from modules.console_colors import ULTRASINGER_HEAD +from modules.Pitcher.pitched_data import PitchedData +from modules.Pitcher.pitcher import get_pitched_data_with_high_confidence +from modules.Speech_Recognition.TranscribedData import TranscribedData + + +@dataclass +class PlottedNote: + """Plotted note""" + + note: str + frequency: float + frequency_log_10: float + octave: int + + +NOTES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"] +OCTAVES = [0, 1, 2, 3, 4, 5, 6, 7, 8] +X_TICK_SIZE = 5 def get_frequency_range(midi_note: str) -> float: + """Get frequency range""" midi = librosa.note_to_midi(midi_note) frequency_range = librosa.midi_to_hz(midi + 1) - librosa.midi_to_hz(midi) return frequency_range -def plot(pitched_data: PitchedData, - transcribed_data: list[TranscribedData], - midi_notes: list[str], - output_path: str - ) -> None: +def create_plot_notes(notes: list[str], octaves: list[int]) -> list[PlottedNote]: + """Create list of notes for plot y axis""" + plotted_notes = [] + for octave in octaves: + for note in notes: + note_with_octave = note + str(octave) + frequency = librosa.note_to_hz(note_with_octave) + frequency_log_10 = numpy.log10([frequency])[0] + plotted_notes.append( + PlottedNote(note_with_octave, frequency, frequency_log_10, octave) + ) + + return plotted_notes + + +PLOTTED_NOTES = create_plot_notes(NOTES, OCTAVES) + + +def plot( + pitched_data: PitchedData, + output_path: str, + transcribed_data: list[TranscribedData] = None, + midi_notes: list[str] = None, + title: str = None, +) -> None: """Plot transcribed data""" - print(f"{ULTRASINGER_HEAD} Creating plot") + # determine time between to datapoints if there is no gap (this is the step size crepe ran with) + step_size = pitched_data.times[1] + pitched_data = get_pitched_data_with_high_confidence(pitched_data) - conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4) + if len(pitched_data.frequencies) < 2: + print(f"{ULTRASINGER_HEAD} Plot can't be created; too few datapoints") + return - plt.ylim(0, max(conf_f)) - plt.xlim(0, pitched_data.times[-1]) - plt.plot(conf_t, conf_f, linewidth=0.1) + print( + f"{ULTRASINGER_HEAD} Creating plot{': ' + title if title is not None else ''}" + ) - for i, data in enumerate(transcribed_data): - note_frequency = librosa.note_to_hz(midi_notes[i]) - frequency_range = get_frequency_range(midi_notes[i]) - xy_start_pos = (data.start, note_frequency - frequency_range / 2) - width = data.end - data.start - height = frequency_range - rect = Rectangle( - xy_start_pos, - width, - height, - edgecolor='none', - facecolor='red', - alpha=0.5) - plt.gca().add_patch(rect) + # map each frequency to logarithm with base 10 for a linear progression of values between the musical notes + # see http://www.phon.ox.ac.uk/jcoleman/LOGARITH.htm + frequencies_log_10 = numpy.log10(pitched_data.frequencies) + + # add 'nan' where there are gaps for frequency values so the graph is only continuous where it should be + pitched_data_with_gaps = create_gaps(pitched_data, step_size) + frequencies_log_10_with_gaps = numpy.log10(pitched_data_with_gaps.frequencies) + + # dynamically set the minimum and maximum values for x and y axes based on data + y_lower_bound, y_upper_bound = determine_bounds(frequencies_log_10) + ymin = max(0, y_lower_bound - 0.05) + ymax = y_upper_bound + 0.05 + plt.ylim(ymin, ymax) + xmin = min(pitched_data.times) + xmax = max(pitched_data.times) + plt.xlim(xmin, xmax) plt.xlabel("Time (s)") - plt.ylabel("Frequency (Hz)") - plt.ylim(0, 800) - dpi = 4000 + plt.ylabel("log10 of Frequency (Hz)") - plt.savefig(os.path.join(output_path, "plot.png"), dpi=dpi) + notes_within_range = set_axes_ticks_and_labels(pitched_data.times, ymin, ymax) + + # draw horizontal lines for each note + for note in notes_within_range: + color = "b" + if note.note.startswith("C") and not note.note.startswith("C#"): + color = "r" + plt.axhline(y=note.frequency_log_10, color=color, linestyle="-", linewidth=0.2) + + # create line and scatter plot of pitched data + plt.plot(pitched_data_with_gaps.times, frequencies_log_10_with_gaps, linewidth=0.1) + scatter_path_collection = plt.scatter( + pitched_data_with_gaps.times, + frequencies_log_10_with_gaps, + s=5, + c=pitched_data_with_gaps.confidence, + cmap=plt.colormaps.get_cmap("gray").reversed(), + vmin=0, + vmax=1, + ) + plt.figure(1).colorbar(scatter_path_collection, label="confidence") + + set_figure_dimensions(xmax - xmin, y_upper_bound - y_lower_bound) + + draw_words(transcribed_data, midi_notes) + + if title is not None: + plt.title(label=title) + + plt.figure(1).tight_layout(h_pad=1.4) + + dpi = 200 + plt.savefig( + os.path.join( + output_path, f"plot{'' if title is None else '_' + snake(title)}.svg" + ), + dpi=dpi, + ) + plt.clf() + plt.cla() -def get_confidence( - pitched_data: PitchedData, - threshold: float -) -> tuple[list[float], list[float], list[float]]: - """Get high confidence data""" - # todo: replace get_frequency_with_high_conf from pitcher - conf_t = [] - conf_f = [] - conf_c = [] - for i in enumerate(pitched_data.times): - pos = i[0] - if pitched_data.confidence[pos] > threshold: - conf_t.append(pitched_data.times[pos]) - conf_f.append(pitched_data.frequencies[pos]) - conf_c.append(pitched_data.confidence[pos]) - return conf_t, conf_f, conf_c +def set_axes_ticks_and_labels(confidence, ymin, ymax): + """Set ticks and their labels for x and y axes""" + notes_within_range = [ + x for x in PLOTTED_NOTES if ymin <= x.frequency_log_10 <= ymax + ] + plt.yticks( + [x.frequency_log_10 for x in notes_within_range], + [x.note for x in notes_within_range], + ) + + first_time = min(confidence) + min_tick = first_time // X_TICK_SIZE * X_TICK_SIZE + X_TICK_SIZE + + last_time = max(confidence) + max_tick = last_time // X_TICK_SIZE * X_TICK_SIZE + 0.1 + ticks = numpy.arange(min_tick, max_tick, X_TICK_SIZE, dtype=int).tolist() + + if len(ticks) == 0 or ticks[0] != first_time: + ticks.insert(0, first_time) + + if len(ticks) == 1 or ticks[-1] != last_time: + ticks.append(last_time) + + plt.xticks(ticks, [str(x) for x in ticks]) + return notes_within_range + + +def determine_bounds(frequency_log_10: list[float]) -> tuple[float, float]: + """Determine bounds based on 1st and 99th percentile of data""" + lower = numpy.percentile(numpy.array(frequency_log_10), 1) + upper = numpy.percentile(numpy.array(frequency_log_10), 99) + + return lower, upper + + +def set_figure_dimensions(time_range, frequency_log_10_range): + """Dynamically scale the figure dimensions based on the duration/frequency amplitude of the song""" + height = frequency_log_10_range / 0.06 + width = time_range / 2 + + plt.figure(1).set_figwidth(max(6.4, width)) + plt.figure(1).set_figheight(max(4, height)) + + +def create_gaps(pitched_data: PitchedData, step_size: float) -> PitchedData: + """ + Add 'nan' where there are no high confidence frequency values. + This way the graph is only continuous where it should be. + + """ + pitched_data_with_gaps = PitchedData([], [], []) + + previous_time = 0 + for i, time in enumerate(pitched_data.times): + comes_right_after_previous = time - previous_time <= step_size + previous_frequency_is_not_gap = ( + len(pitched_data_with_gaps.frequencies) > 0 + and str(pitched_data_with_gaps.frequencies[-1]) != "nan" + ) + if previous_frequency_is_not_gap and not comes_right_after_previous: + pitched_data_with_gaps.times.append(time) + pitched_data_with_gaps.frequencies.append(float("nan")) + pitched_data_with_gaps.confidence.append(pitched_data.confidence[i]) + + pitched_data_with_gaps.times.append(time) + pitched_data_with_gaps.frequencies.append(pitched_data.frequencies[i]) + pitched_data_with_gaps.confidence.append(pitched_data.confidence[i]) + + previous_time = time + + return pitched_data_with_gaps + + +def draw_words(transcribed_data, midi_notes): + """Draw rectangles for each word""" + if transcribed_data is not None: + for i, data in enumerate(transcribed_data): + note_frequency = librosa.note_to_hz(midi_notes[i]) + frequency_range = get_frequency_range(midi_notes[i]) + + half_frequency_range = frequency_range / 2 + height = ( + numpy.log10([note_frequency + half_frequency_range])[0] + - numpy.log10([note_frequency - half_frequency_range])[0] + ) + + xy_start_pos = ( + data.start, + numpy.log10([note_frequency - half_frequency_range])[0], + ) + width = data.end - data.start + rect = Rectangle( + xy_start_pos, + width, + height, + edgecolor="none", + facecolor="red", + alpha=0.5, + ) + plt.gca().add_patch(rect) + + +def snake(s): + """Turn any string into a snake case string""" + return "_".join( + sub( + "([A-Z][a-z]+)", r" \1", sub("([A-Z]+)", r" \1", s.replace("-", " ")) + ).split() + ).lower() diff --git a/src/modules/timer.py b/src/modules/timer.py new file mode 100644 index 0000000..c46be42 --- /dev/null +++ b/src/modules/timer.py @@ -0,0 +1,26 @@ +import atexit +from functools import reduce +from time import process_time + +from modules.console_colors import ULTRASINGER_HEAD + + +def seconds_to_str(t): + """Format seconds to string""" + return "%d:%02d:%02d.%03d" % reduce( + lambda ll, b: divmod(ll[0], b) + ll[1:], [(t * 1000,), 1000, 60, 60] + ) + + +def log(s): + """Log line with optional time elapsed""" + print(f"{ULTRASINGER_HEAD} {seconds_to_str(process_time())} - {s}") + + +def end_log(): + """Log at program end""" + log("End Program") + + +atexit.register(end_log) +log("Initialized...")