Merge pull request #89 from BWagener/feature/crepe-windows-gpu

Crepe with windows GPU support
This commit is contained in:
Vadim Rangnau authored and GitHub committed 2023-09-14 19:44:12 +02:00
commit fb4049ff62
20 files changed
+568 -196

No files matched your search

+7 -1
View File
@@ -1,3 +1,5 @@
.env
.vscode/
# virtual environment
venv
.venv
@@ -5,5 +7,9 @@ venv
# python
**__pycache__
# test data
test_input
test_output
# default output folder
output
output
+27 -9
View File
@@ -112,15 +112,18 @@ _Not all options working now!_
[transcription]
# Default is whisper
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--align_model Use other languages model for Whisper provided from huggingface.co
--vosk Needs model
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--whisper_align_model Use other languages model for Whisper provided from huggingface.co
--language Override the language detected by whisper, does not affect transcription but steps after transcription
--whisper_batch_size Reduce if low on GPU mem >> ((default) is 16)
--whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
--vosk Needs model
[pitcher]
# Default is crepe
--crepe tiny|small|medium|large|full >> ((default) is full)
--crepe tiny|full >> ((default) is full)
--crepe_step_size unit is miliseconds >> ((default) is 10)
[extra]
--hyphenation True|False >> ((default) is True)
@@ -128,6 +131,7 @@ _Not all options working now!_
--disable_karaoke True|False >> ((default) is False)
--create_audio_chunks True|False >> ((default) is False)
--plot True|False >> ((default) is False)
--force_cpu True|False >> ((default) is False)
```
For standard use, you only need to use [opt]. All other options are optional.
@@ -205,7 +209,7 @@ starts at the place or is heard.
### Pitcher
Pitching is done with the `crepe` model.
Pitching is done with the `crepe` model.
Also consider that a bigger model is more accurate, but also takes longer to pitch.
For just testing you should use `tiny`.
If you want solid accurate, then use the `full` model.
@@ -252,9 +256,23 @@ pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu11
When you want to use `conda` instead you need a different installation command. See this [link](https://pytorch.org/get-started/locally/).
#### Considerations for Windows users
The pitch tracker used by UltraSinger (crepe), uses TensorFlow as it's backend.
TensorFlow dropped GPU support for Windows for versions >2.10 as you can see in this [release note](https://github.com/tensorflow/tensorflow/releases/tag/v2.11.1) and their [installation instructions](https://www.tensorflow.org/install/pip#windows-native).
For now UltraSinger runs the latest version available that still supports GPUs on windows.
For running later versions of TensorFlow on windows while still taking advantage of GPU support the suggested solution is:
* [install WSL2](https://learn.microsoft.com/en-us/windows/wsl/install)
* within the Ubuntu WSL2 installation
* run `sudo apt update && sudo apt install nvidia-cuda-toolkit`
* follow the setup instructions for UltraSinger at the top of this document
#### Info
If somthing crash because of low VRAM than use a smaller model.
If something crashes because of low VRAM than use a smaller model.
Whisper needs more than 8GB VRAM in the `large` model!
But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`.
You can also force cpu usage with the extra option `--force_cpu`.
View File
Whitespace-only changes.
+2
View File
@@ -1,3 +1,5 @@
[tool.isort]
profile = "black"
[tool.pytest.ini_options]
pythonpath = [
"src",
+24
View File
@@ -0,0 +1,24 @@
"""Tests for whisper.py"""
import os
import unittest
from src.modules.Audio.denoise import ffmpeg_reduce_noise
import pytest
class DenoiseTest(unittest.TestCase):
@pytest.mark.skip(reason="Skipping this FUNCTION level test, can be used for manual tests")
def test_ffmpeg_reduce_noise(self):
# Arrange
test_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.abspath(test_dir + "/../../..")
test_file_abs_path = os.path.abspath(root_dir + "/test_input/vocals.wav")
test_file_name = os.path.basename(test_file_abs_path)
test_output = test_dir + "/test_output"
# Act
ffmpeg_reduce_noise(test_file_abs_path, test_output + "/output_" + test_file_name)
if __name__ == "__main__":
unittest.main()
+27
View File
@@ -0,0 +1,27 @@
"""Tests for pitcher.py"""
import os
import unittest
import src.modules.Pitcher.pitcher as test_subject
import pytest
from src.modules.plot import plot
class PitcherTest(unittest.TestCase):
@pytest.mark.skip(reason="Skipping this FUNCTION level test, can be used for manual tests")
def test_get_pitch_with_crepe_file(self):
# Arrange
test_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.abspath(test_dir + "/../../..")
# test_file_abs_path = os.path.abspath(root_dir + "/test_input/audio_denoised.wav")
test_file_abs_path = os.path.abspath(root_dir + "/test_input/test_denoised.wav")
test_output = root_dir + "/test_output"
# Act
pitched_data = test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', device="cuda")
# test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', 'cpu', batch_size=1024)
plot(pitched_data, test_output, title="pitching test")
print("done")
if __name__ == "__main__":
unittest.main()
+23 -21
View File
@@ -1,25 +1,27 @@
crepe==0.0.13
librosa==0.9.2
pretty_midi~=0.2.10
pydub==0.25.1
Levenshtein~=0.21.0
scipy~=1.10.1
vosk==0.3.44
tensorflow~=2.12.0
#tensorflow-directml-plugin~=0.4.0.dev230202
#cudatoolkit==11.8.0
matplotlib~=3.7.1
yt_dlp~=2023.7.6
numpy~=1.23.5
git+https://github.com/m-bain/whisperx.git
PyHyphen~=4.0.3
crepe~=0.0.13
demucs~=4.0.0
tqdm~=4.65.0
ffmpeg_python~=0.2.0
git+https://github.com/m-bain/whisperx.git
langcodes~=3.3.0
language-data~=1.1
Pillow~=9.5.0
isort==5.12
black==23.3
pylint==2.17
librosa~=0.9.2
matplotlib~=3.7.2
musicbrainzngs~=0.7.1
numpy~=1.23.5
Pillow~=10.0.0
pretty_midi~=0.2.10
pydub~=0.25.1
PyHyphen~=4.0.3
python_Levenshtein~=0.21.1
scipy~=1.11.2
tensorflow<2.11
tqdm~=4.65.2
vosk~=0.3.44
whisperx~=3.1.1
yt_dlp~=2023.7.6
isort~=5.12
black~=23.3
pylint~=2.17
pytest~=7.3.1
musicbrainzngs~=0.7.1
protobuf==3.20.*
+8 -4
View File
@@ -10,6 +10,8 @@ class Settings:
output_file_path = ""
mono_audio_path = ""
language = None
# Transcribe
audio_chunk_folder_name = "audio-chunks"
@@ -18,15 +20,17 @@ class Settings:
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
whisper_batch_size = 16 # reduce if low on GPU mem
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
# Vosk
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
# Pitch
crepe_model_capacity = "full" # tiny|small|medium|large|full
crepe_step_size = 10
crepe_step_size = 10 # in miliseconds
# Device
device = 'cpu' # cpu|cuda
force_whisper_cpu = False
force_separation_cpu = False
pytorch_device = 'cpu' # cpu|cuda
tensorflow_device = 'cpu' # cpu|cuda
force_cpu = False
+64 -39
View File
@@ -4,20 +4,19 @@ import copy
import getopt
import os
import sys
from time import sleep
import Levenshtein
import librosa
from tqdm import tqdm
from modules import os_helper
from modules import os_helper, timer
from modules.Audio.denoise import ffmpeg_reduce_noise
from modules.Audio.separation import separate_audio
from modules.Audio.vocal_chunks import (
export_chunks_from_transcribed_data,
export_chunks_from_ultrastar_data,
)
from modules.Audio.silence_processing import remove_silence_from_transcribtion_data
from modules.Audio.silence_processing import remove_silence_from_transcription_data
from modules.csv_handler import export_transcribed_data_to_csv
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
from modules.Audio.youtube import (
@@ -26,7 +25,7 @@ from modules.Audio.youtube import (
download_youtube_video,
get_youtube_title,
)
from modules.DeviceDetection.device_detection import get_available_device
from modules.DeviceDetection.device_detection import check_gpu_support
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
@@ -41,7 +40,7 @@ from modules.Midi.midi_creator import (
most_frequent,
)
from modules.Pitcher.pitcher import (
get_frequency_with_high_confidence,
get_frequencies_with_high_confidence,
get_pitch_with_crepe_file,
)
from modules.Pitcher.pitched_data import PitchedData
@@ -91,9 +90,12 @@ def pitch_each_chunk_with_crepe(directory: str) -> list[str]:
filepath = os.path.join(directory, filename)
# todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes
pitched_data = get_pitch_with_crepe_file(
filepath, settings.crepe_step_size, settings.crepe_model_capacity
filepath,
settings.crepe_model_capacity,
settings.crepe_step_size,
settings.tensorflow_device,
)
conf_f = get_frequency_with_high_confidence(
conf_f = get_frequencies_with_high_confidence(
pitched_data.frequencies, pitched_data.confidence
)
@@ -230,7 +232,6 @@ def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData])
)
return None
sleep(0.1)
hyphenator = create_hyphenator(lang_region)
for i in tqdm(enumerate(transcribed_data)):
pos = i[0]
@@ -309,20 +310,21 @@ def run() -> None:
# Audio transcription
transcribed_data = None
language = None
language = settings.language
if is_audio:
language, transcribed_data = transcribe_audio()
detected_language, transcribed_data = transcribe_audio()
if language is None:
language = detected_language
remove_unecessary_punctuations(transcribed_data)
transcribed_data = remove_silence_from_transcribtion_data(
transcribed_data = remove_silence_from_transcription_data(
settings.mono_audio_path, transcribed_data
)
if settings.hyphenation:
hyphen_words = hyphenate_each_word(language, transcribed_data)
if hyphen_words is not None:
transcribed_data = add_hyphen_to_data(
transcribed_data, hyphen_words
)
transcribed_data = add_hyphen_to_data(transcribed_data, hyphen_words)
# todo: do we need to correct words?
# lyric = 'input/faber_lyric.txt'
@@ -345,7 +347,7 @@ def run() -> None:
# Create plot
if settings.create_plot:
plot(pitched_data, transcribed_data, midi_notes, song_output)
plot(pitched_data, song_output, transcribed_data, midi_notes)
# Write Ultrastar txt
if is_audio:
@@ -408,16 +410,21 @@ def get_unused_song_output_dir(path: str) -> str:
def transcribe_audio() -> (str, list[TranscribedData]):
"""Transcribe audio with AI"""
if settings.transcriber == "whisper":
device = "cpu" if settings.force_whisper_cpu else settings.device
transcribed_data, language = transcribe_with_whisper(
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
transcribed_data, detected_language = transcribe_with_whisper(
settings.mono_audio_path,
settings.whisper_model,
settings.pytorch_device,
settings.whisper_align_model,
settings.whisper_batch_size,
settings.whisper_compute_type,
settings.language,
)
else: # vosk
transcribed_data = transcribe_with_vosk(
settings.mono_audio_path, settings.vosk_model_path
)
# todo: make language selectable
language = "en"
return language, transcribed_data
detected_language = "en"
return detected_language, transcribed_data
def separate_vocal_from_audio(
@@ -427,16 +434,16 @@ def separate_vocal_from_audio(
audio_separation_path = os.path.join(
cache_path, "separated", "htdemucs", basename_without_ext
)
device = "cpu" if settings.force_separation_cpu else settings.device
if settings.use_separated_vocal or settings.create_karaoke:
separate_audio(ultrastar_audio_input_path, cache_path, device)
separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device)
if settings.use_separated_vocal:
vocals_path = os.path.join(audio_separation_path, "vocals.wav")
convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path)
input_path = os.path.join(audio_separation_path, "vocals.wav")
else:
convert_audio_to_mono_wav(
ultrastar_audio_input_path, settings.mono_audio_path
)
input_path = ultrastar_audio_input_path
convert_audio_to_mono_wav(input_path, settings.mono_audio_path)
return audio_separation_path
@@ -697,8 +704,9 @@ def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrast
# midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name)
pitched_data = get_pitch_with_crepe_file(
settings.mono_audio_path,
settings.crepe_step_size,
settings.crepe_model_capacity,
settings.crepe_step_size,
settings.tensorflow_device,
)
if is_audio:
start_times = []
@@ -777,15 +785,25 @@ def init_settings(argv: list[str]) -> None:
elif opt in ("--whisper"):
settings.transcriber = "whisper"
settings.whisper_model = arg
elif opt in ("--align_model"):
elif opt in ("--whisper_align_model"):
settings.whisper_align_model = arg
elif opt in ("--whisper_batch_size"):
settings.whisper_batch_size = int(arg)
elif opt in ("--whisper_compute_type"):
settings.whisper_compute_type = arg
elif opt in ("--language"):
settings.language = arg
elif opt in ("--vosk"):
settings.transcriber = "vosk"
settings.vosk_model_path = arg
elif opt in ("--crepe"):
settings.crepe_model_capacity = arg
elif opt in ("--crepe_step_size"):
settings.crepe_step_size = int(arg)
elif opt in ("--plot"):
settings.create_plot = arg
settings.create_plot = arg in ["True", "true"]
elif opt in ("--midi"):
settings.create_midi = arg in ["True", "true"]
elif opt in ("--hyphenation"):
settings.hyphenation = arg
elif opt in ("--disable_separation"):
@@ -794,17 +812,20 @@ def init_settings(argv: list[str]) -> None:
settings.create_karaoke = not arg
elif opt in ("--create_audio_chunks"):
settings.create_audio_chunks = arg
elif opt in ("--force_whisper_cpu"):
settings.force_whisper_cpu = arg
elif opt in ("--force_separation_cpu"):
settings.force_separation_cpu = arg
elif opt in ("--force_cpu"):
settings.force_cpu = arg
if settings.force_cpu:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
if settings.output_file_path == "":
if settings.input_file_path.startswith("https:"):
dirname = os.getcwd()
else:
dirname = os.path.dirname(settings.input_file_path)
settings.output_file_path = os.path.join(dirname, "output")
settings.device = get_available_device()
if not settings.force_cpu:
settings.tensorflow_device, settings.pytorch_device = check_gpu_support()
def arg_options():
@@ -813,16 +834,20 @@ def arg_options():
"ifile=",
"ofile=",
"crepe=",
"crepe_step_size=",
"vosk=",
"whisper=",
"align_model=",
"whisper_align_model=",
"whisper_batch_size=",
"whisper_compute_type=",
"language=",
"plot=",
"midi=",
"hyphenation=",
"disable_separation=",
"disable_karaoke=",
"create_audio_chunks=",
"force_whisper_cpu=",
"force_separation_cpu="
"force_cpu=",
]
return long, short
+11 -6
View File
@@ -22,9 +22,14 @@ def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None:
print(
f"{ULTRASINGER_HEAD} Reduce noise from vocal audio with {blue_highlighted('ffmpeg')}."
)
(
ffmpeg.input(input_file_path)
.output(output_file, af="afftdn=nr=70:nf=-50:tn=1")
.overwrite_output()
.run()
)
try:
(
ffmpeg.input(input_file_path)
.output(output_file, af="afftdn=nr=70:nf=-50:tn=1")
.overwrite_output()
.run(capture_stdout=True, capture_stderr=True)
)
except ffmpeg.Error as ffmpeg_exception:
print("ffmpeg stdout:", ffmpeg_exception.stdout.decode("utf8"))
print("ffmpeg stderr:", ffmpeg_exception.stderr.decode("utf8"))
raise ffmpeg_exception
-1
View File
@@ -19,7 +19,6 @@ def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None
# Model selection?
# -n mdx_q
# -n htdemucs_ft
device = "cpu" if device != "cuda" else "cuda"
subprocess.run(
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
)
+2 -2
View File
@@ -6,8 +6,8 @@ from modules.console_colors import ULTRASINGER_HEAD
from modules.Speech_Recognition.TranscribedData import TranscribedData
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]:
"""Remove silence from given transcribtion data"""
def remove_silence_from_transcription_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]:
"""Remove silence from given transcription data"""
print(
f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data"
@@ -1,19 +1,37 @@
"""Device detection module."""
import torch
import os
import tensorflow as tf
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted, blue_highlighted
tensorflow_gpu_supported = False
pytorch_gpu_supported = False
def get_available_device() -> str:
"""Get available worker device e.g cuda or cpu"""
is_cuda = torch.cuda.is_available()
def check_gpu_support() -> tuple[bool, bool]:
"""Check worker device (e.g cuda or cpu) supported by tensorflow and pytorch"""
if not is_cuda:
print(f"{ULTRASINGER_HEAD} Checking GPU support for {blue_highlighted('tensorflow')} and {blue_highlighted('pytorch')}.")
tensorflow_gpu_supported = False
pytorch_gpu_supported = False
gpus = tf.config.list_physical_devices('GPU')
if gpus:
tensorflow_gpu_supported = True
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - using {red_highlighted('cuda')} gpu.")
else:
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}.")
if os.name == 'nt':
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - versions above 2.10 dropped GPU support for Windows, refer to the readme for possible solutions.")
pytorch_gpu_supported = torch.cuda.is_available()
if not pytorch_gpu_supported:
print(
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
f"{ULTRASINGER_HEAD} {blue_highlighted('pytorch')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}."
)
return "cpu"
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
else:
print(f"{ULTRASINGER_HEAD} {blue_highlighted('pytorch')} - using {red_highlighted('cuda')} gpu.")
return "cuda"
return 'cuda' if tensorflow_gpu_supported else 'cpu', 'cuda' if pytorch_gpu_supported else 'cpu'
+2 -2
View File
@@ -7,7 +7,7 @@ import librosa
import numpy as np
import pretty_midi
from modules.Pitcher.pitcher import get_frequency_with_high_confidence
from modules.Pitcher.pitcher import get_frequencies_with_high_confidence
from modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
@@ -115,7 +115,7 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch
freqs = pitched_data.frequencies[start:end]
confs = pitched_data.confidence[start:end]
conf_f = get_frequency_with_high_confidence(freqs, confs)
conf_f = get_frequencies_with_high_confidence(freqs, confs)
notes = convert_frequencies_to_notes(conf_f)
+5 -3
View File
@@ -1,9 +1,11 @@
"""Pitched data"""
from dataclasses import dataclass
@dataclass
class PitchedData:
"""Pitched data from crepe"""
times = list[float]
frequencies = list[float]
confidence = list[float]
times: list[float]
frequencies: list[float]
confidence: list[float]
+29 -28
View File
@@ -3,49 +3,50 @@
import crepe
from scipy.io import wavfile
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
from modules.Pitcher.pitched_data import PitchedData
def get_pitch_with_crepe_file(filename: str, step_size: int, model_capacity: str) -> PitchedData:
def get_pitch_with_crepe_file(
filename: str, model_capacity: str, step_size: int = 10, device: str = "cpu"
) -> PitchedData:
"""Pitch with crepe"""
print(
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted(device)} as worker"
)
# Todo: add GPU support by using torchcrepe
sample_rate, audio = wavfile.read(filename)
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
return get_pitch_with_crepe(audio, sample_rate, model_capacity, step_size)
def get_pitch_with_crepe(audio, sample_rate: int, step_size: int, model_capacity: str) -> PitchedData:
def get_pitch_with_crepe(
audio, sample_rate: int, model_capacity: str, step_size: int = 10
) -> PitchedData:
"""Pitch with crepe"""
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
times, frequencies, confidence, activation = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
return PitchedData(times, frequencies, confidence)
def get_frequency_with_high_confidence(frequencies: list[float], confidences: list[float], threshold=0.4) -> list[float]:
def get_pitched_data_with_high_confidence(
pitched_data: PitchedData, threshold=0.4
) -> PitchedData:
"""Get frequency with high confidence"""
new_pitched_data = PitchedData([], [], [])
for i, conf in enumerate(pitched_data.confidence):
if conf > threshold:
new_pitched_data.times.append(pitched_data.times[i])
new_pitched_data.frequencies.append(pitched_data.frequencies[i])
new_pitched_data.confidence.append(pitched_data.confidence[i])
return new_pitched_data
def get_frequencies_with_high_confidence(
frequencies: list[float], confidences: list[float], threshold=0.4
) -> list[float]:
"""Get frequency with high confidence"""
conf_f = []
for i, conf in enumerate(confidences):
+55 -24
View File
@@ -1,18 +1,23 @@
"""Whisper Speech Recognition Module"""
import whisperx
import sys
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
import whisperx
from torch.cuda import OutOfMemoryError
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
from modules.Speech_Recognition.TranscribedData import TranscribedData
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (
list[TranscribedData], str):
def transcribe_with_whisper(
audio_path: str,
model: str,
device="cpu",
model_name: str = None,
batch_size: int = 16,
compute_type: str = None,
language: str = None,
) -> (list[TranscribedData], str):
"""Transcribe with whisper"""
print(
@@ -21,31 +26,57 @@ list[TranscribedData], str):
if model_name is not None:
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
batch_size = 16 # reduce if low on GPU mem
compute_type = (
"float16" if device == "cuda" else "int8"
) # change to "int8" if low on GPU mem (may reduce accuracy)
if compute_type is None:
compute_type = "float16" if device == "cuda" else "int8"
loaded_whisper_model = whisperx.load_model(
model, device=device, compute_type=compute_type
)
try:
loaded_whisper_model = whisperx.load_model(
model, language=language, device=device, compute_type=compute_type
)
except ValueError as value_error:
if (
"Requested float16 compute type, but the target device or backend do not support efficient float16 computation."
in str(value_error.args[0])
):
print(value_error)
print(
f"{ULTRASINGER_HEAD} Your GPU does not support efficient float16 computation; run UltraSinger with '--whisper_compute_type int8'"
)
sys.exit(1)
raise value_error
except OutOfMemoryError as oom_exception:
print(oom_exception)
print(
f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu"
)
sys.exit(1)
audio = whisperx.load_audio(audio_path)
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
language = result["language"]
result = loaded_whisper_model.transcribe(
audio, batch_size=batch_size, language=language
)
detected_language = result["language"]
if language is None:
language = detected_language
# load alignment model and metadata
try:
model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name)
model_a, metadata = whisperx.load_align_model(
language_code=language, device=device, model_name=model_name
)
except ValueError as ve:
print(f"{red_highlighted(f'{ve}')}"
f"\n"
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
f"Try add it with --align_model [hugingface].")
sys.exit(0)
print(
f"{red_highlighted(f'{ve}')}"
f"\n"
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
f"Try add it with --align_model [hugingface]."
)
sys.exit(1)
# align whisper output
result_aligned = whisperx.align(
@@ -59,7 +90,7 @@ list[TranscribedData], str):
transcribed_data = convert_to_transcribed_data(result_aligned)
return transcribed_data, language
return transcribed_data, detected_language
def convert_to_transcribed_data(result_aligned):
@@ -88,6 +88,11 @@ def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None:
"""Hyphenate word"""
cleaned_string, removed_indices, removed_symbols = clean_word(word)
# Hyphenation of word longer than 100 characters throws exception
if len(cleaned_string) > 100:
return None
syllabus = hyphenator.syllables(cleaned_string)
length = len(syllabus)
+224 -47
View File
@@ -1,71 +1,248 @@
"""Plot transcribed data"""
import os
from dataclasses import dataclass
from re import sub
import librosa
import numpy
from matplotlib import pyplot as plt
from matplotlib.patches import Rectangle
from modules.Pitcher.pitched_data import PitchedData
from modules.Speech_Recognition.TranscribedData import TranscribedData
from modules.console_colors import ULTRASINGER_HEAD
from modules.Pitcher.pitched_data import PitchedData
from modules.Pitcher.pitcher import get_pitched_data_with_high_confidence
from modules.Speech_Recognition.TranscribedData import TranscribedData
@dataclass
class PlottedNote:
"""Plotted note"""
note: str
frequency: float
frequency_log_10: float
octave: int
NOTES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]
OCTAVES = [0, 1, 2, 3, 4, 5, 6, 7, 8]
X_TICK_SIZE = 5
def get_frequency_range(midi_note: str) -> float:
"""Get frequency range"""
midi = librosa.note_to_midi(midi_note)
frequency_range = librosa.midi_to_hz(midi + 1) - librosa.midi_to_hz(midi)
return frequency_range
def plot(pitched_data: PitchedData,
transcribed_data: list[TranscribedData],
midi_notes: list[str],
output_path: str
) -> None:
def create_plot_notes(notes: list[str], octaves: list[int]) -> list[PlottedNote]:
"""Create list of notes for plot y axis"""
plotted_notes = []
for octave in octaves:
for note in notes:
note_with_octave = note + str(octave)
frequency = librosa.note_to_hz(note_with_octave)
frequency_log_10 = numpy.log10([frequency])[0]
plotted_notes.append(
PlottedNote(note_with_octave, frequency, frequency_log_10, octave)
)
return plotted_notes
PLOTTED_NOTES = create_plot_notes(NOTES, OCTAVES)
def plot(
pitched_data: PitchedData,
output_path: str,
transcribed_data: list[TranscribedData] = None,
midi_notes: list[str] = None,
title: str = None,
) -> None:
"""Plot transcribed data"""
print(f"{ULTRASINGER_HEAD} Creating plot")
# determine time between to datapoints if there is no gap (this is the step size crepe ran with)
step_size = pitched_data.times[1]
pitched_data = get_pitched_data_with_high_confidence(pitched_data)
conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4)
if len(pitched_data.frequencies) < 2:
print(f"{ULTRASINGER_HEAD} Plot can't be created; too few datapoints")
return
plt.ylim(0, max(conf_f))
plt.xlim(0, pitched_data.times[-1])
plt.plot(conf_t, conf_f, linewidth=0.1)
print(
f"{ULTRASINGER_HEAD} Creating plot{': ' + title if title is not None else ''}"
)
for i, data in enumerate(transcribed_data):
note_frequency = librosa.note_to_hz(midi_notes[i])
frequency_range = get_frequency_range(midi_notes[i])
xy_start_pos = (data.start, note_frequency - frequency_range / 2)
width = data.end - data.start
height = frequency_range
rect = Rectangle(
xy_start_pos,
width,
height,
edgecolor='none',
facecolor='red',
alpha=0.5)
plt.gca().add_patch(rect)
# map each frequency to logarithm with base 10 for a linear progression of values between the musical notes
# see http://www.phon.ox.ac.uk/jcoleman/LOGARITH.htm
frequencies_log_10 = numpy.log10(pitched_data.frequencies)
# add 'nan' where there are gaps for frequency values so the graph is only continuous where it should be
pitched_data_with_gaps = create_gaps(pitched_data, step_size)
frequencies_log_10_with_gaps = numpy.log10(pitched_data_with_gaps.frequencies)
# dynamically set the minimum and maximum values for x and y axes based on data
y_lower_bound, y_upper_bound = determine_bounds(frequencies_log_10)
ymin = max(0, y_lower_bound - 0.05)
ymax = y_upper_bound + 0.05
plt.ylim(ymin, ymax)
xmin = min(pitched_data.times)
xmax = max(pitched_data.times)
plt.xlim(xmin, xmax)
plt.xlabel("Time (s)")
plt.ylabel("Frequency (Hz)")
plt.ylim(0, 800)
dpi = 4000
plt.ylabel("log10 of Frequency (Hz)")
plt.savefig(os.path.join(output_path, "plot.png"), dpi=dpi)
notes_within_range = set_axes_ticks_and_labels(pitched_data.times, ymin, ymax)
# draw horizontal lines for each note
for note in notes_within_range:
color = "b"
if note.note.startswith("C") and not note.note.startswith("C#"):
color = "r"
plt.axhline(y=note.frequency_log_10, color=color, linestyle="-", linewidth=0.2)
# create line and scatter plot of pitched data
plt.plot(pitched_data_with_gaps.times, frequencies_log_10_with_gaps, linewidth=0.1)
scatter_path_collection = plt.scatter(
pitched_data_with_gaps.times,
frequencies_log_10_with_gaps,
s=5,
c=pitched_data_with_gaps.confidence,
cmap=plt.colormaps.get_cmap("gray").reversed(),
vmin=0,
vmax=1,
)
plt.figure(1).colorbar(scatter_path_collection, label="confidence")
set_figure_dimensions(xmax - xmin, y_upper_bound - y_lower_bound)
draw_words(transcribed_data, midi_notes)
if title is not None:
plt.title(label=title)
plt.figure(1).tight_layout(h_pad=1.4)
dpi = 200
plt.savefig(
os.path.join(
output_path, f"plot{'' if title is None else '_' + snake(title)}.svg"
),
dpi=dpi,
)
plt.clf()
plt.cla()
def get_confidence(
pitched_data: PitchedData,
threshold: float
) -> tuple[list[float], list[float], list[float]]:
"""Get high confidence data"""
# todo: replace get_frequency_with_high_conf from pitcher
conf_t = []
conf_f = []
conf_c = []
for i in enumerate(pitched_data.times):
pos = i[0]
if pitched_data.confidence[pos] > threshold:
conf_t.append(pitched_data.times[pos])
conf_f.append(pitched_data.frequencies[pos])
conf_c.append(pitched_data.confidence[pos])
return conf_t, conf_f, conf_c
def set_axes_ticks_and_labels(confidence, ymin, ymax):
"""Set ticks and their labels for x and y axes"""
notes_within_range = [
x for x in PLOTTED_NOTES if ymin <= x.frequency_log_10 <= ymax
]
plt.yticks(
[x.frequency_log_10 for x in notes_within_range],
[x.note for x in notes_within_range],
)
first_time = min(confidence)
min_tick = first_time // X_TICK_SIZE * X_TICK_SIZE + X_TICK_SIZE
last_time = max(confidence)
max_tick = last_time // X_TICK_SIZE * X_TICK_SIZE + 0.1
ticks = numpy.arange(min_tick, max_tick, X_TICK_SIZE, dtype=int).tolist()
if len(ticks) == 0 or ticks[0] != first_time:
ticks.insert(0, first_time)
if len(ticks) == 1 or ticks[-1] != last_time:
ticks.append(last_time)
plt.xticks(ticks, [str(x) for x in ticks])
return notes_within_range
def determine_bounds(frequency_log_10: list[float]) -> tuple[float, float]:
"""Determine bounds based on 1st and 99th percentile of data"""
lower = numpy.percentile(numpy.array(frequency_log_10), 1)
upper = numpy.percentile(numpy.array(frequency_log_10), 99)
return lower, upper
def set_figure_dimensions(time_range, frequency_log_10_range):
"""Dynamically scale the figure dimensions based on the duration/frequency amplitude of the song"""
height = frequency_log_10_range / 0.06
width = time_range / 2
plt.figure(1).set_figwidth(max(6.4, width))
plt.figure(1).set_figheight(max(4, height))
def create_gaps(pitched_data: PitchedData, step_size: float) -> PitchedData:
"""
Add 'nan' where there are no high confidence frequency values.
This way the graph is only continuous where it should be.
"""
pitched_data_with_gaps = PitchedData([], [], [])
previous_time = 0
for i, time in enumerate(pitched_data.times):
comes_right_after_previous = time - previous_time <= step_size
previous_frequency_is_not_gap = (
len(pitched_data_with_gaps.frequencies) > 0
and str(pitched_data_with_gaps.frequencies[-1]) != "nan"
)
if previous_frequency_is_not_gap and not comes_right_after_previous:
pitched_data_with_gaps.times.append(time)
pitched_data_with_gaps.frequencies.append(float("nan"))
pitched_data_with_gaps.confidence.append(pitched_data.confidence[i])
pitched_data_with_gaps.times.append(time)
pitched_data_with_gaps.frequencies.append(pitched_data.frequencies[i])
pitched_data_with_gaps.confidence.append(pitched_data.confidence[i])
previous_time = time
return pitched_data_with_gaps
def draw_words(transcribed_data, midi_notes):
"""Draw rectangles for each word"""
if transcribed_data is not None:
for i, data in enumerate(transcribed_data):
note_frequency = librosa.note_to_hz(midi_notes[i])
frequency_range = get_frequency_range(midi_notes[i])
half_frequency_range = frequency_range / 2
height = (
numpy.log10([note_frequency + half_frequency_range])[0]
- numpy.log10([note_frequency - half_frequency_range])[0]
)
xy_start_pos = (
data.start,
numpy.log10([note_frequency - half_frequency_range])[0],
)
width = data.end - data.start
rect = Rectangle(
xy_start_pos,
width,
height,
edgecolor="none",
facecolor="red",
alpha=0.5,
)
plt.gca().add_patch(rect)
def snake(s):
"""Turn any string into a snake case string"""
return "_".join(
sub(
"([A-Z][a-z]+)", r" \1", sub("([A-Z]+)", r" \1", s.replace("-", " "))
).split()
).lower()
+26
View File
@@ -0,0 +1,26 @@
import atexit
from functools import reduce
from time import process_time
from modules.console_colors import ULTRASINGER_HEAD
def seconds_to_str(t):
"""Format seconds to string"""
return "%d:%02d:%02d.%03d" % reduce(
lambda ll, b: divmod(ll[0], b) + ll[1:], [(t * 1000,), 1000, 60, 60]
)
def log(s):
"""Log line with optional time elapsed"""
print(f"{ULTRASINGER_HEAD} {seconds_to_str(process_time())} - {s}")
def end_log():
"""Log at program end"""
log("End Program")
atexit.register(end_log)
log("Initialized...")