mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Merge pull request #89 from BWagener/feature/crepe-windows-gpu
Crepe with windows GPU support
This commit is contained in:
20 files changed
+568
-196
No files matched your search
+7
-1
@@ -1,3 +1,5 @@
|
||||
.env
|
||||
.vscode/
|
||||
# virtual environment
|
||||
venv
|
||||
.venv
|
||||
@@ -5,5 +7,9 @@ venv
|
||||
# python
|
||||
**__pycache__
|
||||
|
||||
# test data
|
||||
test_input
|
||||
test_output
|
||||
|
||||
# default output folder
|
||||
output
|
||||
output
|
||||
@@ -112,15 +112,18 @@ _Not all options working now!_
|
||||
|
||||
[transcription]
|
||||
# Default is whisper
|
||||
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
|
||||
English-only model > tiny.en|base.en|small.en|medium.en
|
||||
--align_model Use other languages model for Whisper provided from huggingface.co
|
||||
|
||||
--vosk Needs model
|
||||
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
|
||||
English-only model > tiny.en|base.en|small.en|medium.en
|
||||
--whisper_align_model Use other languages model for Whisper provided from huggingface.co
|
||||
--language Override the language detected by whisper, does not affect transcription but steps after transcription
|
||||
--whisper_batch_size Reduce if low on GPU mem >> ((default) is 16)
|
||||
--whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
|
||||
--vosk Needs model
|
||||
|
||||
[pitcher]
|
||||
# Default is crepe
|
||||
--crepe tiny|small|medium|large|full >> ((default) is full)
|
||||
--crepe tiny|full >> ((default) is full)
|
||||
--crepe_step_size unit is miliseconds >> ((default) is 10)
|
||||
|
||||
[extra]
|
||||
--hyphenation True|False >> ((default) is True)
|
||||
@@ -128,6 +131,7 @@ _Not all options working now!_
|
||||
--disable_karaoke True|False >> ((default) is False)
|
||||
--create_audio_chunks True|False >> ((default) is False)
|
||||
--plot True|False >> ((default) is False)
|
||||
--force_cpu True|False >> ((default) is False)
|
||||
```
|
||||
|
||||
For standard use, you only need to use [opt]. All other options are optional.
|
||||
@@ -205,7 +209,7 @@ starts at the place or is heard.
|
||||
|
||||
### Pitcher
|
||||
|
||||
Pitching is done with the `crepe` model.
|
||||
Pitching is done with the `crepe` model.
|
||||
Also consider that a bigger model is more accurate, but also takes longer to pitch.
|
||||
For just testing you should use `tiny`.
|
||||
If you want solid accurate, then use the `full` model.
|
||||
@@ -252,9 +256,23 @@ pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu11
|
||||
|
||||
When you want to use `conda` instead you need a different installation command. See this [link](https://pytorch.org/get-started/locally/).
|
||||
|
||||
#### Considerations for Windows users
|
||||
|
||||
The pitch tracker used by UltraSinger (crepe), uses TensorFlow as it's backend.
|
||||
TensorFlow dropped GPU support for Windows for versions >2.10 as you can see in this [release note](https://github.com/tensorflow/tensorflow/releases/tag/v2.11.1) and their [installation instructions](https://www.tensorflow.org/install/pip#windows-native).
|
||||
|
||||
For now UltraSinger runs the latest version available that still supports GPUs on windows.
|
||||
|
||||
For running later versions of TensorFlow on windows while still taking advantage of GPU support the suggested solution is:
|
||||
|
||||
* [install WSL2](https://learn.microsoft.com/en-us/windows/wsl/install)
|
||||
* within the Ubuntu WSL2 installation
|
||||
* run `sudo apt update && sudo apt install nvidia-cuda-toolkit`
|
||||
* follow the setup instructions for UltraSinger at the top of this document
|
||||
|
||||
#### Info
|
||||
|
||||
If somthing crash because of low VRAM than use a smaller model.
|
||||
If something crashes because of low VRAM than use a smaller model.
|
||||
Whisper needs more than 8GB VRAM in the `large` model!
|
||||
|
||||
But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`.
|
||||
You can also force cpu usage with the extra option `--force_cpu`.
|
||||
Whitespace-only changes.
@@ -1,3 +1,5 @@
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = [
|
||||
"src",
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Tests for whisper.py"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from src.modules.Audio.denoise import ffmpeg_reduce_noise
|
||||
import pytest
|
||||
|
||||
|
||||
class DenoiseTest(unittest.TestCase):
|
||||
@pytest.mark.skip(reason="Skipping this FUNCTION level test, can be used for manual tests")
|
||||
def test_ffmpeg_reduce_noise(self):
|
||||
# Arrange
|
||||
test_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
root_dir = os.path.abspath(test_dir + "/../../..")
|
||||
test_file_abs_path = os.path.abspath(root_dir + "/test_input/vocals.wav")
|
||||
test_file_name = os.path.basename(test_file_abs_path)
|
||||
test_output = test_dir + "/test_output"
|
||||
|
||||
# Act
|
||||
ffmpeg_reduce_noise(test_file_abs_path, test_output + "/output_" + test_file_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Tests for pitcher.py"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
import src.modules.Pitcher.pitcher as test_subject
|
||||
import pytest
|
||||
from src.modules.plot import plot
|
||||
|
||||
|
||||
class PitcherTest(unittest.TestCase):
|
||||
@pytest.mark.skip(reason="Skipping this FUNCTION level test, can be used for manual tests")
|
||||
def test_get_pitch_with_crepe_file(self):
|
||||
# Arrange
|
||||
test_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
root_dir = os.path.abspath(test_dir + "/../../..")
|
||||
# test_file_abs_path = os.path.abspath(root_dir + "/test_input/audio_denoised.wav")
|
||||
test_file_abs_path = os.path.abspath(root_dir + "/test_input/test_denoised.wav")
|
||||
test_output = root_dir + "/test_output"
|
||||
|
||||
# Act
|
||||
pitched_data = test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', device="cuda")
|
||||
# test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', 'cpu', batch_size=1024)
|
||||
plot(pitched_data, test_output, title="pitching test")
|
||||
print("done")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+23
-21
@@ -1,25 +1,27 @@
|
||||
crepe==0.0.13
|
||||
librosa==0.9.2
|
||||
pretty_midi~=0.2.10
|
||||
pydub==0.25.1
|
||||
Levenshtein~=0.21.0
|
||||
scipy~=1.10.1
|
||||
vosk==0.3.44
|
||||
tensorflow~=2.12.0
|
||||
#tensorflow-directml-plugin~=0.4.0.dev230202
|
||||
#cudatoolkit==11.8.0
|
||||
matplotlib~=3.7.1
|
||||
yt_dlp~=2023.7.6
|
||||
numpy~=1.23.5
|
||||
git+https://github.com/m-bain/whisperx.git
|
||||
PyHyphen~=4.0.3
|
||||
crepe~=0.0.13
|
||||
demucs~=4.0.0
|
||||
tqdm~=4.65.0
|
||||
ffmpeg_python~=0.2.0
|
||||
git+https://github.com/m-bain/whisperx.git
|
||||
langcodes~=3.3.0
|
||||
language-data~=1.1
|
||||
Pillow~=9.5.0
|
||||
isort==5.12
|
||||
black==23.3
|
||||
pylint==2.17
|
||||
librosa~=0.9.2
|
||||
matplotlib~=3.7.2
|
||||
musicbrainzngs~=0.7.1
|
||||
numpy~=1.23.5
|
||||
Pillow~=10.0.0
|
||||
pretty_midi~=0.2.10
|
||||
pydub~=0.25.1
|
||||
PyHyphen~=4.0.3
|
||||
python_Levenshtein~=0.21.1
|
||||
scipy~=1.11.2
|
||||
tensorflow<2.11
|
||||
tqdm~=4.65.2
|
||||
vosk~=0.3.44
|
||||
whisperx~=3.1.1
|
||||
yt_dlp~=2023.7.6
|
||||
|
||||
isort~=5.12
|
||||
black~=23.3
|
||||
pylint~=2.17
|
||||
pytest~=7.3.1
|
||||
musicbrainzngs~=0.7.1
|
||||
protobuf==3.20.*
|
||||
+8
-4
@@ -10,6 +10,8 @@ class Settings:
|
||||
output_file_path = ""
|
||||
mono_audio_path = ""
|
||||
|
||||
language = None
|
||||
|
||||
# Transcribe
|
||||
audio_chunk_folder_name = "audio-chunks"
|
||||
|
||||
@@ -18,15 +20,17 @@ class Settings:
|
||||
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
|
||||
# English-only model tiny.en|base.en|small.en|medium.en
|
||||
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
|
||||
whisper_batch_size = 16 # reduce if low on GPU mem
|
||||
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
|
||||
# Vosk
|
||||
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
|
||||
|
||||
# Pitch
|
||||
crepe_model_capacity = "full" # tiny|small|medium|large|full
|
||||
crepe_step_size = 10
|
||||
crepe_step_size = 10 # in miliseconds
|
||||
|
||||
# Device
|
||||
device = 'cpu' # cpu|cuda
|
||||
force_whisper_cpu = False
|
||||
force_separation_cpu = False
|
||||
pytorch_device = 'cpu' # cpu|cuda
|
||||
tensorflow_device = 'cpu' # cpu|cuda
|
||||
force_cpu = False
|
||||
+64
-39
@@ -4,20 +4,19 @@ import copy
|
||||
import getopt
|
||||
import os
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
import Levenshtein
|
||||
import librosa
|
||||
from tqdm import tqdm
|
||||
|
||||
from modules import os_helper
|
||||
from modules import os_helper, timer
|
||||
from modules.Audio.denoise import ffmpeg_reduce_noise
|
||||
from modules.Audio.separation import separate_audio
|
||||
from modules.Audio.vocal_chunks import (
|
||||
export_chunks_from_transcribed_data,
|
||||
export_chunks_from_ultrastar_data,
|
||||
)
|
||||
from modules.Audio.silence_processing import remove_silence_from_transcribtion_data
|
||||
from modules.Audio.silence_processing import remove_silence_from_transcription_data
|
||||
from modules.csv_handler import export_transcribed_data_to_csv
|
||||
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
|
||||
from modules.Audio.youtube import (
|
||||
@@ -26,7 +25,7 @@ from modules.Audio.youtube import (
|
||||
download_youtube_video,
|
||||
get_youtube_title,
|
||||
)
|
||||
from modules.DeviceDetection.device_detection import get_available_device
|
||||
from modules.DeviceDetection.device_detection import check_gpu_support
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
@@ -41,7 +40,7 @@ from modules.Midi.midi_creator import (
|
||||
most_frequent,
|
||||
)
|
||||
from modules.Pitcher.pitcher import (
|
||||
get_frequency_with_high_confidence,
|
||||
get_frequencies_with_high_confidence,
|
||||
get_pitch_with_crepe_file,
|
||||
)
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
@@ -91,9 +90,12 @@ def pitch_each_chunk_with_crepe(directory: str) -> list[str]:
|
||||
filepath = os.path.join(directory, filename)
|
||||
# todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
filepath, settings.crepe_step_size, settings.crepe_model_capacity
|
||||
filepath,
|
||||
settings.crepe_model_capacity,
|
||||
settings.crepe_step_size,
|
||||
settings.tensorflow_device,
|
||||
)
|
||||
conf_f = get_frequency_with_high_confidence(
|
||||
conf_f = get_frequencies_with_high_confidence(
|
||||
pitched_data.frequencies, pitched_data.confidence
|
||||
)
|
||||
|
||||
@@ -230,7 +232,6 @@ def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData])
|
||||
)
|
||||
return None
|
||||
|
||||
sleep(0.1)
|
||||
hyphenator = create_hyphenator(lang_region)
|
||||
for i in tqdm(enumerate(transcribed_data)):
|
||||
pos = i[0]
|
||||
@@ -309,20 +310,21 @@ def run() -> None:
|
||||
|
||||
# Audio transcription
|
||||
transcribed_data = None
|
||||
language = None
|
||||
language = settings.language
|
||||
if is_audio:
|
||||
language, transcribed_data = transcribe_audio()
|
||||
detected_language, transcribed_data = transcribe_audio()
|
||||
if language is None:
|
||||
language = detected_language
|
||||
|
||||
remove_unecessary_punctuations(transcribed_data)
|
||||
transcribed_data = remove_silence_from_transcribtion_data(
|
||||
transcribed_data = remove_silence_from_transcription_data(
|
||||
settings.mono_audio_path, transcribed_data
|
||||
)
|
||||
|
||||
if settings.hyphenation:
|
||||
hyphen_words = hyphenate_each_word(language, transcribed_data)
|
||||
if hyphen_words is not None:
|
||||
transcribed_data = add_hyphen_to_data(
|
||||
transcribed_data, hyphen_words
|
||||
)
|
||||
transcribed_data = add_hyphen_to_data(transcribed_data, hyphen_words)
|
||||
|
||||
# todo: do we need to correct words?
|
||||
# lyric = 'input/faber_lyric.txt'
|
||||
@@ -345,7 +347,7 @@ def run() -> None:
|
||||
|
||||
# Create plot
|
||||
if settings.create_plot:
|
||||
plot(pitched_data, transcribed_data, midi_notes, song_output)
|
||||
plot(pitched_data, song_output, transcribed_data, midi_notes)
|
||||
|
||||
# Write Ultrastar txt
|
||||
if is_audio:
|
||||
@@ -408,16 +410,21 @@ def get_unused_song_output_dir(path: str) -> str:
|
||||
def transcribe_audio() -> (str, list[TranscribedData]):
|
||||
"""Transcribe audio with AI"""
|
||||
if settings.transcriber == "whisper":
|
||||
device = "cpu" if settings.force_whisper_cpu else settings.device
|
||||
transcribed_data, language = transcribe_with_whisper(
|
||||
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
|
||||
transcribed_data, detected_language = transcribe_with_whisper(
|
||||
settings.mono_audio_path,
|
||||
settings.whisper_model,
|
||||
settings.pytorch_device,
|
||||
settings.whisper_align_model,
|
||||
settings.whisper_batch_size,
|
||||
settings.whisper_compute_type,
|
||||
settings.language,
|
||||
)
|
||||
else: # vosk
|
||||
transcribed_data = transcribe_with_vosk(
|
||||
settings.mono_audio_path, settings.vosk_model_path
|
||||
)
|
||||
# todo: make language selectable
|
||||
language = "en"
|
||||
return language, transcribed_data
|
||||
detected_language = "en"
|
||||
return detected_language, transcribed_data
|
||||
|
||||
|
||||
def separate_vocal_from_audio(
|
||||
@@ -427,16 +434,16 @@ def separate_vocal_from_audio(
|
||||
audio_separation_path = os.path.join(
|
||||
cache_path, "separated", "htdemucs", basename_without_ext
|
||||
)
|
||||
device = "cpu" if settings.force_separation_cpu else settings.device
|
||||
|
||||
if settings.use_separated_vocal or settings.create_karaoke:
|
||||
separate_audio(ultrastar_audio_input_path, cache_path, device)
|
||||
separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device)
|
||||
|
||||
if settings.use_separated_vocal:
|
||||
vocals_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path)
|
||||
input_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
else:
|
||||
convert_audio_to_mono_wav(
|
||||
ultrastar_audio_input_path, settings.mono_audio_path
|
||||
)
|
||||
input_path = ultrastar_audio_input_path
|
||||
|
||||
convert_audio_to_mono_wav(input_path, settings.mono_audio_path)
|
||||
return audio_separation_path
|
||||
|
||||
|
||||
@@ -697,8 +704,9 @@ def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrast
|
||||
# midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name)
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
settings.mono_audio_path,
|
||||
settings.crepe_step_size,
|
||||
settings.crepe_model_capacity,
|
||||
settings.crepe_step_size,
|
||||
settings.tensorflow_device,
|
||||
)
|
||||
if is_audio:
|
||||
start_times = []
|
||||
@@ -777,15 +785,25 @@ def init_settings(argv: list[str]) -> None:
|
||||
elif opt in ("--whisper"):
|
||||
settings.transcriber = "whisper"
|
||||
settings.whisper_model = arg
|
||||
elif opt in ("--align_model"):
|
||||
elif opt in ("--whisper_align_model"):
|
||||
settings.whisper_align_model = arg
|
||||
elif opt in ("--whisper_batch_size"):
|
||||
settings.whisper_batch_size = int(arg)
|
||||
elif opt in ("--whisper_compute_type"):
|
||||
settings.whisper_compute_type = arg
|
||||
elif opt in ("--language"):
|
||||
settings.language = arg
|
||||
elif opt in ("--vosk"):
|
||||
settings.transcriber = "vosk"
|
||||
settings.vosk_model_path = arg
|
||||
elif opt in ("--crepe"):
|
||||
settings.crepe_model_capacity = arg
|
||||
elif opt in ("--crepe_step_size"):
|
||||
settings.crepe_step_size = int(arg)
|
||||
elif opt in ("--plot"):
|
||||
settings.create_plot = arg
|
||||
settings.create_plot = arg in ["True", "true"]
|
||||
elif opt in ("--midi"):
|
||||
settings.create_midi = arg in ["True", "true"]
|
||||
elif opt in ("--hyphenation"):
|
||||
settings.hyphenation = arg
|
||||
elif opt in ("--disable_separation"):
|
||||
@@ -794,17 +812,20 @@ def init_settings(argv: list[str]) -> None:
|
||||
settings.create_karaoke = not arg
|
||||
elif opt in ("--create_audio_chunks"):
|
||||
settings.create_audio_chunks = arg
|
||||
elif opt in ("--force_whisper_cpu"):
|
||||
settings.force_whisper_cpu = arg
|
||||
elif opt in ("--force_separation_cpu"):
|
||||
settings.force_separation_cpu = arg
|
||||
elif opt in ("--force_cpu"):
|
||||
settings.force_cpu = arg
|
||||
if settings.force_cpu:
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
||||
|
||||
if settings.output_file_path == "":
|
||||
if settings.input_file_path.startswith("https:"):
|
||||
dirname = os.getcwd()
|
||||
else:
|
||||
dirname = os.path.dirname(settings.input_file_path)
|
||||
settings.output_file_path = os.path.join(dirname, "output")
|
||||
settings.device = get_available_device()
|
||||
|
||||
if not settings.force_cpu:
|
||||
settings.tensorflow_device, settings.pytorch_device = check_gpu_support()
|
||||
|
||||
|
||||
def arg_options():
|
||||
@@ -813,16 +834,20 @@ def arg_options():
|
||||
"ifile=",
|
||||
"ofile=",
|
||||
"crepe=",
|
||||
"crepe_step_size=",
|
||||
"vosk=",
|
||||
"whisper=",
|
||||
"align_model=",
|
||||
"whisper_align_model=",
|
||||
"whisper_batch_size=",
|
||||
"whisper_compute_type=",
|
||||
"language=",
|
||||
"plot=",
|
||||
"midi=",
|
||||
"hyphenation=",
|
||||
"disable_separation=",
|
||||
"disable_karaoke=",
|
||||
"create_audio_chunks=",
|
||||
"force_whisper_cpu=",
|
||||
"force_separation_cpu="
|
||||
"force_cpu=",
|
||||
]
|
||||
return long, short
|
||||
|
||||
|
||||
@@ -22,9 +22,14 @@ def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Reduce noise from vocal audio with {blue_highlighted('ffmpeg')}."
|
||||
)
|
||||
(
|
||||
ffmpeg.input(input_file_path)
|
||||
.output(output_file, af="afftdn=nr=70:nf=-50:tn=1")
|
||||
.overwrite_output()
|
||||
.run()
|
||||
)
|
||||
try:
|
||||
(
|
||||
ffmpeg.input(input_file_path)
|
||||
.output(output_file, af="afftdn=nr=70:nf=-50:tn=1")
|
||||
.overwrite_output()
|
||||
.run(capture_stdout=True, capture_stderr=True)
|
||||
)
|
||||
except ffmpeg.Error as ffmpeg_exception:
|
||||
print("ffmpeg stdout:", ffmpeg_exception.stdout.decode("utf8"))
|
||||
print("ffmpeg stderr:", ffmpeg_exception.stderr.decode("utf8"))
|
||||
raise ffmpeg_exception
|
||||
@@ -19,7 +19,6 @@ def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None
|
||||
# Model selection?
|
||||
# -n mdx_q
|
||||
# -n htdemucs_ft
|
||||
device = "cpu" if device != "cuda" else "cuda"
|
||||
subprocess.run(
|
||||
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
|
||||
)
|
||||
|
||||
@@ -6,8 +6,8 @@ from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]:
|
||||
"""Remove silence from given transcribtion data"""
|
||||
def remove_silence_from_transcription_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]:
|
||||
"""Remove silence from given transcription data"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data"
|
||||
|
||||
@@ -1,19 +1,37 @@
|
||||
"""Device detection module."""
|
||||
|
||||
import torch
|
||||
import os
|
||||
import tensorflow as tf
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted
|
||||
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted, blue_highlighted
|
||||
|
||||
tensorflow_gpu_supported = False
|
||||
pytorch_gpu_supported = False
|
||||
|
||||
def get_available_device() -> str:
|
||||
"""Get available worker device e.g cuda or cpu"""
|
||||
is_cuda = torch.cuda.is_available()
|
||||
def check_gpu_support() -> tuple[bool, bool]:
|
||||
"""Check worker device (e.g cuda or cpu) supported by tensorflow and pytorch"""
|
||||
|
||||
if not is_cuda:
|
||||
print(f"{ULTRASINGER_HEAD} Checking GPU support for {blue_highlighted('tensorflow')} and {blue_highlighted('pytorch')}.")
|
||||
|
||||
tensorflow_gpu_supported = False
|
||||
pytorch_gpu_supported = False
|
||||
|
||||
gpus = tf.config.list_physical_devices('GPU')
|
||||
if gpus:
|
||||
tensorflow_gpu_supported = True
|
||||
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - using {red_highlighted('cuda')} gpu.")
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}.")
|
||||
if os.name == 'nt':
|
||||
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - versions above 2.10 dropped GPU support for Windows, refer to the readme for possible solutions.")
|
||||
|
||||
pytorch_gpu_supported = torch.cuda.is_available()
|
||||
if not pytorch_gpu_supported:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('pytorch')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}."
|
||||
)
|
||||
return "cpu"
|
||||
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} {blue_highlighted('pytorch')} - using {red_highlighted('cuda')} gpu.")
|
||||
|
||||
return "cuda"
|
||||
return 'cuda' if tensorflow_gpu_supported else 'cpu', 'cuda' if pytorch_gpu_supported else 'cpu'
|
||||
@@ -7,7 +7,7 @@ import librosa
|
||||
import numpy as np
|
||||
import pretty_midi
|
||||
|
||||
from modules.Pitcher.pitcher import get_frequency_with_high_confidence
|
||||
from modules.Pitcher.pitcher import get_frequencies_with_high_confidence
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
@@ -115,7 +115,7 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch
|
||||
freqs = pitched_data.frequencies[start:end]
|
||||
confs = pitched_data.confidence[start:end]
|
||||
|
||||
conf_f = get_frequency_with_high_confidence(freqs, confs)
|
||||
conf_f = get_frequencies_with_high_confidence(freqs, confs)
|
||||
|
||||
notes = convert_frequencies_to_notes(conf_f)
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Pitched data"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class PitchedData:
|
||||
"""Pitched data from crepe"""
|
||||
|
||||
times = list[float]
|
||||
frequencies = list[float]
|
||||
confidence = list[float]
|
||||
times: list[float]
|
||||
frequencies: list[float]
|
||||
confidence: list[float]
|
||||
@@ -3,49 +3,50 @@
|
||||
import crepe
|
||||
from scipy.io import wavfile
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
|
||||
|
||||
def get_pitch_with_crepe_file(filename: str, step_size: int, model_capacity: str) -> PitchedData:
|
||||
def get_pitch_with_crepe_file(
|
||||
filename: str, model_capacity: str, step_size: int = 10, device: str = "cpu"
|
||||
) -> PitchedData:
|
||||
"""Pitch with crepe"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
|
||||
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted(device)} as worker"
|
||||
)
|
||||
# Todo: add GPU support by using torchcrepe
|
||||
sample_rate, audio = wavfile.read(filename)
|
||||
|
||||
pitched_data = PitchedData()
|
||||
(
|
||||
pitched_data.times,
|
||||
pitched_data.frequencies,
|
||||
pitched_data.confidence,
|
||||
activation,
|
||||
) = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
return pitched_data
|
||||
return get_pitch_with_crepe(audio, sample_rate, model_capacity, step_size)
|
||||
|
||||
|
||||
def get_pitch_with_crepe(audio, sample_rate: int, step_size: int, model_capacity: str) -> PitchedData:
|
||||
def get_pitch_with_crepe(
|
||||
audio, sample_rate: int, model_capacity: str, step_size: int = 10
|
||||
) -> PitchedData:
|
||||
"""Pitch with crepe"""
|
||||
pitched_data = PitchedData()
|
||||
(
|
||||
pitched_data.times,
|
||||
pitched_data.frequencies,
|
||||
pitched_data.confidence,
|
||||
activation,
|
||||
) = crepe.predict(
|
||||
times, frequencies, confidence, activation = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
return pitched_data
|
||||
return PitchedData(times, frequencies, confidence)
|
||||
|
||||
|
||||
def get_frequency_with_high_confidence(frequencies: list[float], confidences: list[float], threshold=0.4) -> list[float]:
|
||||
def get_pitched_data_with_high_confidence(
|
||||
pitched_data: PitchedData, threshold=0.4
|
||||
) -> PitchedData:
|
||||
"""Get frequency with high confidence"""
|
||||
new_pitched_data = PitchedData([], [], [])
|
||||
for i, conf in enumerate(pitched_data.confidence):
|
||||
if conf > threshold:
|
||||
new_pitched_data.times.append(pitched_data.times[i])
|
||||
new_pitched_data.frequencies.append(pitched_data.frequencies[i])
|
||||
new_pitched_data.confidence.append(pitched_data.confidence[i])
|
||||
|
||||
return new_pitched_data
|
||||
|
||||
|
||||
def get_frequencies_with_high_confidence(
|
||||
frequencies: list[float], confidences: list[float], threshold=0.4
|
||||
) -> list[float]:
|
||||
"""Get frequency with high confidence"""
|
||||
conf_f = []
|
||||
for i, conf in enumerate(confidences):
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
"""Whisper Speech Recognition Module"""
|
||||
|
||||
import whisperx
|
||||
import sys
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
import whisperx
|
||||
from torch.cuda import OutOfMemoryError
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (
|
||||
list[TranscribedData], str):
|
||||
def transcribe_with_whisper(
|
||||
audio_path: str,
|
||||
model: str,
|
||||
device="cpu",
|
||||
model_name: str = None,
|
||||
batch_size: int = 16,
|
||||
compute_type: str = None,
|
||||
language: str = None,
|
||||
) -> (list[TranscribedData], str):
|
||||
"""Transcribe with whisper"""
|
||||
|
||||
print(
|
||||
@@ -21,31 +26,57 @@ list[TranscribedData], str):
|
||||
if model_name is not None:
|
||||
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
|
||||
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
compute_type = (
|
||||
"float16" if device == "cuda" else "int8"
|
||||
) # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
if compute_type is None:
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
|
||||
loaded_whisper_model = whisperx.load_model(
|
||||
model, device=device, compute_type=compute_type
|
||||
)
|
||||
try:
|
||||
loaded_whisper_model = whisperx.load_model(
|
||||
model, language=language, device=device, compute_type=compute_type
|
||||
)
|
||||
except ValueError as value_error:
|
||||
if (
|
||||
"Requested float16 compute type, but the target device or backend do not support efficient float16 computation."
|
||||
in str(value_error.args[0])
|
||||
):
|
||||
print(value_error)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Your GPU does not support efficient float16 computation; run UltraSinger with '--whisper_compute_type int8'"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
raise value_error
|
||||
except OutOfMemoryError as oom_exception:
|
||||
print(oom_exception)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
|
||||
language = result["language"]
|
||||
result = loaded_whisper_model.transcribe(
|
||||
audio, batch_size=batch_size, language=language
|
||||
)
|
||||
|
||||
detected_language = result["language"]
|
||||
if language is None:
|
||||
language = detected_language
|
||||
|
||||
# load alignment model and metadata
|
||||
try:
|
||||
model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name)
|
||||
model_a, metadata = whisperx.load_align_model(
|
||||
language_code=language, device=device, model_name=model_name
|
||||
)
|
||||
except ValueError as ve:
|
||||
print(f"{red_highlighted(f'{ve}')}"
|
||||
f"\n"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
|
||||
f"Try add it with --align_model [hugingface].")
|
||||
sys.exit(0)
|
||||
print(
|
||||
f"{red_highlighted(f'{ve}')}"
|
||||
f"\n"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
|
||||
f"Try add it with --align_model [hugingface]."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(
|
||||
@@ -59,7 +90,7 @@ list[TranscribedData], str):
|
||||
|
||||
transcribed_data = convert_to_transcribed_data(result_aligned)
|
||||
|
||||
return transcribed_data, language
|
||||
return transcribed_data, detected_language
|
||||
|
||||
|
||||
def convert_to_transcribed_data(result_aligned):
|
||||
|
||||
@@ -88,6 +88,11 @@ def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None:
|
||||
"""Hyphenate word"""
|
||||
|
||||
cleaned_string, removed_indices, removed_symbols = clean_word(word)
|
||||
|
||||
# Hyphenation of word longer than 100 characters throws exception
|
||||
if len(cleaned_string) > 100:
|
||||
return None
|
||||
|
||||
syllabus = hyphenator.syllables(cleaned_string)
|
||||
|
||||
length = len(syllabus)
|
||||
|
||||
+224
-47
@@ -1,71 +1,248 @@
|
||||
"""Plot transcribed data"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from re import sub
|
||||
|
||||
import librosa
|
||||
import numpy
|
||||
from matplotlib import pyplot as plt
|
||||
from matplotlib.patches import Rectangle
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
from modules.Pitcher.pitcher import get_pitched_data_with_high_confidence
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlottedNote:
|
||||
"""Plotted note"""
|
||||
|
||||
note: str
|
||||
frequency: float
|
||||
frequency_log_10: float
|
||||
octave: int
|
||||
|
||||
|
||||
NOTES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]
|
||||
OCTAVES = [0, 1, 2, 3, 4, 5, 6, 7, 8]
|
||||
X_TICK_SIZE = 5
|
||||
|
||||
|
||||
def get_frequency_range(midi_note: str) -> float:
|
||||
"""Get frequency range"""
|
||||
midi = librosa.note_to_midi(midi_note)
|
||||
frequency_range = librosa.midi_to_hz(midi + 1) - librosa.midi_to_hz(midi)
|
||||
return frequency_range
|
||||
|
||||
|
||||
def plot(pitched_data: PitchedData,
|
||||
transcribed_data: list[TranscribedData],
|
||||
midi_notes: list[str],
|
||||
output_path: str
|
||||
) -> None:
|
||||
def create_plot_notes(notes: list[str], octaves: list[int]) -> list[PlottedNote]:
|
||||
"""Create list of notes for plot y axis"""
|
||||
plotted_notes = []
|
||||
for octave in octaves:
|
||||
for note in notes:
|
||||
note_with_octave = note + str(octave)
|
||||
frequency = librosa.note_to_hz(note_with_octave)
|
||||
frequency_log_10 = numpy.log10([frequency])[0]
|
||||
plotted_notes.append(
|
||||
PlottedNote(note_with_octave, frequency, frequency_log_10, octave)
|
||||
)
|
||||
|
||||
return plotted_notes
|
||||
|
||||
|
||||
PLOTTED_NOTES = create_plot_notes(NOTES, OCTAVES)
|
||||
|
||||
|
||||
def plot(
|
||||
pitched_data: PitchedData,
|
||||
output_path: str,
|
||||
transcribed_data: list[TranscribedData] = None,
|
||||
midi_notes: list[str] = None,
|
||||
title: str = None,
|
||||
) -> None:
|
||||
"""Plot transcribed data"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Creating plot")
|
||||
# determine time between to datapoints if there is no gap (this is the step size crepe ran with)
|
||||
step_size = pitched_data.times[1]
|
||||
pitched_data = get_pitched_data_with_high_confidence(pitched_data)
|
||||
|
||||
conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4)
|
||||
if len(pitched_data.frequencies) < 2:
|
||||
print(f"{ULTRASINGER_HEAD} Plot can't be created; too few datapoints")
|
||||
return
|
||||
|
||||
plt.ylim(0, max(conf_f))
|
||||
plt.xlim(0, pitched_data.times[-1])
|
||||
plt.plot(conf_t, conf_f, linewidth=0.1)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Creating plot{': ' + title if title is not None else ''}"
|
||||
)
|
||||
|
||||
for i, data in enumerate(transcribed_data):
|
||||
note_frequency = librosa.note_to_hz(midi_notes[i])
|
||||
frequency_range = get_frequency_range(midi_notes[i])
|
||||
xy_start_pos = (data.start, note_frequency - frequency_range / 2)
|
||||
width = data.end - data.start
|
||||
height = frequency_range
|
||||
rect = Rectangle(
|
||||
xy_start_pos,
|
||||
width,
|
||||
height,
|
||||
edgecolor='none',
|
||||
facecolor='red',
|
||||
alpha=0.5)
|
||||
plt.gca().add_patch(rect)
|
||||
# map each frequency to logarithm with base 10 for a linear progression of values between the musical notes
|
||||
# see http://www.phon.ox.ac.uk/jcoleman/LOGARITH.htm
|
||||
frequencies_log_10 = numpy.log10(pitched_data.frequencies)
|
||||
|
||||
# add 'nan' where there are gaps for frequency values so the graph is only continuous where it should be
|
||||
pitched_data_with_gaps = create_gaps(pitched_data, step_size)
|
||||
frequencies_log_10_with_gaps = numpy.log10(pitched_data_with_gaps.frequencies)
|
||||
|
||||
# dynamically set the minimum and maximum values for x and y axes based on data
|
||||
y_lower_bound, y_upper_bound = determine_bounds(frequencies_log_10)
|
||||
ymin = max(0, y_lower_bound - 0.05)
|
||||
ymax = y_upper_bound + 0.05
|
||||
plt.ylim(ymin, ymax)
|
||||
xmin = min(pitched_data.times)
|
||||
xmax = max(pitched_data.times)
|
||||
plt.xlim(xmin, xmax)
|
||||
|
||||
plt.xlabel("Time (s)")
|
||||
plt.ylabel("Frequency (Hz)")
|
||||
plt.ylim(0, 800)
|
||||
dpi = 4000
|
||||
plt.ylabel("log10 of Frequency (Hz)")
|
||||
|
||||
plt.savefig(os.path.join(output_path, "plot.png"), dpi=dpi)
|
||||
notes_within_range = set_axes_ticks_and_labels(pitched_data.times, ymin, ymax)
|
||||
|
||||
# draw horizontal lines for each note
|
||||
for note in notes_within_range:
|
||||
color = "b"
|
||||
if note.note.startswith("C") and not note.note.startswith("C#"):
|
||||
color = "r"
|
||||
plt.axhline(y=note.frequency_log_10, color=color, linestyle="-", linewidth=0.2)
|
||||
|
||||
# create line and scatter plot of pitched data
|
||||
plt.plot(pitched_data_with_gaps.times, frequencies_log_10_with_gaps, linewidth=0.1)
|
||||
scatter_path_collection = plt.scatter(
|
||||
pitched_data_with_gaps.times,
|
||||
frequencies_log_10_with_gaps,
|
||||
s=5,
|
||||
c=pitched_data_with_gaps.confidence,
|
||||
cmap=plt.colormaps.get_cmap("gray").reversed(),
|
||||
vmin=0,
|
||||
vmax=1,
|
||||
)
|
||||
plt.figure(1).colorbar(scatter_path_collection, label="confidence")
|
||||
|
||||
set_figure_dimensions(xmax - xmin, y_upper_bound - y_lower_bound)
|
||||
|
||||
draw_words(transcribed_data, midi_notes)
|
||||
|
||||
if title is not None:
|
||||
plt.title(label=title)
|
||||
|
||||
plt.figure(1).tight_layout(h_pad=1.4)
|
||||
|
||||
dpi = 200
|
||||
plt.savefig(
|
||||
os.path.join(
|
||||
output_path, f"plot{'' if title is None else '_' + snake(title)}.svg"
|
||||
),
|
||||
dpi=dpi,
|
||||
)
|
||||
plt.clf()
|
||||
plt.cla()
|
||||
|
||||
|
||||
def get_confidence(
|
||||
pitched_data: PitchedData,
|
||||
threshold: float
|
||||
) -> tuple[list[float], list[float], list[float]]:
|
||||
"""Get high confidence data"""
|
||||
# todo: replace get_frequency_with_high_conf from pitcher
|
||||
conf_t = []
|
||||
conf_f = []
|
||||
conf_c = []
|
||||
for i in enumerate(pitched_data.times):
|
||||
pos = i[0]
|
||||
if pitched_data.confidence[pos] > threshold:
|
||||
conf_t.append(pitched_data.times[pos])
|
||||
conf_f.append(pitched_data.frequencies[pos])
|
||||
conf_c.append(pitched_data.confidence[pos])
|
||||
return conf_t, conf_f, conf_c
|
||||
def set_axes_ticks_and_labels(confidence, ymin, ymax):
|
||||
"""Set ticks and their labels for x and y axes"""
|
||||
notes_within_range = [
|
||||
x for x in PLOTTED_NOTES if ymin <= x.frequency_log_10 <= ymax
|
||||
]
|
||||
plt.yticks(
|
||||
[x.frequency_log_10 for x in notes_within_range],
|
||||
[x.note for x in notes_within_range],
|
||||
)
|
||||
|
||||
first_time = min(confidence)
|
||||
min_tick = first_time // X_TICK_SIZE * X_TICK_SIZE + X_TICK_SIZE
|
||||
|
||||
last_time = max(confidence)
|
||||
max_tick = last_time // X_TICK_SIZE * X_TICK_SIZE + 0.1
|
||||
ticks = numpy.arange(min_tick, max_tick, X_TICK_SIZE, dtype=int).tolist()
|
||||
|
||||
if len(ticks) == 0 or ticks[0] != first_time:
|
||||
ticks.insert(0, first_time)
|
||||
|
||||
if len(ticks) == 1 or ticks[-1] != last_time:
|
||||
ticks.append(last_time)
|
||||
|
||||
plt.xticks(ticks, [str(x) for x in ticks])
|
||||
return notes_within_range
|
||||
|
||||
|
||||
def determine_bounds(frequency_log_10: list[float]) -> tuple[float, float]:
|
||||
"""Determine bounds based on 1st and 99th percentile of data"""
|
||||
lower = numpy.percentile(numpy.array(frequency_log_10), 1)
|
||||
upper = numpy.percentile(numpy.array(frequency_log_10), 99)
|
||||
|
||||
return lower, upper
|
||||
|
||||
|
||||
def set_figure_dimensions(time_range, frequency_log_10_range):
|
||||
"""Dynamically scale the figure dimensions based on the duration/frequency amplitude of the song"""
|
||||
height = frequency_log_10_range / 0.06
|
||||
width = time_range / 2
|
||||
|
||||
plt.figure(1).set_figwidth(max(6.4, width))
|
||||
plt.figure(1).set_figheight(max(4, height))
|
||||
|
||||
|
||||
def create_gaps(pitched_data: PitchedData, step_size: float) -> PitchedData:
|
||||
"""
|
||||
Add 'nan' where there are no high confidence frequency values.
|
||||
This way the graph is only continuous where it should be.
|
||||
|
||||
"""
|
||||
pitched_data_with_gaps = PitchedData([], [], [])
|
||||
|
||||
previous_time = 0
|
||||
for i, time in enumerate(pitched_data.times):
|
||||
comes_right_after_previous = time - previous_time <= step_size
|
||||
previous_frequency_is_not_gap = (
|
||||
len(pitched_data_with_gaps.frequencies) > 0
|
||||
and str(pitched_data_with_gaps.frequencies[-1]) != "nan"
|
||||
)
|
||||
if previous_frequency_is_not_gap and not comes_right_after_previous:
|
||||
pitched_data_with_gaps.times.append(time)
|
||||
pitched_data_with_gaps.frequencies.append(float("nan"))
|
||||
pitched_data_with_gaps.confidence.append(pitched_data.confidence[i])
|
||||
|
||||
pitched_data_with_gaps.times.append(time)
|
||||
pitched_data_with_gaps.frequencies.append(pitched_data.frequencies[i])
|
||||
pitched_data_with_gaps.confidence.append(pitched_data.confidence[i])
|
||||
|
||||
previous_time = time
|
||||
|
||||
return pitched_data_with_gaps
|
||||
|
||||
|
||||
def draw_words(transcribed_data, midi_notes):
|
||||
"""Draw rectangles for each word"""
|
||||
if transcribed_data is not None:
|
||||
for i, data in enumerate(transcribed_data):
|
||||
note_frequency = librosa.note_to_hz(midi_notes[i])
|
||||
frequency_range = get_frequency_range(midi_notes[i])
|
||||
|
||||
half_frequency_range = frequency_range / 2
|
||||
height = (
|
||||
numpy.log10([note_frequency + half_frequency_range])[0]
|
||||
- numpy.log10([note_frequency - half_frequency_range])[0]
|
||||
)
|
||||
|
||||
xy_start_pos = (
|
||||
data.start,
|
||||
numpy.log10([note_frequency - half_frequency_range])[0],
|
||||
)
|
||||
width = data.end - data.start
|
||||
rect = Rectangle(
|
||||
xy_start_pos,
|
||||
width,
|
||||
height,
|
||||
edgecolor="none",
|
||||
facecolor="red",
|
||||
alpha=0.5,
|
||||
)
|
||||
plt.gca().add_patch(rect)
|
||||
|
||||
|
||||
def snake(s):
|
||||
"""Turn any string into a snake case string"""
|
||||
return "_".join(
|
||||
sub(
|
||||
"([A-Z][a-z]+)", r" \1", sub("([A-Z]+)", r" \1", s.replace("-", " "))
|
||||
).split()
|
||||
).lower()
|
||||
@@ -0,0 +1,26 @@
|
||||
import atexit
|
||||
from functools import reduce
|
||||
from time import process_time
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def seconds_to_str(t):
|
||||
"""Format seconds to string"""
|
||||
return "%d:%02d:%02d.%03d" % reduce(
|
||||
lambda ll, b: divmod(ll[0], b) + ll[1:], [(t * 1000,), 1000, 60, 60]
|
||||
)
|
||||
|
||||
|
||||
def log(s):
|
||||
"""Log line with optional time elapsed"""
|
||||
print(f"{ULTRASINGER_HEAD} {seconds_to_str(process_time())} - {s}")
|
||||
|
||||
|
||||
def end_log():
|
||||
"""Log at program end"""
|
||||
log("End Program")
|
||||
|
||||
|
||||
atexit.register(end_log)
|
||||
log("Initialized...")
|
||||
Reference in new issue
Block a user