mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
- merge
This commit is contained in:
commit
65d2dda38f
40 files changed
+1712
-1063
No files matched your search
+9
-9
@@ -6,23 +6,23 @@ class Settings:
|
||||
use_separated_vocal = True
|
||||
create_karaoke = True
|
||||
|
||||
input_file_path = ''
|
||||
output_file_path = ''
|
||||
mono_audio_path = ''
|
||||
input_file_path = ""
|
||||
output_file_path = ""
|
||||
mono_audio_path = ""
|
||||
|
||||
# Transcribe
|
||||
audio_chunk_folder_name = 'audio-chunks'
|
||||
audio_chunk_folder_name = "audio-chunks"
|
||||
|
||||
# Whisper
|
||||
transcriber = 'whisper' # whisper|vosk
|
||||
whisper_model = 'large-v2' # Multilingual model tiny|base|small|medium|large-v1|large-v2
|
||||
# English-only model tiny.en|base.en|small.en|medium.en
|
||||
transcriber = "whisper" # whisper|vosk
|
||||
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
|
||||
# English-only model tiny.en|base.en|small.en|medium.en
|
||||
|
||||
# Vosk
|
||||
vosk_model_path = '' # "models/vosk-model-small-en-us-0.15"
|
||||
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
|
||||
|
||||
# Pitch
|
||||
crepe_model_capacity = 'full' # tiny|small|medium|large|full
|
||||
crepe_model_capacity = "full" # tiny|small|medium|large|full
|
||||
crepe_step_size = 10
|
||||
|
||||
# Device
|
||||
|
||||
+449
-181
@@ -1,73 +1,117 @@
|
||||
import getopt
|
||||
"""UltraSinger uses AI to automatically create UltraStar song files"""
|
||||
|
||||
import copy
|
||||
import getopt
|
||||
import os
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
import Levenshtein
|
||||
import librosa
|
||||
|
||||
from moduls import os_helper
|
||||
from moduls.Audio.vocal_chunks import export_chunks_from_ultrastar_data, convert_audio_to_mono_wav, \
|
||||
export_chunks_from_transcribed_data, remove_silence_from_transcribtion_data, convert_wav_to_mp3, \
|
||||
export_transcribed_data_to_csv
|
||||
from moduls.Audio.youtube import download_youtube_video, download_youtube_audio, get_youtube_title, download_youtube_thumbnail
|
||||
from moduls.Audio.separation import separate_audio
|
||||
from moduls.Audio.denoise import ffmpeg_reduce_noise
|
||||
from moduls.Midi import midi_creator
|
||||
from moduls.Midi.midi_creator import convert_frequencies_to_notes, most_frequent, create_midi_notes_from_pitched_data
|
||||
from moduls.Pitcher.pitcher import get_frequency_with_high_confidence, get_pitch_with_crepe_file
|
||||
from moduls.Ultrastar import ultrastar_parser, ultrastar_converter, ultrastar_writer, ultrastar_score_calculator
|
||||
from moduls.Speech_Recognition.Vosk import transcribe_with_vosk
|
||||
from moduls.Speech_Recognition.hyphenation import hyphenation, language_check
|
||||
from moduls.Speech_Recognition.Whisper import transcribe_with_whisper
|
||||
from moduls.Log import PRINT_ULTRASTAR, print_red_highlighted_text, print_blue_highlighted_text, \
|
||||
print_gold_highlighted_text, \
|
||||
print_light_blue_highlighted_text
|
||||
from matplotlib import pyplot as plt
|
||||
from moduls.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
from Settings import Settings
|
||||
from tqdm import tqdm
|
||||
from moduls.DeviceDetection.device_detection import get_available_device
|
||||
from time import sleep
|
||||
|
||||
from modules import os_helper
|
||||
from modules.Audio.denoise import ffmpeg_reduce_noise
|
||||
from modules.Audio.separation import separate_audio
|
||||
from modules.Audio.vocal_chunks import (
|
||||
export_chunks_from_transcribed_data,
|
||||
export_chunks_from_ultrastar_data,
|
||||
)
|
||||
from modules.Audio.silence_processing import remove_silence_from_transcribtion_data
|
||||
from modules.csv_export import export_transcribed_data_to_csv
|
||||
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
|
||||
from modules.Audio.youtube import (
|
||||
download_youtube_audio,
|
||||
download_youtube_thumbnail,
|
||||
download_youtube_video,
|
||||
get_youtube_title,
|
||||
)
|
||||
from modules.DeviceDetection.device_detection import get_available_device
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
gold_highlighted,
|
||||
light_blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.Midi import midi_creator
|
||||
from modules.Midi.midi_creator import (
|
||||
convert_frequencies_to_notes,
|
||||
create_midi_notes_from_pitched_data,
|
||||
most_frequent,
|
||||
)
|
||||
from modules.Pitcher.pitcher import (
|
||||
get_frequency_with_high_confidence,
|
||||
get_pitch_with_crepe_file,
|
||||
)
|
||||
from modules.Speech_Recognition.hyphenation import hyphenation, language_check
|
||||
from modules.Speech_Recognition.Vosk import transcribe_with_vosk
|
||||
from modules.Speech_Recognition.Whisper import transcribe_with_whisper
|
||||
from modules.Ultrastar import (
|
||||
ultrastar_converter,
|
||||
ultrastar_parser,
|
||||
ultrastar_score_calculator,
|
||||
ultrastar_writer,
|
||||
)
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
from Settings import Settings
|
||||
|
||||
settings = Settings()
|
||||
|
||||
|
||||
def get_confidence(pitched_data, threshold):
|
||||
"""Docstring"""
|
||||
# todo: replace get_frequency_with_high_conf from pitcher
|
||||
conf_t = []
|
||||
conf_f = []
|
||||
conf_c = []
|
||||
for i in range(len(pitched_data.times)):
|
||||
for i in enumerate(pitched_data.times):
|
||||
pos = i[0]
|
||||
if pitched_data.confidence[i] > threshold:
|
||||
conf_t.append(pitched_data.times[i])
|
||||
conf_f.append(pitched_data.frequencies[i])
|
||||
conf_c.append(pitched_data.confidence[i])
|
||||
conf_t.append(pitched_data.times[pos])
|
||||
conf_f.append(pitched_data.frequencies[pos])
|
||||
conf_c.append(pitched_data.confidence[pos])
|
||||
return conf_t, conf_f, conf_c
|
||||
|
||||
|
||||
def convert_ultrastar_note_numbers(midi_notes):
|
||||
print(f"{PRINT_ULTRASTAR} Creating Ultrastar notes from midi data")
|
||||
"""Docstring"""
|
||||
print(f"{ULTRASINGER_HEAD} Creating Ultrastar notes from midi data")
|
||||
|
||||
ultrastar_note_numbers = []
|
||||
for i in range(len(midi_notes)):
|
||||
note_number_librosa = librosa.note_to_midi(midi_notes[i])
|
||||
pitch = ultrastar_converter.midi_note_to_ultrastar_note(note_number_librosa)
|
||||
for i, note in enumerate(midi_notes):
|
||||
note_number_librosa = librosa.note_to_midi(note)
|
||||
pitch = ultrastar_converter.midi_note_to_ultrastar_note(
|
||||
note_number_librosa
|
||||
)
|
||||
ultrastar_note_numbers.append(pitch)
|
||||
# todo: Progress?
|
||||
# print("Note: " + midi_notes[i] + " midi_note: " + str(note_number_librosa) + ' pitch: ' + str(pitch))
|
||||
# print(
|
||||
# f"Note: {midi_notes[i]} midi_note: {str(note_number_librosa)} pitch: {str(pitch)}"
|
||||
# )
|
||||
return ultrastar_note_numbers
|
||||
|
||||
|
||||
def pitch_each_chunk_with_crepe(directory):
|
||||
print(f"{PRINT_ULTRASTAR} Pitching each chunk with {print_blue_highlighted_text('crepe')}")
|
||||
"""Docstring"""
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}"
|
||||
)
|
||||
|
||||
midi_notes = []
|
||||
for filename in sorted([f for f in os.listdir(directory) if f.endswith('.wav')],
|
||||
key=lambda x: int(x.split("_")[1])):
|
||||
for filename in sorted(
|
||||
[f for f in os.listdir(directory) if f.endswith(".wav")],
|
||||
key=lambda x: int(x.split("_")[1]),
|
||||
):
|
||||
filepath = os.path.join(directory, filename)
|
||||
# todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes
|
||||
pitched_data = get_pitch_with_crepe_file(filepath, settings.crepe_step_size, settings.crepe_model_capacity)
|
||||
conf_f = get_frequency_with_high_confidence(pitched_data.frequencies, pitched_data.confidence)
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
filepath, settings.crepe_step_size, settings.crepe_model_capacity
|
||||
)
|
||||
conf_f = get_frequency_with_high_confidence(
|
||||
pitched_data.frequencies, pitched_data.confidence
|
||||
)
|
||||
|
||||
notes = convert_frequencies_to_notes(conf_f)
|
||||
note = most_frequent(notes)[0][0]
|
||||
@@ -80,58 +124,69 @@ def pitch_each_chunk_with_crepe(directory):
|
||||
|
||||
|
||||
def add_hyphen_to_data(transcribed_data, hyphen_words):
|
||||
data = []
|
||||
"""Docstring"""
|
||||
new_data = []
|
||||
|
||||
for i in range(len(transcribed_data)):
|
||||
for i, data in enumerate(transcribed_data):
|
||||
if not hyphen_words[i]:
|
||||
data.append(transcribed_data[i])
|
||||
new_data.append(data)
|
||||
else:
|
||||
chunk_duration = transcribed_data[i].end - transcribed_data[i].start
|
||||
chunk_duration = data.end - data.start
|
||||
chunk_duration = chunk_duration / (len(hyphen_words[i]))
|
||||
|
||||
next_start = transcribed_data[i].start
|
||||
for j in range(len(hyphen_words[i])):
|
||||
dup = copy.copy(transcribed_data[i])
|
||||
next_start = data.start
|
||||
for j, hyphens in enumerate(hyphen_words[i]):
|
||||
dup = copy.copy(data)
|
||||
dup.start = next_start
|
||||
next_start = transcribed_data[i].end - chunk_duration * (len(hyphen_words[i]) - 1 - j)
|
||||
next_start = data.end - chunk_duration * (
|
||||
len(hyphens) - 1 - j
|
||||
)
|
||||
dup.end = next_start
|
||||
dup.word = hyphen_words[i][j]
|
||||
dup.is_hyphen = True
|
||||
data.append(dup)
|
||||
new_data.append(dup)
|
||||
|
||||
return data
|
||||
return new_data
|
||||
|
||||
|
||||
def get_bpm_from_data(data, sr):
|
||||
onset_env = librosa.onset.onset_strength(y=data, sr=sr)
|
||||
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sr)
|
||||
def get_bpm_from_data(data, sampling_rate):
|
||||
"""Docstring"""
|
||||
onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate)
|
||||
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} BPM is {print_blue_highlighted_text(str(round(wav_tempo[0], 2)))}")
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}"
|
||||
)
|
||||
return wav_tempo[0]
|
||||
|
||||
|
||||
def get_bpm_from_file(wav_file):
|
||||
data, sr = librosa.load(wav_file, sr=None)
|
||||
return get_bpm_from_data(data, sr)
|
||||
"""Docstring"""
|
||||
data, sampling_rate = librosa.load(wav_file, sr=None)
|
||||
return get_bpm_from_data(data, sampling_rate)
|
||||
|
||||
|
||||
def correct_words(recognized_words, word_list_file):
|
||||
with open(word_list_file, 'r') as f:
|
||||
text = f.read()
|
||||
"""Docstring"""
|
||||
with open(word_list_file, "r", encoding="utf-8") as file:
|
||||
text = file.read()
|
||||
word_list = text.split()
|
||||
|
||||
for i, rec_word in enumerate(recognized_words):
|
||||
if rec_word.word in word_list:
|
||||
continue
|
||||
else:
|
||||
closest_word = min(word_list, key=lambda x: Levenshtein.distance(rec_word.word, x))
|
||||
print(recognized_words[i].word + " - " + closest_word)
|
||||
recognized_words[i].word = closest_word
|
||||
|
||||
closest_word = min(
|
||||
word_list, key=lambda x: Levenshtein.distance(rec_word.word, x)
|
||||
)
|
||||
print(recognized_words[i].word + " - " + closest_word)
|
||||
recognized_words[i].word = closest_word
|
||||
return recognized_words
|
||||
|
||||
|
||||
def print_help():
|
||||
m = '''
|
||||
"""Docstring"""
|
||||
help_string = """
|
||||
UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]
|
||||
|
||||
[opt]
|
||||
@@ -167,72 +222,120 @@ def print_help():
|
||||
|
||||
[pitcher]
|
||||
--crepe (default) tiny|small|medium|large|full
|
||||
'''
|
||||
print(m)
|
||||
"""
|
||||
print(help_string)
|
||||
|
||||
|
||||
def plot(input_file, vosk_transcribed_data, midi_notes):
|
||||
pitched_data = get_pitch_with_crepe_file(input_file, settings.crepe_step_size, settings.crepe_model_capacity)
|
||||
t, f, c = get_confidence(pitched_data, 0.4)
|
||||
"""Docstring"""
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
input_file, settings.crepe_step_size, settings.crepe_model_capacity
|
||||
)
|
||||
conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4)
|
||||
|
||||
plt.ylim(0, 600)
|
||||
plt.xlim(0, 50)
|
||||
plt.plot(t, f, linewidth=0.1)
|
||||
plt.plot(conf_t, conf_f, linewidth=0.1)
|
||||
plt.savefig(os.path.join("test", "crepe_0.4.png"))
|
||||
|
||||
for i in range(len(vosk_transcribed_data)):
|
||||
nf = librosa.note_to_hz(midi_notes[i])
|
||||
plt.plot([vosk_transcribed_data[i].start, vosk_transcribed_data[i].end], [nf, nf], linewidth=1, alpha=0.5)
|
||||
for i, data in enumerate(vosk_transcribed_data):
|
||||
note_frequency = librosa.note_to_hz(midi_notes[i])
|
||||
plt.plot(
|
||||
[data.start, data.end],
|
||||
[note_frequency, note_frequency],
|
||||
linewidth=1,
|
||||
alpha=0.5,
|
||||
)
|
||||
plt.savefig(os.path.join("test", "pit.png"), dpi=2000)
|
||||
|
||||
|
||||
def remove_unecessary_punctuations(transcribed_data):
|
||||
"""Docstring"""
|
||||
punctuation = ".,"
|
||||
for i in range(len(transcribed_data)):
|
||||
transcribed_data[i].word = transcribed_data[i].word.translate({ord(i): None for i in punctuation})
|
||||
for i, data in enumerate(transcribed_data):
|
||||
data.word = data.word.translate(
|
||||
{ord(i): None for i in punctuation}
|
||||
)
|
||||
|
||||
|
||||
def hyphenate_each_word(language, transcribed_data):
|
||||
"""Docstring"""
|
||||
hyphenated_word = []
|
||||
lang_region = language_check(language)
|
||||
if lang_region is None:
|
||||
print(f"{PRINT_ULTRASTAR} {print_red_highlighted_text('Error in hyphenation for language ')} {print_blue_highlighted_text(language)} {print_red_highlighted_text(', maybe you want to disable it?')}")
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error in hyphenation for language ')} {blue_highlighted(language)} {red_highlighted(', maybe you want to disable it?')}"
|
||||
)
|
||||
return None
|
||||
|
||||
sleep(.1)
|
||||
for i in tqdm(range(len(transcribed_data))):
|
||||
hyphenated_word.append(hyphenation(transcribed_data[i].word, lang_region))
|
||||
sleep(0.1)
|
||||
for i, data in tqdm(enumerate(transcribed_data)):
|
||||
hyphenated_word.append(
|
||||
hyphenation(data.word, lang_region)
|
||||
)
|
||||
return hyphenated_word
|
||||
|
||||
|
||||
def print_support():
|
||||
"""Docstring"""
|
||||
print()
|
||||
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('Do you like UltraSinger? And want it to be even better? Then help with your')} {print_light_blue_highlighted_text('support')}{print_gold_highlighted_text('!')}")
|
||||
print(f"{PRINT_ULTRASTAR} See project page -> https://github.com/rakuri255/UltraSinger")
|
||||
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('This will help alot to keep this project alive and improved.')}")
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('Do you like UltraSinger? And want it to be even better? Then help with your')} {light_blue_highlighted('support')}{gold_highlighted('!')}"
|
||||
)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} See project page -> https://github.com/rakuri255/UltraSinger"
|
||||
)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('This will help alot to keep this project alive and improved.')}"
|
||||
)
|
||||
|
||||
|
||||
def run():
|
||||
isAudio = ".txt" not in settings.input_file_path
|
||||
"""Docstring"""
|
||||
is_audio = ".txt" not in settings.input_file_path
|
||||
ultrastar_class = None
|
||||
real_bpm = None
|
||||
|
||||
if not isAudio: # Parse Ultrastar txt
|
||||
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('re-pitch mode')}")
|
||||
basename_without_ext, real_bpm, song_output, ultrastar_audio_input_path, ultrastar_class = parse_ultrastar_txt()
|
||||
elif settings.input_file_path.startswith('https:'): # Youtube
|
||||
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('full automatic mode')}")
|
||||
basename_without_ext, song_output, ultrastar_audio_input_path = download_from_youtube()
|
||||
if not is_audio: # Parse Ultrastar txt
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('re-pitch mode')}"
|
||||
)
|
||||
(
|
||||
basename_without_ext,
|
||||
real_bpm,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_class,
|
||||
) = parse_ultrastar_txt()
|
||||
elif settings.input_file_path.startswith("https:"): # Youtube
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
|
||||
)
|
||||
(
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
) = download_from_youtube()
|
||||
else: # Audio File
|
||||
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('full automatic mode')}")
|
||||
basename_without_ext, song_output, ultrastar_audio_input_path = setup_audio_input_file()
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
|
||||
)
|
||||
(
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
) = setup_audio_input_file()
|
||||
|
||||
cache_path = os.path.join(song_output, 'cache')
|
||||
settings.mono_audio_path = os.path.join(cache_path, basename_without_ext + '.wav')
|
||||
cache_path = os.path.join(song_output, "cache")
|
||||
settings.mono_audio_path = os.path.join(
|
||||
cache_path, basename_without_ext + ".wav"
|
||||
)
|
||||
os_helper.create_folder(cache_path)
|
||||
|
||||
# Separate vocal from audio
|
||||
audio_separation_path = separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_input_path)
|
||||
audio_separation_path = separate_vocal_from_audio(
|
||||
basename_without_ext, cache_path, ultrastar_audio_input_path
|
||||
)
|
||||
|
||||
# Denoise vocal audio
|
||||
denoise_vocal_audio(basename_without_ext, cache_path)
|
||||
@@ -240,15 +343,19 @@ def run():
|
||||
# Audio transcription
|
||||
transcribed_data = None
|
||||
language = None
|
||||
if isAudio:
|
||||
if is_audio:
|
||||
language, transcribed_data = transcribe_audio(transcribed_data)
|
||||
remove_unecessary_punctuations(transcribed_data)
|
||||
transcribed_data = remove_silence_from_transcribtion_data(settings.mono_audio_path, transcribed_data)
|
||||
transcribed_data = remove_silence_from_transcribtion_data(
|
||||
settings.mono_audio_path, transcribed_data
|
||||
)
|
||||
|
||||
if settings.hyphenation:
|
||||
hyphen_words = hyphenate_each_word(language, transcribed_data)
|
||||
if hyphen_words is not None:
|
||||
transcribed_data = add_hyphen_to_data(transcribed_data, hyphen_words)
|
||||
transcribed_data = add_hyphen_to_data(
|
||||
transcribed_data, hyphen_words
|
||||
)
|
||||
|
||||
# todo: do we need to correct words?
|
||||
# lyric = 'input/faber_lyric.txt'
|
||||
@@ -256,41 +363,59 @@ def run():
|
||||
|
||||
# Create audio chunks
|
||||
if settings.create_audio_chunks:
|
||||
create_audio_chunks(cache_path, isAudio, transcribed_data, ultrastar_audio_input_path, ultrastar_class)
|
||||
create_audio_chunks(
|
||||
cache_path,
|
||||
is_audio,
|
||||
transcribed_data,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_class,
|
||||
)
|
||||
|
||||
# Pitch the audio
|
||||
midi_notes, pitched_data, ultrastar_note_numbers = pitch_audio(isAudio, transcribed_data, ultrastar_class)
|
||||
midi_notes, pitched_data, ultrastar_note_numbers = pitch_audio(
|
||||
is_audio, transcribed_data, ultrastar_class
|
||||
)
|
||||
|
||||
# Create plot
|
||||
if isAudio and settings.create_plot:
|
||||
if is_audio and settings.create_plot:
|
||||
plot(ultrastar_audio_input_path, transcribed_data, midi_notes)
|
||||
|
||||
# Write Ultrastar txt
|
||||
if isAudio:
|
||||
real_bpm, ultrastar_file_output = create_ultrastar_txt_from_automation(audio_separation_path,
|
||||
basename_without_ext,
|
||||
song_output, transcribed_data,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_note_numbers, language)
|
||||
if is_audio:
|
||||
real_bpm, ultrastar_file_output = create_ultrastar_txt_from_automation(
|
||||
audio_separation_path,
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
transcribed_data,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_note_numbers,
|
||||
language,
|
||||
)
|
||||
else:
|
||||
ultrastar_file_output = create_ultrastar_txt_from_ultrastar_data(song_output, ultrastar_class,
|
||||
ultrastar_note_numbers)
|
||||
ultrastar_file_output = create_ultrastar_txt_from_ultrastar_data(
|
||||
song_output, ultrastar_class, ultrastar_note_numbers
|
||||
)
|
||||
|
||||
# Calc Points
|
||||
ultrastar_class, simple_score, accurate_score = calculate_score_points(isAudio, pitched_data, ultrastar_class, ultrastar_file_output)
|
||||
ultrastar_class, simple_score, accurate_score = calculate_score_points(
|
||||
is_audio, pitched_data, ultrastar_class, ultrastar_file_output
|
||||
)
|
||||
|
||||
# Add calculated score to Ultrastar txt
|
||||
ultrastar_writer.add_score_to_ultrastar_txt(ultrastar_file_output, simple_score)
|
||||
ultrastar_writer.add_score_to_ultrastar_txt(
|
||||
ultrastar_file_output, simple_score
|
||||
)
|
||||
|
||||
# Midi
|
||||
if settings.create_midi:
|
||||
create_midi_file(isAudio, real_bpm, song_output, ultrastar_class)
|
||||
create_midi_file(is_audio, real_bpm, song_output, ultrastar_class)
|
||||
|
||||
# Print Support
|
||||
print_support()
|
||||
|
||||
|
||||
def get_unused_song_output_dir(path):
|
||||
"""Docstring"""
|
||||
# check if dir exists and add (i) if it does
|
||||
i = 1
|
||||
if os_helper.check_if_folder_exists(path):
|
||||
@@ -302,25 +427,35 @@ def get_unused_song_output_dir(path):
|
||||
path = path.replace(f"({i - 1})", f"({i})")
|
||||
i += 1
|
||||
if i > 999:
|
||||
print(f"{PRINT_ULTRASTAR} {print_red_highlighted_text('Error: Could not create output folder! (999) is the maximum number of tries.')}")
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error: Could not create output folder! (999) is the maximum number of tries.')}"
|
||||
)
|
||||
sys.exit(1)
|
||||
return path
|
||||
|
||||
|
||||
def transcribe_audio(transcribed_data):
|
||||
"""Docstring"""
|
||||
if settings.transcriber == "whisper":
|
||||
device = "cpu" if settings.force_whisper_cpu else settings.device
|
||||
transcribed_data, language = transcribe_with_whisper(settings.mono_audio_path, settings.whisper_model,
|
||||
device)
|
||||
transcribed_data, language = transcribe_with_whisper(
|
||||
settings.mono_audio_path, settings.whisper_model, device)
|
||||
else: # vosk
|
||||
transcribed_data = transcribe_with_vosk(settings.mono_audio_path, settings.vosk_model_path)
|
||||
transcribed_data = transcribe_with_vosk(
|
||||
settings.mono_audio_path, settings.vosk_model_path
|
||||
)
|
||||
# todo: make language selectable
|
||||
language = 'en'
|
||||
language = "en"
|
||||
return language, transcribed_data
|
||||
|
||||
|
||||
def separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_input_path):
|
||||
audio_separation_path = os.path.join(cache_path, "separated", "htdemucs", basename_without_ext)
|
||||
def separate_vocal_from_audio(
|
||||
basename_without_ext, cache_path, ultrastar_audio_input_path
|
||||
):
|
||||
"""Docstring"""
|
||||
audio_separation_path = os.path.join(
|
||||
cache_path, "separated", "htdemucs", basename_without_ext
|
||||
)
|
||||
device = "cpu" if settings.force_separation_cpu else settings.device
|
||||
if settings.use_separated_vocal or settings.create_karaoke:
|
||||
separate_audio(ultrastar_audio_input_path, cache_path, device)
|
||||
@@ -328,36 +463,85 @@ def separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_
|
||||
vocals_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path)
|
||||
else:
|
||||
convert_audio_to_mono_wav(ultrastar_audio_input_path, settings.mono_audio_path)
|
||||
convert_audio_to_mono_wav(
|
||||
ultrastar_audio_input_path, settings.mono_audio_path
|
||||
)
|
||||
return audio_separation_path
|
||||
|
||||
|
||||
def calculate_score_points(isAudio, pitched_data, ultrastar_class, ultrastar_file_output):
|
||||
if isAudio:
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output)
|
||||
simple_score, accurate_score = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
|
||||
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
|
||||
def calculate_score_points(
|
||||
is_audio, pitched_data, ultrastar_class, ultrastar_file_output
|
||||
):
|
||||
"""Docstring"""
|
||||
if is_audio:
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
ultrastar_file_output
|
||||
)
|
||||
(
|
||||
simple_score,
|
||||
accurate_score,
|
||||
) = ultrastar_score_calculator.calculate_score(
|
||||
pitched_data, ultrastar_class
|
||||
)
|
||||
ultrastar_score_calculator.print_score_calculation(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
else:
|
||||
print(f"{PRINT_ULTRASTAR} {print_blue_highlighted_text('Score of original Ultrastar txt')}")
|
||||
simple_score, accurate_score = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
|
||||
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
|
||||
print(f"{PRINT_ULTRASTAR} {print_blue_highlighted_text('Score of re-pitched Ultrastar txt')}")
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output)
|
||||
simple_score, accurate_score = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
|
||||
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of original Ultrastar txt')}"
|
||||
)
|
||||
(
|
||||
simple_score,
|
||||
accurate_score,
|
||||
) = ultrastar_score_calculator.calculate_score(
|
||||
pitched_data, ultrastar_class
|
||||
)
|
||||
ultrastar_score_calculator.print_score_calculation(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of re-pitched Ultrastar txt')}"
|
||||
)
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
ultrastar_file_output
|
||||
)
|
||||
(
|
||||
simple_score,
|
||||
accurate_score,
|
||||
) = ultrastar_score_calculator.calculate_score(
|
||||
pitched_data, ultrastar_class
|
||||
)
|
||||
ultrastar_score_calculator.print_score_calculation(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
return ultrastar_class, simple_score, accurate_score
|
||||
|
||||
|
||||
def create_ultrastar_txt_from_ultrastar_data(song_output, ultrastar_class, ultrastar_note_numbers):
|
||||
output_repitched_ultrastar = os.path.join(song_output, ultrastar_class.title + '.txt')
|
||||
ultrastar_writer.create_repitched_txt_from_ultrastar_data(settings.input_file_path, ultrastar_note_numbers,
|
||||
output_repitched_ultrastar)
|
||||
def create_ultrastar_txt_from_ultrastar_data(
|
||||
song_output, ultrastar_class, ultrastar_note_numbers
|
||||
):
|
||||
"""Docstring"""
|
||||
output_repitched_ultrastar = os.path.join(
|
||||
song_output, ultrastar_class.title + ".txt"
|
||||
)
|
||||
ultrastar_writer.create_repitched_txt_from_ultrastar_data(
|
||||
settings.input_file_path,
|
||||
ultrastar_note_numbers,
|
||||
output_repitched_ultrastar,
|
||||
)
|
||||
return output_repitched_ultrastar
|
||||
|
||||
|
||||
def create_ultrastar_txt_from_automation(audio_separation_path, basename_without_ext, song_output,
|
||||
transcribed_data, ultrastar_audio_input_path, ultrastar_note_numbers,
|
||||
language):
|
||||
def create_ultrastar_txt_from_automation(
|
||||
audio_separation_path,
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
transcribed_data,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_note_numbers,
|
||||
language,
|
||||
):
|
||||
"""Docstring"""
|
||||
ultrastar_header = UltrastarTxt()
|
||||
ultrastar_header.title = basename_without_ext
|
||||
ultrastar_header.artist = basename_without_ext
|
||||
@@ -365,13 +549,23 @@ def create_ultrastar_txt_from_automation(audio_separation_path, basename_without
|
||||
ultrastar_header.video = basename_without_ext + ".mp4"
|
||||
ultrastar_header.language = language
|
||||
cover = basename_without_ext + " [CO].jpg"
|
||||
ultrastar_header.cover = cover if os_helper.check_file_exists(os.path.join(song_output, cover)) else None
|
||||
ultrastar_header.cover = (
|
||||
cover
|
||||
if os_helper.check_file_exists(os.path.join(song_output, cover))
|
||||
else None
|
||||
)
|
||||
|
||||
real_bpm = get_bpm_from_file(ultrastar_audio_input_path)
|
||||
ultrastar_file_output = os.path.join(song_output, basename_without_ext + '.txt')
|
||||
ultrastar_writer.create_ultrastar_txt_from_automation(transcribed_data, ultrastar_note_numbers,
|
||||
ultrastar_file_output,
|
||||
ultrastar_header, real_bpm)
|
||||
ultrastar_file_output = os.path.join(
|
||||
song_output, basename_without_ext + ".txt"
|
||||
)
|
||||
ultrastar_writer.create_ultrastar_txt_from_automation(
|
||||
transcribed_data,
|
||||
ultrastar_note_numbers,
|
||||
ultrastar_file_output,
|
||||
ultrastar_header,
|
||||
real_bpm,
|
||||
)
|
||||
if settings.create_karaoke:
|
||||
no_vocals_path = os.path.join(audio_separation_path, "no_vocals.wav")
|
||||
title = basename_without_ext + " [Karaoke]"
|
||||
@@ -381,13 +575,18 @@ def create_ultrastar_txt_from_automation(audio_separation_path, basename_without
|
||||
karaoke_audio_output_path = karaoke_output_path + ".mp3"
|
||||
convert_wav_to_mp3(no_vocals_path, karaoke_audio_output_path)
|
||||
karaoke_txt_output_path = karaoke_output_path + ".txt"
|
||||
ultrastar_writer.create_ultrastar_txt_from_automation(transcribed_data, ultrastar_note_numbers,
|
||||
karaoke_txt_output_path,
|
||||
ultrastar_header, real_bpm)
|
||||
ultrastar_writer.create_ultrastar_txt_from_automation(
|
||||
transcribed_data,
|
||||
ultrastar_note_numbers,
|
||||
karaoke_txt_output_path,
|
||||
ultrastar_header,
|
||||
real_bpm,
|
||||
)
|
||||
return real_bpm, ultrastar_file_output
|
||||
|
||||
|
||||
def setup_audio_input_file():
|
||||
"""Docstring"""
|
||||
basename = os.path.basename(settings.input_file_path)
|
||||
basename_without_ext = os.path.splitext(basename)[0]
|
||||
song_output = os.path.join(settings.output_file_path, basename_without_ext)
|
||||
@@ -402,6 +601,7 @@ FILENAME_REPLACEMENTS = (('?:"', ""), ("<", "("), (">", ")"), ("/\\|*", "-"))
|
||||
|
||||
|
||||
def sanitize_filename(fname: str) -> str:
|
||||
"""Docstring"""
|
||||
for old, new in FILENAME_REPLACEMENTS:
|
||||
for char in old:
|
||||
fname = fname.replace(char, new)
|
||||
@@ -411,82 +611,138 @@ def sanitize_filename(fname: str) -> str:
|
||||
|
||||
|
||||
def download_from_youtube():
|
||||
"""Docstring"""
|
||||
title = get_youtube_title(settings.input_file_path)
|
||||
basename_without_ext = sanitize_filename(title)
|
||||
basename = basename_without_ext + '.mp3'
|
||||
basename = basename_without_ext + ".mp3"
|
||||
song_output = os.path.join(settings.output_file_path, basename_without_ext)
|
||||
song_output = get_unused_song_output_dir(song_output)
|
||||
os_helper.create_folder(song_output)
|
||||
download_youtube_audio(settings.input_file_path, basename_without_ext, song_output)
|
||||
download_youtube_video(settings.input_file_path, basename_without_ext, song_output)
|
||||
download_youtube_thumbnail(settings.input_file_path, basename_without_ext, song_output)
|
||||
download_youtube_audio(
|
||||
settings.input_file_path, basename_without_ext, song_output
|
||||
)
|
||||
download_youtube_video(
|
||||
settings.input_file_path, basename_without_ext, song_output
|
||||
)
|
||||
download_youtube_thumbnail(
|
||||
settings.input_file_path, basename_without_ext, song_output
|
||||
)
|
||||
ultrastar_audio_input_path = os.path.join(song_output, basename)
|
||||
return basename_without_ext, song_output, ultrastar_audio_input_path
|
||||
|
||||
|
||||
def parse_ultrastar_txt():
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(settings.input_file_path)
|
||||
real_bpm = ultrastar_converter.ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(',', '.')))
|
||||
"""Docstring"""
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
settings.input_file_path
|
||||
)
|
||||
real_bpm = ultrastar_converter.ultrastar_bpm_to_real_bpm(
|
||||
float(ultrastar_class.bpm.replace(",", "."))
|
||||
)
|
||||
ultrastar_mp3_name = ultrastar_class.mp3
|
||||
basename_without_ext = os.path.splitext(ultrastar_mp3_name)[0]
|
||||
dirname = os.path.dirname(settings.input_file_path)
|
||||
ultrastar_audio_input_path = os.path.join(dirname, ultrastar_mp3_name)
|
||||
song_output = os.path.join(settings.output_file_path, ultrastar_class.artist + ' - ' + ultrastar_class.title)
|
||||
song_output = os.path.join(
|
||||
settings.output_file_path,
|
||||
ultrastar_class.artist + " - " + ultrastar_class.title,
|
||||
)
|
||||
song_output = get_unused_song_output_dir(song_output)
|
||||
os_helper.create_folder(song_output)
|
||||
return basename_without_ext, real_bpm, song_output, ultrastar_audio_input_path, ultrastar_class
|
||||
return (
|
||||
basename_without_ext,
|
||||
real_bpm,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_class,
|
||||
)
|
||||
|
||||
|
||||
def create_midi_file(isAudio, real_bpm, song_output, ultrastar_class):
|
||||
print(f"{PRINT_ULTRASTAR} Creating Midi with {print_blue_highlighted_text('pretty_midi')}")
|
||||
if isAudio:
|
||||
voice_instrument = [midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)]
|
||||
midi_output = os.path.join(song_output, ultrastar_class.title + '.mid')
|
||||
midi_creator.instruments_to_midi(voice_instrument, real_bpm, midi_output)
|
||||
def create_midi_file(is_audio, real_bpm, song_output, ultrastar_class):
|
||||
"""Docstring"""
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}"
|
||||
)
|
||||
if is_audio:
|
||||
voice_instrument = [
|
||||
midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)
|
||||
]
|
||||
midi_output = os.path.join(song_output, ultrastar_class.title + ".mid")
|
||||
midi_creator.instruments_to_midi(
|
||||
voice_instrument, real_bpm, midi_output
|
||||
)
|
||||
else:
|
||||
voice_instrument = [midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)]
|
||||
midi_output = os.path.join(song_output, ultrastar_class.title + '.mid')
|
||||
midi_creator.instruments_to_midi(voice_instrument, real_bpm, midi_output)
|
||||
voice_instrument = [
|
||||
midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)
|
||||
]
|
||||
midi_output = os.path.join(song_output, ultrastar_class.title + ".mid")
|
||||
midi_creator.instruments_to_midi(
|
||||
voice_instrument, real_bpm, midi_output
|
||||
)
|
||||
|
||||
|
||||
def pitch_audio(isAudio, transcribed_data, ultrastar_class):
|
||||
def pitch_audio(is_audio, transcribed_data, ultrastar_class):
|
||||
"""Docstring"""
|
||||
# todo: chunk pitching as option?
|
||||
# midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name)
|
||||
pitched_data = get_pitch_with_crepe_file(settings.mono_audio_path, settings.crepe_step_size,
|
||||
settings.crepe_model_capacity)
|
||||
if isAudio:
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
settings.mono_audio_path,
|
||||
settings.crepe_step_size,
|
||||
settings.crepe_model_capacity,
|
||||
)
|
||||
if is_audio:
|
||||
start_times = []
|
||||
end_times = []
|
||||
for i in range(len(transcribed_data)):
|
||||
start_times.append(transcribed_data[i].start)
|
||||
end_times.append(transcribed_data[i].end)
|
||||
midi_notes = create_midi_notes_from_pitched_data(start_times, end_times, pitched_data)
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_times.append(data.start)
|
||||
end_times.append(data.end)
|
||||
midi_notes = create_midi_notes_from_pitched_data(
|
||||
start_times, end_times, pitched_data
|
||||
)
|
||||
|
||||
else:
|
||||
midi_notes = create_midi_notes_from_pitched_data(ultrastar_class.startTimes, ultrastar_class.endTimes,
|
||||
pitched_data)
|
||||
midi_notes = create_midi_notes_from_pitched_data(
|
||||
ultrastar_class.startTimes, ultrastar_class.endTimes, pitched_data
|
||||
)
|
||||
ultrastar_note_numbers = convert_ultrastar_note_numbers(midi_notes)
|
||||
return midi_notes, pitched_data, ultrastar_note_numbers
|
||||
|
||||
|
||||
def create_audio_chunks(cache_path, isAudio, transcribed_data, ultrastar_audio_input_path, ultrastar_class):
|
||||
audio_chunks_path = os.path.join(cache_path, settings.audio_chunk_folder_name)
|
||||
def create_audio_chunks(
|
||||
cache_path,
|
||||
is_audio,
|
||||
transcribed_data,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_class,
|
||||
):
|
||||
"""Docstring"""
|
||||
audio_chunks_path = os.path.join(
|
||||
cache_path, settings.audio_chunk_folder_name
|
||||
)
|
||||
os_helper.create_folder(audio_chunks_path)
|
||||
if isAudio: # and csv
|
||||
if is_audio: # and csv
|
||||
csv_filename = os.path.join(audio_chunks_path, "_chunks.csv")
|
||||
export_chunks_from_transcribed_data(settings.mono_audio_path, transcribed_data, audio_chunks_path)
|
||||
export_chunks_from_transcribed_data(
|
||||
settings.mono_audio_path, transcribed_data, audio_chunks_path
|
||||
)
|
||||
export_transcribed_data_to_csv(transcribed_data, csv_filename)
|
||||
else:
|
||||
export_chunks_from_ultrastar_data(ultrastar_audio_input_path, ultrastar_class, audio_chunks_path)
|
||||
export_chunks_from_ultrastar_data(
|
||||
ultrastar_audio_input_path, ultrastar_class, audio_chunks_path
|
||||
)
|
||||
|
||||
|
||||
def denoise_vocal_audio(basename_without_ext, cache_path):
|
||||
denoised_path = os.path.join(cache_path, basename_without_ext + '_denoised.wav')
|
||||
"""Docstring"""
|
||||
denoised_path = os.path.join(
|
||||
cache_path, basename_without_ext + "_denoised.wav"
|
||||
)
|
||||
ffmpeg_reduce_noise(settings.mono_audio_path, denoised_path)
|
||||
settings.mono_audio_path = denoised_path
|
||||
|
||||
|
||||
def main(argv):
|
||||
"""Docstring"""
|
||||
init_settings(argv)
|
||||
run()
|
||||
# todo: cleanup
|
||||
@@ -494,15 +750,27 @@ def main(argv):
|
||||
|
||||
|
||||
def init_settings(argv):
|
||||
"""Docstring"""
|
||||
short = "hi:o:amv:"
|
||||
long = ["ifile=", "ofile=", "crepe=", "vosk=", "whisper=", "hyphenation=", "disable_separation=",
|
||||
"disable_karaoke=", "create_audio_chunks=", "force_whisper_cpu=", "force_separation_cpu="]
|
||||
long = [
|
||||
"ifile=",
|
||||
"ofile=",
|
||||
"crepe=",
|
||||
"vosk=",
|
||||
"whisper=",
|
||||
"hyphenation=",
|
||||
"disable_separation=",
|
||||
"disable_karaoke=",
|
||||
"create_audio_chunks=",
|
||||
"force_whisper_cpu=",
|
||||
"force_separation_cpu="
|
||||
]
|
||||
opts, args = getopt.getopt(argv, short, long)
|
||||
if len(opts) == 0:
|
||||
print_help()
|
||||
sys.exit()
|
||||
for opt, arg in opts:
|
||||
if opt == '-h':
|
||||
if opt == "-h":
|
||||
print_help()
|
||||
sys.exit()
|
||||
elif opt in ("-i", "--ifile"):
|
||||
@@ -513,7 +781,7 @@ def init_settings(argv):
|
||||
settings.transcriber = "whisper"
|
||||
settings.whisper_model = arg
|
||||
elif opt in ("--vosk"):
|
||||
settings.transcriber = 'vosk'
|
||||
settings.transcriber = "vosk"
|
||||
settings.vosk_model_path = arg
|
||||
elif opt in ("--crepe"):
|
||||
settings.crepe_model_capacity = arg
|
||||
@@ -529,12 +797,12 @@ def init_settings(argv):
|
||||
settings.force_whisper_cpu = arg
|
||||
elif opt in ("--force_separation_cpu"):
|
||||
settings.force_separation_cpu = arg
|
||||
if settings.output_file_path == '':
|
||||
if settings.input_file_path.startswith('https:'):
|
||||
if settings.output_file_path == "":
|
||||
if settings.input_file_path.startswith("https:"):
|
||||
dirname = os.getcwd()
|
||||
else:
|
||||
dirname = os.path.dirname(settings.input_file_path)
|
||||
settings.output_file_path = os.path.join(dirname, 'output')
|
||||
settings.output_file_path = os.path.join(dirname, "output")
|
||||
settings.device = get_available_device()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Convert audio to other formats"""
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def convert_audio_to_mono_wav(input_file: str, output_file: str) -> None:
|
||||
"""Convert audio to mono wav"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting audio for AI")
|
||||
|
||||
if ".mp3" in input_file:
|
||||
sound = AudioSegment.from_mp3(input_file)
|
||||
elif ".wav" in input_file:
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
else:
|
||||
raise ValueError("data format not supported")
|
||||
|
||||
sound = sound.set_channels(1)
|
||||
sound.export(output_file, format="wav")
|
||||
|
||||
|
||||
def convert_wav_to_mp3(input_file: str, output_file: str) -> None:
|
||||
"""Convert wav to mp3"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting wav to mp3")
|
||||
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
sound.export(output_file, format="mp3")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Reduce noise from audio"""
|
||||
|
||||
import ffmpeg
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
|
||||
|
||||
|
||||
def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None:
|
||||
"""Reduce noise from vocal audio with ffmpeg."""
|
||||
|
||||
# Denoise audio samples with FFT.
|
||||
# A description of the accepted parameters follows.
|
||||
|
||||
# noise_reduction, nr
|
||||
# Set the noise reduction in dB, allowed range is 0.01 to 97. Default value is 12 dB.
|
||||
# noise_floor, nf
|
||||
# Set the noise floor in dB, allowed range is -80 to -20. Default value is -50 dB.
|
||||
# track_noise, tn
|
||||
# Enable noise floor tracking. By default is disabled.
|
||||
# With this enabled, noise floor is automatically adjusted.
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Reduce noise from vocal audio with {blue_highlighted('ffmpeg')}."
|
||||
)
|
||||
(
|
||||
ffmpeg.input(input_file_path)
|
||||
.output(output_file, af="afftdn=nr=70:nf=-50:tn=1")
|
||||
.overwrite_output()
|
||||
.run()
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Separate vocals from audio"""
|
||||
|
||||
import subprocess
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.os_helper import current_executor_path, move, path_join
|
||||
|
||||
|
||||
def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None:
|
||||
"""Separate vocals from audio with demucs."""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker."
|
||||
)
|
||||
# Model selection?
|
||||
# -n mdx_q
|
||||
# -n htdemucs_ft
|
||||
device = "cpu" if device != "cuda" else "cuda"
|
||||
subprocess.run(
|
||||
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
|
||||
)
|
||||
separated_folder = path_join(current_executor_path(), "separated")
|
||||
move(separated_folder, output_file)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Silence processing module"""
|
||||
|
||||
import librosa
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: []) -> []:
|
||||
"""Remove silence from given transcribtion data"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data"
|
||||
)
|
||||
|
||||
audio, sample_rate = librosa.load(audio_path, sr=None)
|
||||
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_time = data.start
|
||||
end_time = data.end
|
||||
start_sample = int(start_time * sample_rate)
|
||||
end_sample = int(end_time * sample_rate)
|
||||
chunk = audio[start_sample:end_sample]
|
||||
|
||||
# todo: why 5 works good? It should be 40db ?!?
|
||||
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
|
||||
silence_threshold = 5
|
||||
onsets = librosa.effects.split(
|
||||
chunk, top_db=silence_threshold, frame_length=2048, hop_length=100
|
||||
)
|
||||
|
||||
# Get the duration of the first and last silent intervals
|
||||
if len(onsets) > 0:
|
||||
first_silence = onsets[0][0]
|
||||
last_silence = len(chunk) - onsets[-1][1]
|
||||
|
||||
first_silence_duration = librosa.samples_to_time(
|
||||
first_silence, sr=sample_rate
|
||||
)
|
||||
last_silence_duration = librosa.samples_to_time(
|
||||
last_silence, sr=sample_rate
|
||||
)
|
||||
else:
|
||||
first_silence_duration = 0
|
||||
last_silence_duration = 0
|
||||
|
||||
data.start = (
|
||||
data.start + first_silence_duration
|
||||
)
|
||||
data.end = (
|
||||
data.end - last_silence_duration
|
||||
)
|
||||
|
||||
return transcribed_data
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Vocal chunks module."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import wave
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.os_helper import create_folder
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
)
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
|
||||
|
||||
class AudioManipulation:
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
def export_chunks_from_transcribed_data(
|
||||
audio_filename: str, transcribed_data: [], output_folder_name: str
|
||||
) -> None:
|
||||
"""Export transcribed_data as vocal chunks wav files"""
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Export transcribed data as vocal chunks wav files"
|
||||
)
|
||||
|
||||
wave_file = wave.open(audio_filename, "rb")
|
||||
sample_rate, n_channels = wave_file.getparams()[2], wave_file.getparams()[0]
|
||||
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_byte = int(data.start * sample_rate * n_channels)
|
||||
end_byte = int(data.end * sample_rate * n_channels)
|
||||
|
||||
chunk = get_chunk(end_byte, start_byte, wave_file)
|
||||
export_chunk_to_wav_file(
|
||||
chunk, output_folder_name, i, data.word, wave_file
|
||||
)
|
||||
|
||||
wave_file.close()
|
||||
|
||||
|
||||
def export_chunks_from_ultrastar_data(
|
||||
audio_filename: str, ultrastar_data: UltrastarTxt, folder_name: str
|
||||
) -> None:
|
||||
"""Export ultrastar data as vocal chunks wav files"""
|
||||
print(f"{ULTRASINGER_HEAD} Export Ultrastar data as vocal chunks wav files")
|
||||
|
||||
create_folder(folder_name)
|
||||
|
||||
wave_file = wave.open(audio_filename, "rb")
|
||||
sample_rate, n_channels = wave_file.getparams()[2], wave_file.getparams()[0]
|
||||
|
||||
for i, word in enumerate(ultrastar_data.words):
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_data, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_data, i)
|
||||
|
||||
start_byte = int(start_time * sample_rate * n_channels)
|
||||
end_byte = int(end_time * sample_rate * n_channels)
|
||||
|
||||
chunk = get_chunk(end_byte, start_byte, wave_file)
|
||||
export_chunk_to_wav_file(
|
||||
chunk, folder_name, i, word, wave_file
|
||||
)
|
||||
|
||||
|
||||
def export_chunk_to_wav_file(chunk, folder_name: str, i: int, word: str, wave_file) -> None:
|
||||
"""Export vocal chunks to wav file"""
|
||||
|
||||
clean_word = re.sub("[^A-Za-z0-9]+", "", word)
|
||||
# todo: Progress?
|
||||
# print(f"{str(i)} {clean_word}")
|
||||
with wave.open(
|
||||
os.path.join(folder_name, f"chunk_{i}_{clean_word}.wav"), "wb"
|
||||
) as chunk_file:
|
||||
chunk_file.setparams(wave_file.getparams())
|
||||
chunk_file.writeframes(chunk)
|
||||
|
||||
|
||||
def get_chunk(end_byte: int, start_byte: int, wave_file):
|
||||
"""
|
||||
Gets the chunk from wave file.
|
||||
Returns chunk as n frames of audio, as a bytes object.
|
||||
"""
|
||||
|
||||
# todo: get out of position error message
|
||||
wave_file.setpos(start_byte) # ({:.2f})
|
||||
chunk = wave_file.readframes(end_byte - start_byte)
|
||||
return chunk
|
||||
@@ -0,0 +1,83 @@
|
||||
"""YouTube Downloader"""
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
import yt_dlp
|
||||
from PIL import Image
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def get_youtube_title(url: str) -> str:
|
||||
"""Get the title of the YouTube video"""
|
||||
|
||||
ydl_opts = {}
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
result = ydl.extract_info(
|
||||
url, download=False # We just want to extract the info
|
||||
)
|
||||
|
||||
return result["title"]
|
||||
|
||||
|
||||
def download_youtube_audio(url: str, clear_filename: str, output_path: str):
|
||||
"""Download audio from YouTube"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Audio")
|
||||
ydl_opts = {
|
||||
"format": "bestaudio/best",
|
||||
"outtmpl": output_path + "/" + clear_filename,
|
||||
"postprocessors": [
|
||||
{"key": "FFmpegExtractAudio", "preferredcodec": "mp3"}
|
||||
],
|
||||
}
|
||||
|
||||
start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def download_youtube_thumbnail(url: str, clear_filename: str, output_path: str):
|
||||
"""Download thumbnail from YouTube"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Downloading thumbnail")
|
||||
ydl_opts = {
|
||||
"skip_download": True,
|
||||
"writethumbnail": True,
|
||||
}
|
||||
|
||||
download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path)
|
||||
|
||||
|
||||
def download_and_convert_thumbnail(ydl_opts, url: str, clear_filename: str, output_path: str) -> None:
|
||||
"""Download and convert thumbnail from YouTube"""
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
info_dict = ydl.extract_info(url, download=False)
|
||||
thumbnail_url = info_dict.get("thumbnail")
|
||||
if thumbnail_url:
|
||||
response = ydl.urlopen(thumbnail_url)
|
||||
image_data = response.read()
|
||||
image = Image.open(io.BytesIO(image_data))
|
||||
image.save(
|
||||
os.path.join(output_path, clear_filename + " [CO].jpg"), "JPEG"
|
||||
)
|
||||
|
||||
|
||||
def download_youtube_video(url: str, clear_filename: str, output_path: str) -> None:
|
||||
"""Download video from YouTube"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Video")
|
||||
ydl_opts = {
|
||||
"format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4",
|
||||
"outtmpl": output_path + "/" + clear_filename + ".mp4",
|
||||
}
|
||||
start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def start_download(ydl_opts, url: str) -> None:
|
||||
"""Start the download the ydl_opts"""
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
errors = ydl.download(url)
|
||||
if errors:
|
||||
raise Exception("Download failed with error: " + str(errors))
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Docstring"""
|
||||
|
||||
import torch
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted
|
||||
|
||||
|
||||
def get_available_device():
|
||||
"""Docstring"""
|
||||
isCuda = torch.cuda.is_available()
|
||||
|
||||
if not isCuda:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
|
||||
)
|
||||
|
||||
return "cpu"
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
|
||||
|
||||
return "cuda"
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Docstring"""
|
||||
|
||||
import math
|
||||
from collections import Counter
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import pretty_midi
|
||||
|
||||
from modules.Pitcher.pitcher import get_frequency_with_high_confidence
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
ultrastar_note_to_midi_note,
|
||||
)
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
red_highlighted,
|
||||
)
|
||||
|
||||
|
||||
def convert_ultrastar_to_midi_instrument(ultrastar_class):
|
||||
"""Docstring"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi instrument from Ultrastar txt")
|
||||
|
||||
instrument = pretty_midi.Instrument(program=0)
|
||||
velocity = 100
|
||||
|
||||
for i in enumerate(ultrastar_class.words):
|
||||
pos = i[0]
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, pos)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, pos)
|
||||
pitch = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[pos]))
|
||||
|
||||
note = pretty_midi.Note(velocity, pitch, start_time, end_time)
|
||||
instrument.notes.append(note)
|
||||
|
||||
return instrument
|
||||
|
||||
|
||||
def instruments_to_midi(instruments, bpm, midi_output):
|
||||
"""Docstring"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi file -> {midi_output}")
|
||||
|
||||
midi_data = pretty_midi.PrettyMIDI(initial_tempo=bpm)
|
||||
for instrument in instruments:
|
||||
midi_data.instruments.append(instrument)
|
||||
midi_data.write(midi_output)
|
||||
|
||||
|
||||
class MidiCreator:
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
def convert_frequencies_to_notes(frequency):
|
||||
"""Docstring"""
|
||||
notes = []
|
||||
for freq in frequency:
|
||||
notes.append(librosa.hz_to_note(float(freq)))
|
||||
return notes
|
||||
|
||||
|
||||
def most_frequent(array):
|
||||
"""Docstring"""
|
||||
return Counter(array).most_common(1)
|
||||
|
||||
|
||||
def find_nearest_index(array, value):
|
||||
"""Docstring"""
|
||||
idx = np.searchsorted(array, value, side="left")
|
||||
if idx > 0 and (
|
||||
idx == len(array)
|
||||
or math.fabs(value - array[idx - 1]) < math.fabs(value - array[idx])
|
||||
):
|
||||
return idx - 1
|
||||
|
||||
return idx
|
||||
|
||||
|
||||
def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
|
||||
"""Docstring"""
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi notes from pitched data")
|
||||
|
||||
midi_notes = []
|
||||
|
||||
for i in enumerate(start_times):
|
||||
pos = i[0]
|
||||
start_time = start_times[pos]
|
||||
end_time = end_times[pos]
|
||||
|
||||
note = create_midi_note_from_pitched_data(
|
||||
start_time, end_time, pitched_data
|
||||
)
|
||||
|
||||
midi_notes.append(note)
|
||||
# todo: Progress?
|
||||
# print(filename + " f: " + str(mean))
|
||||
return midi_notes
|
||||
|
||||
|
||||
def create_midi_note_from_pitched_data(start_time, end_time, pitched_data):
|
||||
"""Docstring"""
|
||||
|
||||
start = find_nearest_index(pitched_data.times, start_time)
|
||||
end = find_nearest_index(pitched_data.times, end_time)
|
||||
|
||||
# fixme: why is end smaller as start?
|
||||
if end < start:
|
||||
print(f"{ULTRASINGER_HEAD} {red_highlighted('Error: end time is smaller as start time. The note is C0 now ...')}")
|
||||
return "C0"
|
||||
|
||||
if start == end:
|
||||
freqs = [pitched_data.frequencies[start]]
|
||||
confs = [pitched_data.confidence[start]]
|
||||
else:
|
||||
freqs = pitched_data.frequencies[start:end]
|
||||
confs = pitched_data.confidence[start:end]
|
||||
|
||||
conf_f = get_frequency_with_high_confidence(freqs, confs)
|
||||
|
||||
notes = convert_frequencies_to_notes(conf_f)
|
||||
|
||||
note = most_frequent(notes)[0][0]
|
||||
|
||||
return note
|
||||
@@ -1,4 +1,9 @@
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
class PitchedData:
|
||||
"""Docstring"""
|
||||
|
||||
times = []
|
||||
frequencies = []
|
||||
confidence = []
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Docstring"""
|
||||
|
||||
import csv
|
||||
|
||||
import crepe
|
||||
from scipy.io import wavfile
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
|
||||
|
||||
def get_pitch_with_crepe_file(filename, step_size, model_capacity):
|
||||
"""Docstring"""
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
|
||||
)
|
||||
# Todo: add GPU support by using torchcrepe
|
||||
sample_rate, audio = wavfile.read(filename)
|
||||
|
||||
pitched_data = PitchedData()
|
||||
(
|
||||
pitched_data.times,
|
||||
pitched_data.frequencies,
|
||||
pitched_data.confidence,
|
||||
activation,
|
||||
) = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
return pitched_data
|
||||
|
||||
|
||||
def get_pitch_with_crepe(audio, sample_rate, step_size, model_capacity):
|
||||
"""Docstring"""
|
||||
pitched_data = PitchedData()
|
||||
(
|
||||
pitched_data.times,
|
||||
pitched_data.frequencies,
|
||||
pitched_data.confidence,
|
||||
activation,
|
||||
) = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
return pitched_data
|
||||
|
||||
|
||||
def write_lists_to_csv(times, frequencies, confidences, filename):
|
||||
"""Docstring"""
|
||||
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["time", "frequency", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i in enumerate(times):
|
||||
pos = i[0]
|
||||
writer.writerow([times[pos], frequencies[pos], confidences[pos]])
|
||||
|
||||
|
||||
def read_data_from_csv(filename):
|
||||
"""Docstring"""
|
||||
csv_data = []
|
||||
with open(filename, "r", encoding="utf-8") as csv_file:
|
||||
csv_reader = csv.reader(csv_file)
|
||||
for line in csv_reader:
|
||||
csv_data.append(line)
|
||||
headless_data = csv_data[1:]
|
||||
return headless_data
|
||||
|
||||
|
||||
def get_frequency_with_high_confidence(freqs, confs, threshold=0.4):
|
||||
"""Docstring"""
|
||||
conf_f = []
|
||||
for i, conf in enumerate(confs):
|
||||
if conf > threshold:
|
||||
conf_f.append(freqs[i])
|
||||
if not conf_f:
|
||||
conf_f = freqs
|
||||
return conf_f
|
||||
|
||||
|
||||
class Pitcher:
|
||||
"""Docstring"""
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
class TranscribedData:
|
||||
"""Transcribed data from json file"""
|
||||
|
||||
def __init__(self, transcribed_json):
|
||||
# Vosk = conf, Whisper = confidence
|
||||
self.conf = transcribed_json.get(
|
||||
"conf", transcribed_json.get("confidence", None)
|
||||
)
|
||||
# Vosk = word, Whisper = text
|
||||
self.word = transcribed_json.get(
|
||||
"word", transcribed_json.get("text", None)
|
||||
)
|
||||
self.end = transcribed_json["end"]
|
||||
self.start = transcribed_json["start"]
|
||||
self.is_hyphen = None
|
||||
@@ -1,21 +1,26 @@
|
||||
import wave
|
||||
"""Docstring"""
|
||||
|
||||
import json
|
||||
import wave
|
||||
|
||||
from vosk import Model, KaldiRecognizer
|
||||
from moduls.Speech_Recognition.TranscribedData import TranscribedData
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
from vosk import KaldiRecognizer, Model
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
# todo: Rename to Transcoder?
|
||||
|
||||
|
||||
def transcribe_with_vosk(audio_filename, model_path):
|
||||
"""Docstring"""
|
||||
# Code from here: https://towardsdatascience.com/speech-recognition-with-timestamps-934ede4234b2
|
||||
print(f"{PRINT_ULTRASTAR} Transcribing {audio_filename} with vosk and model {model_path}")
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Transcribing {audio_filename} with vosk and model {model_path}"
|
||||
)
|
||||
|
||||
model = Model(model_path)
|
||||
wf = wave.open(audio_filename, "rb")
|
||||
recognizer = KaldiRecognizer(model, wf.getframerate())
|
||||
wave_file = wave.open(audio_filename, "rb")
|
||||
recognizer = KaldiRecognizer(model, wave_file.getframerate())
|
||||
|
||||
recognizer.SetWords(True)
|
||||
|
||||
@@ -23,13 +28,13 @@ def transcribe_with_vosk(audio_filename, model_path):
|
||||
results = []
|
||||
# recognize speech using vosk model
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
data = wave_file.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if recognizer.AcceptWaveform(data):
|
||||
part_result = json.loads(recognizer.Result())
|
||||
results.append(part_result)
|
||||
wf.close() # close audiofile
|
||||
wave_file.close() # close audiofile
|
||||
part_result = json.loads(recognizer.FinalResult())
|
||||
results.append(part_result)
|
||||
|
||||
@@ -41,9 +46,9 @@ def transcribe_with_vosk(audio_filename, model_path):
|
||||
# and it returns an empty dictionary
|
||||
# {'text': ''}
|
||||
continue
|
||||
for obj in sentence['result']:
|
||||
for obj in sentence["result"]:
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
vtd.word = vtd.word + ' '
|
||||
vtd.word = vtd.word + " "
|
||||
transcribed_data.append(vtd) # and add it to list
|
||||
|
||||
# Todo: remove silent part from each word
|
||||
@@ -57,4 +62,4 @@ def transcribe_with_vosk(audio_filename, model_path):
|
||||
|
||||
|
||||
class SpeechToText:
|
||||
pass
|
||||
"""Docstring"""
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Docstring"""
|
||||
|
||||
import whisperx
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
def transcribe_with_whisper(audio_path, model, device="cpu"):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
|
||||
)
|
||||
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
compute_type = (
|
||||
"float16" if device == "cuda" else "int8"
|
||||
) # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
|
||||
# transcribe with original whisper
|
||||
loaded_whisper_model = whisperx.load_model(
|
||||
model, device=device, compute_type=compute_type
|
||||
)
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
|
||||
language = result["language"]
|
||||
|
||||
# load alignment model and metadata
|
||||
model_a, metadata = whisperx.load_align_model(
|
||||
language_code=language, device=device
|
||||
)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(
|
||||
result["segments"],
|
||||
model_a,
|
||||
metadata,
|
||||
audio,
|
||||
device,
|
||||
return_char_alignments=False,
|
||||
)
|
||||
|
||||
transcribed_data = []
|
||||
|
||||
for segment in result_aligned["segments"]:
|
||||
for obj in segment["words"]:
|
||||
if len(obj) < 4:
|
||||
print(
|
||||
f"{red_highlighted('Error: Skipping Word {}, because of missing timings'.format(obj['word']))}"
|
||||
)
|
||||
continue
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
vtd.word = vtd.word + " "
|
||||
transcribed_data.append(vtd) # and add it to list
|
||||
|
||||
return transcribed_data, language
|
||||
+21
-8
@@ -1,11 +1,18 @@
|
||||
from hyphen import Hyphenator
|
||||
from hyphen import dictools
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
from moduls.Log import print_blue_highlighted_text, print_gold_highlighted_text
|
||||
"""Docstring"""
|
||||
|
||||
import string
|
||||
|
||||
from hyphen import Hyphenator, dictools
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
)
|
||||
|
||||
|
||||
def language_check(language="en"):
|
||||
"""Docstring"""
|
||||
|
||||
def language_check(language='en'):
|
||||
dict_langs = dictools.LANGUAGES
|
||||
|
||||
lang_region = None
|
||||
@@ -17,20 +24,26 @@ def language_check(language='en'):
|
||||
if lang_region is None:
|
||||
return None
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Hyphenate using language code: {print_blue_highlighted_text(lang_region)}")
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Hyphenate using language code: {blue_highlighted(lang_region)}"
|
||||
)
|
||||
return lang_region
|
||||
|
||||
|
||||
def contains_punctuation(word):
|
||||
"""Docstring"""
|
||||
|
||||
return any(elem in word for elem in string.punctuation)
|
||||
|
||||
|
||||
def hyphenation(word, lang_region):
|
||||
"""Docstring"""
|
||||
|
||||
if contains_punctuation(word):
|
||||
return None
|
||||
|
||||
hy = Hyphenator(lang_region)
|
||||
syllabus = hy.syllables(word)
|
||||
hyphenator = Hyphenator(lang_region)
|
||||
syllabus = hyphenator.syllables(word)
|
||||
|
||||
length = len(syllabus)
|
||||
if length > 1:
|
||||
+44
-27
@@ -1,22 +1,28 @@
|
||||
"""Docstring"""
|
||||
|
||||
import os
|
||||
|
||||
import speech_recognition as sr
|
||||
from pydub import AudioSegment
|
||||
from pydub.silence import split_on_silence
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
import os
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
# todo: Code from here: https://www.thepythoncode.com/article/using-speech-recognition-to-convert-speech-to-text-python
|
||||
|
||||
def PrintText(wav_file):
|
||||
# English speach!
|
||||
r = sr.Recognizer()
|
||||
|
||||
def print_text(wav_file):
|
||||
"""Docstring"""
|
||||
|
||||
# English speech!
|
||||
recognizer = sr.Recognizer()
|
||||
|
||||
# open the file
|
||||
with sr.AudioFile(wav_file) as source:
|
||||
# listen for the data (load audio to memory)
|
||||
audio_data = r.record(source)
|
||||
audio_data = recognizer.record(source)
|
||||
# recognize (convert from speech to text)
|
||||
text = r.recognize_google(audio_data)
|
||||
text = recognizer.recognize_google(audio_data)
|
||||
|
||||
print(text)
|
||||
|
||||
@@ -30,14 +36,15 @@ def get_large_audio_transcription(wav_file):
|
||||
sound = AudioSegment.from_wav(wav_file)
|
||||
|
||||
# split audio sound where silence is 700 miliseconds or more and get chunks
|
||||
chunks = split_on_silence(sound,
|
||||
# experiment with this value for your target audio file
|
||||
min_silence_len=500,
|
||||
# adjust this per requirement
|
||||
silence_thresh=sound.dBFS - 14,
|
||||
# keep the silence for 1 second, adjustable as well
|
||||
keep_silence=500,
|
||||
)
|
||||
chunks = split_on_silence(
|
||||
sound,
|
||||
# experiment with this value for your target audio file
|
||||
min_silence_len=500,
|
||||
# adjust this per requirement
|
||||
silence_thresh=sound.dBFS - 14,
|
||||
# keep the silence for 1 second, adjustable as well
|
||||
keep_silence=500,
|
||||
)
|
||||
|
||||
folder_name = "audio-chunks"
|
||||
# create a directory to store the audio chunks
|
||||
@@ -45,7 +52,7 @@ def get_large_audio_transcription(wav_file):
|
||||
os.mkdir(folder_name)
|
||||
whole_text = ""
|
||||
|
||||
r = sr.Recognizer()
|
||||
recognizer = sr.Recognizer()
|
||||
|
||||
# process each chunk
|
||||
for i, audio_chunk in enumerate(chunks, start=1):
|
||||
@@ -55,12 +62,12 @@ def get_large_audio_transcription(wav_file):
|
||||
audio_chunk.export(chunk_filename, format="wav")
|
||||
# recognize the chunk
|
||||
with sr.AudioFile(chunk_filename) as source:
|
||||
audio_listened = r.record(source)
|
||||
audio_listened = recognizer.record(source)
|
||||
# try converting it to text
|
||||
try:
|
||||
text = r.recognize_google(audio_listened)
|
||||
except sr.UnknownValueError as e:
|
||||
print("Error:", str(e))
|
||||
text = recognizer.recognize_google(audio_listened)
|
||||
except sr.UnknownValueError as error:
|
||||
print("Error:", str(error))
|
||||
else:
|
||||
text = f"{text.capitalize()}. "
|
||||
print(chunk_filename, ":", text)
|
||||
@@ -70,19 +77,29 @@ def get_large_audio_transcription(wav_file):
|
||||
|
||||
|
||||
def transcribe_audio(audio_file):
|
||||
"""Docstring"""
|
||||
|
||||
recognizer = sr.Recognizer()
|
||||
with sr.AudioFile(audio_file) as source:
|
||||
audio = recognizer.record(source)
|
||||
try:
|
||||
transcript = recognizer.recognize_google(audio, show_all=True)
|
||||
start_time = transcript['result'][0]['alternative'][0]['words'][0]['startTime']
|
||||
end_time = transcript['result'][0]['alternative'][0]['words'][-1]['endTime']
|
||||
return transcript['result'][0]['alternative'][0]['transcript'], start_time, end_time
|
||||
start_time = transcript["result"][0]["alternative"][0]["words"][0][
|
||||
"startTime"
|
||||
]
|
||||
end_time = transcript["result"][0]["alternative"][0]["words"][-1][
|
||||
"endTime"
|
||||
]
|
||||
return (
|
||||
transcript["result"][0]["alternative"][0]["transcript"],
|
||||
start_time,
|
||||
end_time,
|
||||
)
|
||||
except sr.UnknownValueError:
|
||||
print(f"{PRINT_ULTRASTAR} Could not understand audio")
|
||||
except sr.RequestError as e:
|
||||
print(f"{PRINT_ULTRASTAR} Error with recognizing service; {e}")
|
||||
print(f"{ULTRASINGER_HEAD} Could not understand audio")
|
||||
except sr.RequestError as error:
|
||||
print(f"{ULTRASINGER_HEAD} Error with recognizing service; {error}")
|
||||
|
||||
|
||||
class SpeechToText:
|
||||
pass
|
||||
"""Docstring"""
|
||||
@@ -1,16 +1,22 @@
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
def real_bpm_to_ultrastar_bpm(real_bpm):
|
||||
"""Docstring"""
|
||||
# The UltraStar BPM info is a fifth beat of the real BPM
|
||||
ultrastar_bpm = real_bpm / 4
|
||||
return ultrastar_bpm
|
||||
|
||||
|
||||
def ultrastar_bpm_to_real_bpm(ultrastar_bpm):
|
||||
"""Docstring"""
|
||||
# The UltraStar BPM info is a fifth beat of the real BPM
|
||||
bpm = ultrastar_bpm * 4
|
||||
return bpm
|
||||
|
||||
|
||||
def second_to_beat(seconds, real_bpm):
|
||||
"""Docstring"""
|
||||
# BPM = 60 * beat / T
|
||||
# T * BPM = 60 * beat
|
||||
# beat = T * BPM / 60
|
||||
@@ -19,32 +25,54 @@ def second_to_beat(seconds, real_bpm):
|
||||
|
||||
|
||||
def beat_to_second(beat, real_bpm):
|
||||
"""Docstring"""
|
||||
|
||||
seconds = beat * 60 / real_bpm
|
||||
return seconds
|
||||
|
||||
|
||||
def midi_note_to_ultrastar_note(midi_note):
|
||||
"""Docstring"""
|
||||
|
||||
# C4 == 48 in Midi
|
||||
ultrastar_note = midi_note - 48
|
||||
return ultrastar_note
|
||||
|
||||
|
||||
def ultrastar_note_to_midi_note(ultrastar_note):
|
||||
"""Docstring"""
|
||||
|
||||
# C4 == 48 in Midi
|
||||
midi_note = ultrastar_note + 48
|
||||
return midi_note
|
||||
|
||||
|
||||
def get_start_time_from_ultrastar(ultrastar_class, pos):
|
||||
"""Docstring"""
|
||||
|
||||
gap = int(ultrastar_class.gap) / 1000
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(',', '.')))
|
||||
start_time = beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(
|
||||
float(ultrastar_class.bpm.replace(",", "."))
|
||||
)
|
||||
start_time = (
|
||||
beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap
|
||||
)
|
||||
return start_time
|
||||
|
||||
|
||||
def get_end_time_from_ultrastar(ultrastar_class, pos):
|
||||
"""Docstring"""
|
||||
|
||||
gap = int(ultrastar_class.gap) / 1000
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(',', '.')))
|
||||
end_time = beat_to_second(int(ultrastar_class.startBeat[pos]) + int(ultrastar_class.durations[pos]),
|
||||
real_bpm) + gap
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(
|
||||
float(ultrastar_class.bpm.replace(",", "."))
|
||||
)
|
||||
end_time = (
|
||||
beat_to_second(
|
||||
int(ultrastar_class.startBeat[pos])
|
||||
+ int(ultrastar_class.durations[pos]),
|
||||
real_bpm,
|
||||
)
|
||||
+ gap
|
||||
)
|
||||
return end_time
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Docstring"""
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
)
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
|
||||
|
||||
def parse_ultrastar_txt(input_file):
|
||||
"""Docstring"""
|
||||
print(f"{ULTRASINGER_HEAD} Parse ultrastar txt -> {input_file}")
|
||||
|
||||
with open(input_file, "r", encoding="utf-8") as file:
|
||||
txt = file.readlines()
|
||||
|
||||
ultrastar_class = UltrastarTxt()
|
||||
count = 0
|
||||
|
||||
# Strips the newline character
|
||||
for line in txt:
|
||||
count += 1
|
||||
if line.startswith("#"):
|
||||
if line.startswith("#ARTIST"):
|
||||
ultrastar_class.artist = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith("#TITLE"):
|
||||
ultrastar_class.title = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith("#MP3"):
|
||||
ultrastar_class.mp3 = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith("#GAP"):
|
||||
ultrastar_class.gap = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith("#BPM"):
|
||||
ultrastar_class.bpm = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(("F ", ": ", "* ", "R ", "G ")):
|
||||
parts = line.split()
|
||||
# [0] F : * R G
|
||||
# [1] start beat
|
||||
# [2] duration
|
||||
# [3] pitch
|
||||
# [4] word
|
||||
|
||||
ultrastar_class.noteType.append(parts[0])
|
||||
ultrastar_class.startBeat.append(parts[1])
|
||||
ultrastar_class.durations.append(parts[2])
|
||||
ultrastar_class.pitches.append(parts[3])
|
||||
ultrastar_class.words.append(parts[4] if len(parts) > 4 else "")
|
||||
|
||||
# do always as last
|
||||
pos = len(ultrastar_class.startBeat) - 1
|
||||
ultrastar_class.startTimes.append(
|
||||
get_start_time_from_ultrastar(ultrastar_class, pos)
|
||||
)
|
||||
ultrastar_class.endTimes.append(
|
||||
get_end_time_from_ultrastar(ultrastar_class, pos)
|
||||
)
|
||||
# todo: Progress?
|
||||
|
||||
return ultrastar_class
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Docstring"""
|
||||
|
||||
import librosa
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
cyan_highlighted,
|
||||
gold_highlighted,
|
||||
light_blue_highlighted,
|
||||
underlined,
|
||||
)
|
||||
from modules.Midi.midi_creator import create_midi_note_from_pitched_data
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
ultrastar_note_to_midi_note,
|
||||
)
|
||||
|
||||
MAX_SONG_SCORE = 10000
|
||||
MAX_SONG_LINE_BONUS = 1000
|
||||
|
||||
|
||||
class Points:
|
||||
"""Docstring"""
|
||||
|
||||
notes = 0
|
||||
golden_notes = 0
|
||||
rap = 0
|
||||
golden_rap = 0
|
||||
line_bonus = 0
|
||||
parts = 0
|
||||
|
||||
|
||||
def add_point(note_type, points):
|
||||
"""Docstring"""
|
||||
|
||||
if note_type == ":":
|
||||
points.notes += 1
|
||||
elif note_type == "*":
|
||||
points.golden_notes += 2
|
||||
elif note_type == "R":
|
||||
points.rap += 1
|
||||
elif note_type == "G":
|
||||
points.golden_rap += 2
|
||||
return points
|
||||
|
||||
|
||||
class Score:
|
||||
"""Docstring"""
|
||||
|
||||
max_score = 0
|
||||
notes = 0
|
||||
golden = 0
|
||||
line_bonus = 0
|
||||
score = 0
|
||||
|
||||
|
||||
def get_score(points):
|
||||
"""Docstring"""
|
||||
|
||||
score = Score()
|
||||
score.max_score = (
|
||||
MAX_SONG_SCORE
|
||||
if points.line_bonus == 0
|
||||
else MAX_SONG_SCORE - MAX_SONG_LINE_BONUS
|
||||
)
|
||||
score.notes = round(
|
||||
score.max_score * (points.notes + points.rap) / points.parts
|
||||
)
|
||||
score.golden = round(points.golden_notes + points.golden_rap)
|
||||
score.score = round(score.notes + points.line_bonus + score.golden)
|
||||
score.line_bonus = round(points.line_bonus)
|
||||
return score
|
||||
|
||||
|
||||
def print_score(score):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Total: {cyan_highlighted(str(score.score))}, notes: {blue_highlighted(str(score.notes))}, line bonus: {light_blue_highlighted(str(score.line_bonus))}, golden notes: {gold_highlighted(str(score.golden))}"
|
||||
)
|
||||
|
||||
|
||||
def calculate_score(pitched_data, ultrastar_class):
|
||||
"""Docstring"""
|
||||
|
||||
print(ULTRASINGER_HEAD + " Calculating Ultrastar Points")
|
||||
|
||||
simple_points = Points()
|
||||
accurate_points = Points()
|
||||
|
||||
reachable_line_bonus_per_word = MAX_SONG_LINE_BONUS / len(
|
||||
ultrastar_class.words
|
||||
)
|
||||
|
||||
for i in enumerate(ultrastar_class.words):
|
||||
pos = i[0]
|
||||
if ultrastar_class.words == "":
|
||||
continue
|
||||
|
||||
if ultrastar_class.noteType[pos] == "F":
|
||||
continue
|
||||
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, pos)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, pos)
|
||||
duration = end_time - start_time
|
||||
step_size = 0.01 # todo: should be beat length ?
|
||||
parts = int(duration / step_size)
|
||||
parts = 1 if parts == 0 else parts
|
||||
|
||||
accurate_part_line_bonus_points = 0
|
||||
simple_part_line_bonus_points = 0
|
||||
|
||||
ultrastar_midi_note = ultrastar_note_to_midi_note(
|
||||
int(ultrastar_class.pitches[pos])
|
||||
)
|
||||
ultrastar_note = librosa.midi_to_note(ultrastar_midi_note)
|
||||
|
||||
for part in range(parts):
|
||||
start = start_time + step_size * part
|
||||
end = start + step_size
|
||||
pitch_note = create_midi_note_from_pitched_data(
|
||||
start, end, pitched_data
|
||||
)
|
||||
|
||||
if pitch_note[:-1] == ultrastar_note[:-1]:
|
||||
# Ignore octave high
|
||||
simple_points = add_point(
|
||||
ultrastar_class.noteType[pos], simple_points
|
||||
)
|
||||
simple_part_line_bonus_points += 1
|
||||
|
||||
if pitch_note == ultrastar_note:
|
||||
# Octave high must be the same
|
||||
accurate_points = add_point(
|
||||
ultrastar_class.noteType[pos], accurate_points
|
||||
)
|
||||
accurate_part_line_bonus_points += 1
|
||||
|
||||
accurate_points.parts += 1
|
||||
simple_points.parts += 1
|
||||
|
||||
if accurate_part_line_bonus_points >= parts:
|
||||
accurate_points.line_bonus += reachable_line_bonus_per_word
|
||||
|
||||
if simple_part_line_bonus_points >= parts:
|
||||
simple_points.line_bonus += reachable_line_bonus_per_word
|
||||
|
||||
return get_score(simple_points), get_score(accurate_points)
|
||||
|
||||
|
||||
def print_score_calculation(simple_points, accurate_points):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {underlined('Simple (octave high ignored)')} points"
|
||||
)
|
||||
print_score(simple_points)
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {underlined('Accurate (octave high matches)')} points:"
|
||||
)
|
||||
print_score(accurate_points)
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
class UltrastarTxt:
|
||||
"""Docstring"""
|
||||
|
||||
artist = ""
|
||||
title = ""
|
||||
mp3 = ""
|
||||
gap = ""
|
||||
bpm = ""
|
||||
cover = None
|
||||
creator = "UltraSinger [GitHub]"
|
||||
fixer = "YOUR NAME" # Only for UltraStar World Party
|
||||
comment = "UltraSinger [GitHub]"
|
||||
startBeat = []
|
||||
startTimes = []
|
||||
endTimes = []
|
||||
durations = []
|
||||
pitches = []
|
||||
words = []
|
||||
noteType = []
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Docstring"""
|
||||
|
||||
import re
|
||||
|
||||
import langcodes
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
real_bpm_to_ultrastar_bpm,
|
||||
second_to_beat,
|
||||
)
|
||||
|
||||
|
||||
def get_multiplier(real_bpm):
|
||||
"""Docstring"""
|
||||
|
||||
if real_bpm == 0:
|
||||
raise Exception("BPM is 0")
|
||||
|
||||
multiplier = 1
|
||||
result = 0
|
||||
while result < 400:
|
||||
result = real_bpm * multiplier
|
||||
multiplier += 1
|
||||
return multiplier - 2
|
||||
|
||||
|
||||
def get_language_name(language):
|
||||
"""Docstring"""
|
||||
|
||||
return langcodes.Language.make(language=language).display_name()
|
||||
|
||||
|
||||
def create_ultrastar_txt_from_automation(
|
||||
transcribed_data,
|
||||
note_numbers,
|
||||
ultrastar_file_output,
|
||||
ultrastar_class,
|
||||
bpm=120,
|
||||
):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Creating {ultrastar_file_output} from transcription."
|
||||
)
|
||||
|
||||
real_bpm = real_bpm_to_ultrastar_bpm(bpm)
|
||||
multiplication = get_multiplier(real_bpm)
|
||||
ultrastar_bpm = real_bpm * get_multiplier(real_bpm)
|
||||
|
||||
with open(ultrastar_file_output, "w", encoding="utf-8") as file:
|
||||
gap = transcribed_data[0].start
|
||||
|
||||
file.write(f"#ARTIST:{ultrastar_class.artist}\n")
|
||||
file.write(f"#TITLE:{ultrastar_class.title}\n")
|
||||
file.write(f"#CREATOR:{ultrastar_class.creator}\n")
|
||||
file.write(f"#FIXER:{ultrastar_class.fixer}\n")
|
||||
if ultrastar_class.language is not None:
|
||||
file.write(
|
||||
f"#LANGUAGE:{get_language_name(ultrastar_class.language)}\n"
|
||||
)
|
||||
if ultrastar_class.cover is not None:
|
||||
file.write(f"#COVER:{ultrastar_class.cover}\n")
|
||||
file.write(f"#MP3:{ultrastar_class.mp3}\n")
|
||||
file.write(f"#VIDEO:{ultrastar_class.video}\n")
|
||||
file.write(f"#BPM:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
|
||||
file.write(f"#GAP:{int(gap * 1000)}\n")
|
||||
file.write(f"#COMMENT:{ultrastar_class.comment}\n")
|
||||
|
||||
# Write the singing part
|
||||
previous_end_beat = 0
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_time = (data.start - gap) * multiplication
|
||||
end_time = (
|
||||
data.end - data.start
|
||||
) * multiplication
|
||||
start_beat = round(second_to_beat(start_time, bpm))
|
||||
duration = round(second_to_beat(end_time, bpm))
|
||||
|
||||
# Fix the round issue, so the beats don’t overlap
|
||||
start_beat = max(start_beat, previous_end_beat)
|
||||
previous_end_beat = start_beat + duration
|
||||
|
||||
# : 10 10 10 w
|
||||
# ':' start midi part
|
||||
# 'n1' start at real beat
|
||||
# 'n2' duration at real beat
|
||||
# 'n3' pitch where 0 == C4
|
||||
# 'w' lyric
|
||||
file.write(": ")
|
||||
file.write(str(start_beat) + " ")
|
||||
file.write(str(duration) + " ")
|
||||
file.write(str(note_numbers[i]) + " ")
|
||||
file.write(data.word)
|
||||
file.write("\n")
|
||||
|
||||
# detect silence between words
|
||||
if i < len(transcribed_data) - 1:
|
||||
silence = (
|
||||
transcribed_data[i + 1].start - data.end
|
||||
)
|
||||
else:
|
||||
silence = 0
|
||||
|
||||
if silence > 0.3 and i != len(transcribed_data) - 1:
|
||||
# - 10
|
||||
# '-' end of current sing part
|
||||
# 'n1' show next at time in real beat
|
||||
file.write("- ")
|
||||
show_next = (
|
||||
second_to_beat(data.end - gap, bpm)
|
||||
* multiplication
|
||||
)
|
||||
file.write(str(round(show_next)))
|
||||
file.write("\n")
|
||||
file.write("E")
|
||||
|
||||
|
||||
def create_repitched_txt_from_ultrastar_data(
|
||||
input_file, note_numbers, output_repitched_ultrastar
|
||||
):
|
||||
"""Docstring"""
|
||||
# todo: just add '_repitched' to input_file
|
||||
print(
|
||||
"{PRINT_ULTRASTAR} Creating repitched ultrastar txt -> {input_file}_repitch.txt"
|
||||
)
|
||||
|
||||
# todo: to reader
|
||||
with open(input_file, "r", encoding="utf-8") as file:
|
||||
txt = file.readlines()
|
||||
|
||||
i = 0
|
||||
# todo: just add '_repitched' to input_file
|
||||
with open(output_repitched_ultrastar, "w", encoding="utf-8") as file:
|
||||
for line in txt:
|
||||
if line.startswith(":"):
|
||||
parts = re.findall(r"\S+|\s+", line)
|
||||
# between are whitespaces
|
||||
# [0] :
|
||||
# [2] start beat
|
||||
# [4] duration
|
||||
# [6] pitch
|
||||
# [8] word
|
||||
parts[6] = str(note_numbers[i])
|
||||
delimiter = ""
|
||||
file.write(delimiter.join(parts))
|
||||
i += 1
|
||||
else:
|
||||
file.write(line)
|
||||
|
||||
|
||||
def add_score_to_ultrastar_txt(ultrastar_file_output, score):
|
||||
"""Docstring"""
|
||||
with open(ultrastar_file_output, "r", encoding="utf-8") as file:
|
||||
text = file.read()
|
||||
text = text.split("\n")
|
||||
|
||||
for i, line in enumerate(text):
|
||||
if line.startswith("#COMMENT:"):
|
||||
text[
|
||||
i
|
||||
] = f"{line} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
|
||||
break
|
||||
|
||||
if line.startswith(("F ", ": ", "* ", "R ", "G ")):
|
||||
text.insert(
|
||||
i,
|
||||
f"#COMMENT: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
|
||||
)
|
||||
break
|
||||
|
||||
text = "\n".join(text)
|
||||
|
||||
with open(ultrastar_file_output, "w", encoding="utf-8") as file:
|
||||
file.write(text)
|
||||
|
||||
|
||||
class UltraStarWriter:
|
||||
"""Docstring"""
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Colors for the console"""
|
||||
|
||||
ULTRASINGER_HEAD = "\033[92m[UltraSinger]\033[0m"
|
||||
|
||||
|
||||
def blue_highlighted(text: str) -> str:
|
||||
"""Returns a blue highlighted text"""
|
||||
return f"{Bcolors.blue}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def gold_highlighted(text: str) -> str:
|
||||
"""Returns a gold highlighted text"""
|
||||
return f"{Bcolors.gold}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def light_blue_highlighted(text: str) -> str:
|
||||
"""Returns a light blue highlighted text"""
|
||||
return f"{Bcolors.light_blue}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def underlined(text: str) -> str:
|
||||
"""Returns an underlined text"""
|
||||
return f"{Bcolors.underline}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def red_highlighted(text: str) -> str:
|
||||
"""Returns a red highlighted text"""
|
||||
return f"{Bcolors.red}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def cyan_highlighted(text: str) -> str:
|
||||
"""Returns a cyan highlighted text"""
|
||||
return f"{Bcolors.cyan}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
class Bcolors:
|
||||
"""Colors for the console"""
|
||||
|
||||
blue = "\033[94m"
|
||||
red = "\033[91m"
|
||||
light_blue = "\033[96m"
|
||||
cyan = "\033[36m"
|
||||
gold = "\033[93m"
|
||||
underline = "\033[4m"
|
||||
endc = "\033[0m"
|
||||
@@ -0,0 +1,24 @@
|
||||
"""CSV export module"""
|
||||
|
||||
import csv
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def export_transcribed_data_to_csv(transcribed_data: [], filename: str) -> None:
|
||||
"""Export transcribed data to csv"""
|
||||
print(f"{ULTRASINGER_HEAD} Exporting transcribed data to CSV")
|
||||
|
||||
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["word", "start", "end", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i, data in enumerate(transcribed_data):
|
||||
writer.writerow(
|
||||
[
|
||||
data.word,
|
||||
data.start,
|
||||
data.end,
|
||||
data.conf,
|
||||
]
|
||||
)
|
||||
@@ -1,34 +1,44 @@
|
||||
"""Docstring"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def create_folder(folder_name):
|
||||
print(f"{PRINT_ULTRASTAR} Creating output folder. -> {folder_name}")
|
||||
"""Docstring"""
|
||||
print(f"{ULTRASINGER_HEAD} Creating output folder. -> {folder_name}")
|
||||
is_exist = os.path.exists(folder_name)
|
||||
if not is_exist:
|
||||
os.makedirs(folder_name)
|
||||
|
||||
|
||||
def move(original, target):
|
||||
"""Docstring"""
|
||||
shutil.move(original, target)
|
||||
|
||||
|
||||
def copy(original, target):
|
||||
"""Docstring"""
|
||||
shutil.copy(original, target)
|
||||
|
||||
|
||||
def current_executor_path():
|
||||
"""Docstring"""
|
||||
return os.getcwd()
|
||||
|
||||
|
||||
def path_join(path1, path2):
|
||||
"""Docstring"""
|
||||
return os.path.join(path1, path2)
|
||||
|
||||
|
||||
def check_file_exists(file_path):
|
||||
"""Docstring"""
|
||||
return os.path.isfile(file_path)
|
||||
|
||||
|
||||
def check_if_folder_exists(song_output):
|
||||
return os.path.isdir(song_output)
|
||||
"""Docstring"""
|
||||
return os.path.isdir(song_output)
|
||||
@@ -1,23 +0,0 @@
|
||||
import ffmpeg
|
||||
from moduls.Log import PRINT_ULTRASTAR, print_blue_highlighted_text
|
||||
|
||||
|
||||
def ffmpeg_reduce_noise(input_file_path, output_file):
|
||||
# Denoise audio samples with FFT.
|
||||
# A description of the accepted parameters follows.
|
||||
|
||||
# noise_reduction, nr
|
||||
# Set the noise reduction in dB, allowed range is 0.01 to 97. Default value is 12 dB.
|
||||
# noise_floor, nf
|
||||
# Set the noise floor in dB, allowed range is -80 to -20. Default value is -50 dB.
|
||||
# track_noise, tn
|
||||
# Enable noise floor tracking. By default is disabled. With this enabled, noise floor is automatically adjusted.
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Reduce noise from vocal audio with {print_blue_highlighted_text('ffmpeg')}.")
|
||||
(
|
||||
ffmpeg
|
||||
.input(input_file_path)
|
||||
.output(output_file, af='afftdn=nr=70:nf=-50:tn=1')
|
||||
.overwrite_output()
|
||||
.run()
|
||||
)
|
||||
@@ -1,15 +0,0 @@
|
||||
import subprocess
|
||||
from moduls.os_helper import move, current_executor_path, path_join
|
||||
from moduls.Log import PRINT_ULTRASTAR, print_blue_highlighted_text, print_red_highlighted_text
|
||||
|
||||
|
||||
def separate_audio(input_file_path, output_file, device="cpu"):
|
||||
print(f"{PRINT_ULTRASTAR} Separating vocals from audio with {print_blue_highlighted_text('demucs')} and {print_red_highlighted_text(device)} as worker.")
|
||||
# Model selection?
|
||||
# -n mdx_q
|
||||
# -n htdemucs_ft
|
||||
device = "cpu" if device != "cuda" else "cuda"
|
||||
subprocess.run(["demucs", "-d", device, "--two-stems=vocals", input_file_path])
|
||||
|
||||
separated_folder = path_join(current_executor_path(), "separated")
|
||||
move(separated_folder, output_file)
|
||||
@@ -1,144 +0,0 @@
|
||||
import csv
|
||||
import os
|
||||
import wave
|
||||
import re
|
||||
|
||||
import librosa
|
||||
|
||||
from moduls.os_helper import create_folder
|
||||
from moduls.Ultrastar.ultrastar_converter import get_start_time_from_ultrastar, get_end_time_from_ultrastar
|
||||
from pydub import AudioSegment
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
|
||||
|
||||
def convert_audio_to_mono_wav(input_file, output_file):
|
||||
"""Convert audio to mono wav"""
|
||||
print(f"{PRINT_ULTRASTAR} Converting audio for AI")
|
||||
|
||||
if '.mp3' in input_file:
|
||||
sound = AudioSegment.from_mp3(input_file)
|
||||
elif '.wav' in input_file:
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
else:
|
||||
raise ValueError('data format not supported')
|
||||
|
||||
sound = sound.set_channels(1)
|
||||
sound.export(output_file, format="wav")
|
||||
|
||||
|
||||
def convert_wav_to_mp3(input_file, output_file):
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
sound.export(output_file, format="mp3")
|
||||
|
||||
|
||||
class AudioManipulation:
|
||||
pass
|
||||
|
||||
|
||||
def export_chunks_from_transcribed_data(audio_filename, transcribed_data, output_folder_name):
|
||||
"""Export transcribed_data as vocal chunks wav files"""
|
||||
print(f"{PRINT_ULTRASTAR} Export transcribed data as vocal chunks wav files")
|
||||
|
||||
wf = wave.open(audio_filename, "rb")
|
||||
sr, n_channels = wf.getparams()[2], wf.getparams()[0]
|
||||
|
||||
for i in range(len(transcribed_data)):
|
||||
start_byte = int(transcribed_data[i].start * sr * n_channels)
|
||||
end_byte = int(transcribed_data[i].end * sr * n_channels)
|
||||
|
||||
chunk = get_chunk(end_byte, start_byte, wf)
|
||||
export_chunk_to_wav_file(chunk, output_folder_name, i, transcribed_data[i].word, wf)
|
||||
|
||||
wf.close()
|
||||
|
||||
|
||||
def remove_silence_from_transcribtion_data(audio_path, transcribed_data):
|
||||
print(f"{PRINT_ULTRASTAR} Removing silent start and ending, from transcription data")
|
||||
|
||||
y, sr = librosa.load(audio_path, sr=None)
|
||||
|
||||
for i in range(len(transcribed_data)):
|
||||
start_time = transcribed_data[i].start
|
||||
end_time = transcribed_data[i].end
|
||||
start_sample = int(start_time * sr)
|
||||
end_sample = int(end_time * sr)
|
||||
chunk = y[start_sample:end_sample]
|
||||
|
||||
# todo: why 5 works good? It should be 40db ?!?
|
||||
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
|
||||
silence_threshold = 5
|
||||
onsets = librosa.effects.split(chunk, top_db=silence_threshold, frame_length=2048, hop_length=100)
|
||||
|
||||
# Get the duration of the first and last silent intervals
|
||||
if len(onsets) > 0:
|
||||
first_silence = onsets[0][0]
|
||||
last_silence = len(chunk) - onsets[-1][1]
|
||||
|
||||
first_silence_duration = librosa.samples_to_time(first_silence, sr=sr)
|
||||
last_silence_duration = librosa.samples_to_time(last_silence, sr=sr)
|
||||
else:
|
||||
first_silence_duration = 0
|
||||
last_silence_duration = 0
|
||||
|
||||
transcribed_data[i].start = transcribed_data[i].start + first_silence_duration
|
||||
transcribed_data[i].end = transcribed_data[i].end - last_silence_duration
|
||||
|
||||
return transcribed_data
|
||||
|
||||
|
||||
def export_chunks_from_ultrastar_data(audio_filename, ultrastar_data, folder_name):
|
||||
"""Export ultrastar data as vocal chunks wav files"""
|
||||
print(f"{PRINT_ULTRASTAR} Export Ultrastar data as vocal chunks wav files")
|
||||
|
||||
create_folder(folder_name)
|
||||
|
||||
wf = wave.open(audio_filename, "rb")
|
||||
sr, n_channels = wf.getparams()[2], wf.getparams()[0]
|
||||
|
||||
for i in range(len(ultrastar_data.words)):
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_data, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_data, i)
|
||||
|
||||
start_byte = int(start_time * sr * n_channels)
|
||||
end_byte = int(end_time * sr * n_channels)
|
||||
|
||||
chunk = get_chunk(end_byte, start_byte, wf)
|
||||
export_chunk_to_wav_file(chunk, folder_name, i, ultrastar_data.words[i], wf)
|
||||
|
||||
|
||||
def export_chunk_to_wav_file(chunk, folder_name, i, word, wf):
|
||||
"""Export vocal chunks to wav file"""
|
||||
|
||||
clean_word = re.sub('[^A-Za-z0-9]+', '', word)
|
||||
# todo: Progress?
|
||||
# print(str(i) + ' ' + clean_word)
|
||||
with wave.open(os.path.join(folder_name, f"chunk_{i}_{clean_word}.wav"),
|
||||
"wb") as chunk_file:
|
||||
chunk_file.setparams(wf.getparams())
|
||||
chunk_file.writeframes(chunk)
|
||||
|
||||
|
||||
def get_chunk(end_byte, start_byte, wf):
|
||||
"""
|
||||
Gets the chunk from wave file.
|
||||
Returns chunk as n frames of audio, as a bytes object.
|
||||
"""
|
||||
|
||||
# todo: get out of position error message
|
||||
wf.setpos(start_byte) # ({:.2f})
|
||||
chunk = wf.readframes(end_byte - start_byte)
|
||||
return chunk
|
||||
|
||||
|
||||
def export_transcribed_data_to_csv(transcribed_data, filename):
|
||||
"""Export transcribed data to csv"""
|
||||
print(f"{PRINT_ULTRASTAR} Exporting transcribed data to CSV")
|
||||
|
||||
with open(filename, 'w', newline='') as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["word", "start", "end", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i in range(len(transcribed_data)):
|
||||
writer.writerow(
|
||||
[transcribed_data[i].word, transcribed_data[i].start, transcribed_data[i].end,
|
||||
transcribed_data[i].conf])
|
||||
@@ -1,67 +0,0 @@
|
||||
import yt_dlp
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
from PIL import Image
|
||||
import io
|
||||
import os
|
||||
|
||||
|
||||
def get_youtube_title(url):
|
||||
ydl_opts = {}
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
result = ydl.extract_info(
|
||||
url,
|
||||
download=False # We just want to extract the info
|
||||
)
|
||||
|
||||
return result['title']
|
||||
|
||||
|
||||
def download_youtube_audio(url, clear_filename, output_path):
|
||||
print(f"{PRINT_ULTRASTAR} Downloading Audio")
|
||||
ydl_opts = {
|
||||
'format': 'bestaudio/best',
|
||||
'outtmpl': output_path + '/' + clear_filename,
|
||||
'postprocessors': [{
|
||||
'key': 'FFmpegExtractAudio',
|
||||
'preferredcodec': 'mp3'
|
||||
}]
|
||||
}
|
||||
|
||||
start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def download_youtube_thumbnail(url, clear_filename, output_path):
|
||||
print(f"{PRINT_ULTRASTAR} Downloading thumbnail")
|
||||
ydl_opts = {
|
||||
'skip_download': True,
|
||||
'writethumbnail': True,
|
||||
}
|
||||
|
||||
download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path)
|
||||
|
||||
|
||||
def download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path):
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
info_dict = ydl.extract_info(url, download=False)
|
||||
thumbnail_url = info_dict.get('thumbnail')
|
||||
if thumbnail_url:
|
||||
response = ydl.urlopen(thumbnail_url)
|
||||
image_data = response.read()
|
||||
image = Image.open(io.BytesIO(image_data))
|
||||
image.save(os.path.join(output_path, clear_filename + " [CO].jpg"), "JPEG")
|
||||
|
||||
|
||||
def download_youtube_video(url, clear_filename, output_path):
|
||||
print(f"{PRINT_ULTRASTAR} Downloading Video")
|
||||
ydl_opts = {
|
||||
'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4',
|
||||
'outtmpl': output_path + '/' + clear_filename + '.mp4'
|
||||
}
|
||||
start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def start_download(ydl_opts, url):
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
errors = ydl.download(url)
|
||||
if errors:
|
||||
raise Exception("Download failed with error: " + str(errors))
|
||||
@@ -1,13 +0,0 @@
|
||||
from moduls.Log import PRINT_ULTRASTAR, print_red_highlighted_text
|
||||
import torch
|
||||
|
||||
def get_available_device():
|
||||
isCuda = torch.cuda.is_available()
|
||||
|
||||
if not isCuda:
|
||||
print(f"{PRINT_ULTRASTAR} There are no {print_red_highlighted_text('cuda')} devices available. -> Using {print_red_highlighted_text('cpu')}.")
|
||||
return "cpu"
|
||||
else:
|
||||
print(f"{PRINT_ULTRASTAR} Using {print_red_highlighted_text('cuda')} GPU.")
|
||||
|
||||
return "cuda"
|
||||
@@ -1,35 +0,0 @@
|
||||
PRINT_ULTRASTAR = "\033[92m[UltraSinger]\033[0m"
|
||||
|
||||
|
||||
def print_blue_highlighted_text(text):
|
||||
return Bcolors.BLUE + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_gold_highlighted_text(text):
|
||||
return Bcolors.Gold + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_light_blue_highlighted_text(text):
|
||||
return Bcolors.LightBlue + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_underlined_text(text):
|
||||
return Bcolors.Underline + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_red_highlighted_text(text):
|
||||
return Bcolors.RED + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_cyan_highlighted_text(text):
|
||||
return Bcolors.Cyan + text + Bcolors.ENDC
|
||||
|
||||
|
||||
class Bcolors:
|
||||
BLUE = '\033[94m'
|
||||
RED = '\033[91m'
|
||||
LightBlue = '\033[96m'
|
||||
Cyan = '\033[36m'
|
||||
Gold = '\033[93m'
|
||||
Underline = '\033[4m'
|
||||
ENDC = '\033[0m'
|
||||
@@ -1,96 +0,0 @@
|
||||
import math
|
||||
from collections import Counter
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import pretty_midi
|
||||
|
||||
from moduls.Pitcher.pitcher import get_frequency_with_high_confidence
|
||||
from moduls.Ultrastar.ultrastar_converter import ultrastar_note_to_midi_note, get_start_time_from_ultrastar, \
|
||||
get_end_time_from_ultrastar
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
|
||||
def convert_ultrastar_to_midi_instrument(ultrastar_class):
|
||||
print(f"{PRINT_ULTRASTAR} Creating midi instrument from Ultrastar txt")
|
||||
|
||||
instrument = pretty_midi.Instrument(program=0)
|
||||
velocity = 100
|
||||
|
||||
for i in range(len(ultrastar_class.words)):
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, i)
|
||||
pitch = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[i]))
|
||||
|
||||
note = pretty_midi.Note(velocity, pitch, start_time, end_time)
|
||||
instrument.notes.append(note)
|
||||
|
||||
return instrument
|
||||
|
||||
|
||||
def instruments_to_midi(instruments, bpm, midi_output):
|
||||
print(f"{PRINT_ULTRASTAR} Creating midi file -> {midi_output}")
|
||||
|
||||
midi_data = pretty_midi.PrettyMIDI(initial_tempo=bpm)
|
||||
for instrument in instruments:
|
||||
midi_data.instruments.append(instrument)
|
||||
midi_data.write(midi_output)
|
||||
|
||||
|
||||
class MidiCreator:
|
||||
pass
|
||||
|
||||
|
||||
def convert_frequencies_to_notes(frequency):
|
||||
notes = []
|
||||
for f in frequency:
|
||||
notes.append(librosa.hz_to_note(float(f)))
|
||||
return notes
|
||||
|
||||
|
||||
def most_frequent(ar):
|
||||
return Counter(ar).most_common(1)
|
||||
|
||||
|
||||
def find_nearest_index(array, value):
|
||||
idx = np.searchsorted(array, value, side="left")
|
||||
if idx > 0 and (idx == len(array) or math.fabs(value - array[idx - 1]) < math.fabs(value - array[idx])):
|
||||
return idx - 1
|
||||
else:
|
||||
return idx
|
||||
|
||||
|
||||
def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
|
||||
print(f"{PRINT_ULTRASTAR} Creating midi notes from pitched data")
|
||||
|
||||
midi_notes = []
|
||||
|
||||
for i in range(len(start_times)):
|
||||
start_time = start_times[i]
|
||||
end_time = end_times[i]
|
||||
|
||||
note = create_midi_note_from_pitched_data(start_time, end_time, pitched_data)
|
||||
|
||||
midi_notes.append(note)
|
||||
# todo: Progress?
|
||||
# print(filename + " f: " + str(mean))
|
||||
return midi_notes
|
||||
|
||||
|
||||
def create_midi_note_from_pitched_data(start_time, end_time, pitched_data):
|
||||
s = find_nearest_index(pitched_data.times, start_time)
|
||||
e = find_nearest_index(pitched_data.times, end_time)
|
||||
|
||||
if s == e:
|
||||
f = [pitched_data.frequencies[s]]
|
||||
c = [pitched_data.confidence[s]]
|
||||
else:
|
||||
f = pitched_data.frequencies[s:e]
|
||||
c = pitched_data.confidence[s:e]
|
||||
|
||||
conf_f = get_frequency_with_high_confidence(f, c)
|
||||
|
||||
notes = convert_frequencies_to_notes(conf_f)
|
||||
|
||||
note = most_frequent(notes)[0][0]
|
||||
|
||||
return note
|
||||
@@ -1,61 +0,0 @@
|
||||
import csv
|
||||
import crepe
|
||||
from moduls.Pitcher.pitched_data import PitchedData
|
||||
from scipy.io import wavfile
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
from moduls.Log import print_blue_highlighted_text, print_red_highlighted_text
|
||||
|
||||
|
||||
def get_pitch_with_crepe_file(filename, step_size, model_capacity):
|
||||
print(f"{PRINT_ULTRASTAR} Pitching with {print_blue_highlighted_text('crepe')} and model {print_blue_highlighted_text(model_capacity)} and {print_red_highlighted_text('cpu')} as worker")
|
||||
# Todo: add GPU support by using torchcrepe
|
||||
sr, audio = wavfile.read(filename)
|
||||
|
||||
pitched_data = PitchedData()
|
||||
pitched_data.times, pitched_data.frequencies, pitched_data.confidence, activation = crepe.predict(audio, sr,
|
||||
model_capacity,
|
||||
step_size=step_size,
|
||||
viterbi=True)
|
||||
return pitched_data
|
||||
|
||||
|
||||
def get_pitch_with_crepe(y, sr, step_size, model_capacity):
|
||||
pitched_data = PitchedData()
|
||||
pitched_data.times, pitched_data.frequencies, pitched_data.confidence, activation = crepe.predict(y, sr,
|
||||
model_capacity,
|
||||
step_size=step_size,
|
||||
viterbi=True)
|
||||
return pitched_data
|
||||
|
||||
|
||||
def write_lists_to_csv(time, frequency, confidence, filename):
|
||||
with open(filename, 'w', newline='') as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["time", "frequency", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i in range(len(time)):
|
||||
writer.writerow([time[i], frequency[i], confidence[i]])
|
||||
|
||||
|
||||
def read_data_from_csv(filename):
|
||||
csv_data = []
|
||||
with open(filename, 'r') as csv_file:
|
||||
csv_reader = csv.reader(csv_file)
|
||||
for line in csv_reader:
|
||||
csv_data.append(line)
|
||||
headless_data = csv_data[1:]
|
||||
return headless_data
|
||||
|
||||
|
||||
def get_frequency_with_high_confidence(f, c, threshold=0.4):
|
||||
conf_f = []
|
||||
for i in range(len(c)):
|
||||
if c[i] > threshold:
|
||||
conf_f.append(f[i])
|
||||
if not conf_f:
|
||||
conf_f = f
|
||||
return conf_f
|
||||
|
||||
|
||||
class Pitcher:
|
||||
pass
|
||||
@@ -1,11 +0,0 @@
|
||||
class TranscribedData:
|
||||
""" Transcribed data from json file """
|
||||
|
||||
def __init__(self, transcribed_json):
|
||||
# Vosk = conf, Whisper = confidence
|
||||
self.conf = transcribed_json.get("conf", transcribed_json.get("confidence", None))
|
||||
# Vosk = word, Whisper = text
|
||||
self.word = transcribed_json.get("word", transcribed_json.get("text", None))
|
||||
self.end = transcribed_json["end"]
|
||||
self.start = transcribed_json["start"]
|
||||
self.is_hyphen = None
|
||||
@@ -1,39 +0,0 @@
|
||||
import whisperx
|
||||
from moduls.Speech_Recognition.TranscribedData import TranscribedData
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
from moduls.Log import print_blue_highlighted_text, print_red_highlighted_text
|
||||
|
||||
|
||||
def transcribe_with_whisper(audio_path, model, device="cpu"):
|
||||
print(f"{PRINT_ULTRASTAR} Loading {print_blue_highlighted_text('whisper')} with model {print_blue_highlighted_text(model)} and {print_red_highlighted_text(device)} as worker")
|
||||
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
compute_type = "float16" if device == "cuda" else "int8" # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
|
||||
# transcribe with original whisper
|
||||
loaded_whisper_model = whisperx.load_model(model, device=device, compute_type=compute_type)
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
|
||||
language = result["language"]
|
||||
|
||||
# load alignment model and metadata
|
||||
model_a, metadata = whisperx.load_align_model(language_code=language, device=device)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(result["segments"], model_a, metadata, audio, device, return_char_alignments=False)
|
||||
|
||||
transcribed_data = []
|
||||
|
||||
for segment in result_aligned["segments"]:
|
||||
for obj in segment["words"]:
|
||||
if len(obj) < 4:
|
||||
print(f"{print_red_highlighted_text('Error: Skipping Word {}, because of missing timings'.format(obj['word']))}")
|
||||
continue
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
vtd.word = vtd.word + ' '
|
||||
transcribed_data.append(vtd) # and add it to list
|
||||
|
||||
return transcribed_data, language
|
||||
@@ -1,49 +0,0 @@
|
||||
from moduls.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
from moduls.Ultrastar.ultrastar_converter import get_end_time_from_ultrastar, get_start_time_from_ultrastar
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
|
||||
|
||||
def parse_ultrastar_txt(input_file):
|
||||
print(f"{PRINT_ULTRASTAR} Parse ultrastar txt -> {input_file}")
|
||||
|
||||
file = open(input_file, 'r', encoding='utf8')
|
||||
txt = file.readlines()
|
||||
|
||||
ultrastar_class = UltrastarTxt()
|
||||
count = 0
|
||||
|
||||
# Strips the newline character
|
||||
for line in txt:
|
||||
count += 1
|
||||
if line.startswith('#'):
|
||||
if line.startswith('#ARTIST'):
|
||||
ultrastar_class.artist = line.split(':')[1].replace('\n', '')
|
||||
elif line.startswith('#TITLE'):
|
||||
ultrastar_class.title = line.split(':')[1].replace('\n', '')
|
||||
elif line.startswith('#MP3'):
|
||||
ultrastar_class.mp3 = line.split(':')[1].replace('\n', '')
|
||||
elif line.startswith('#GAP'):
|
||||
ultrastar_class.gap = line.split(':')[1].replace('\n', '')
|
||||
elif line.startswith('#BPM'):
|
||||
ultrastar_class.bpm = line.split(':')[1].replace('\n', '')
|
||||
elif line.startswith(('F ', ': ', '* ', 'R ', 'G ')):
|
||||
parts = line.split()
|
||||
# [0] F : * R G
|
||||
# [1] start beat
|
||||
# [2] duration
|
||||
# [3] pitch
|
||||
# [4] word
|
||||
|
||||
ultrastar_class.noteType.append(parts[0])
|
||||
ultrastar_class.startBeat.append(parts[1])
|
||||
ultrastar_class.durations.append(parts[2])
|
||||
ultrastar_class.pitches.append(parts[3])
|
||||
ultrastar_class.words.append(parts[4] if len(parts) > 4 else '')
|
||||
|
||||
# do always as last
|
||||
pos = len(ultrastar_class.startBeat) - 1
|
||||
ultrastar_class.startTimes.append(get_start_time_from_ultrastar(ultrastar_class, pos))
|
||||
ultrastar_class.endTimes.append(get_end_time_from_ultrastar(ultrastar_class, pos))
|
||||
# todo: Progress?
|
||||
|
||||
return ultrastar_class
|
||||
@@ -1,111 +0,0 @@
|
||||
from moduls.Midi.midi_creator import create_midi_note_from_pitched_data
|
||||
from moduls.Ultrastar.ultrastar_converter import get_start_time_from_ultrastar, get_end_time_from_ultrastar, \
|
||||
ultrastar_note_to_midi_note
|
||||
from moduls.Log import PRINT_ULTRASTAR, print_blue_highlighted_text, print_gold_highlighted_text, \
|
||||
print_light_blue_highlighted_text, print_underlined_text, print_cyan_highlighted_text
|
||||
import librosa
|
||||
|
||||
MAX_SONG_SCORE = 10000
|
||||
MAX_SONG_LINE_BONUS = 1000
|
||||
|
||||
|
||||
class Points:
|
||||
notes = 0
|
||||
golden_notes = 0
|
||||
rap = 0
|
||||
golden_rap = 0
|
||||
line_bonus = 0
|
||||
parts = 0
|
||||
|
||||
|
||||
def add_point(noteType, points):
|
||||
if noteType == ':':
|
||||
points.notes += 1
|
||||
elif noteType == '*':
|
||||
points.golden_notes += 2
|
||||
elif noteType == 'R':
|
||||
points.rap += 1
|
||||
elif noteType == 'G':
|
||||
points.golden_rap += 2
|
||||
return points
|
||||
|
||||
class Score:
|
||||
max_score = 0
|
||||
notes = 0
|
||||
golden = 0
|
||||
line_bonus = 0
|
||||
score = 0
|
||||
|
||||
def get_score(points):
|
||||
score = Score()
|
||||
score.max_score = MAX_SONG_SCORE if points.line_bonus == 0 else MAX_SONG_SCORE - MAX_SONG_LINE_BONUS
|
||||
score.notes = round(score.max_score * (points.notes + points.rap) / points.parts)
|
||||
score.golden = round(points.golden_notes + points.golden_rap)
|
||||
score.score = round(score.notes + points.line_bonus + score.golden)
|
||||
score.line_bonus = round(points.line_bonus)
|
||||
return score
|
||||
|
||||
def print_score(score):
|
||||
print(f"{PRINT_ULTRASTAR} Total: {print_cyan_highlighted_text(str(score.score))}, notes: {print_blue_highlighted_text(str(score.notes))}, line bonus: {print_light_blue_highlighted_text(str(score.line_bonus))}, golden notes: {print_gold_highlighted_text(str(score.golden))}")
|
||||
|
||||
|
||||
def calculate_score(pitched_data, ultrastar_class):
|
||||
print(PRINT_ULTRASTAR + " Calculating Ultrastar Points")
|
||||
|
||||
simple_points = Points()
|
||||
accurate_points = Points()
|
||||
|
||||
reachable_line_bonus_per_word = MAX_SONG_LINE_BONUS / len(ultrastar_class.words)
|
||||
|
||||
for i in range(len(ultrastar_class.words)):
|
||||
if ultrastar_class.words == "":
|
||||
continue
|
||||
|
||||
if ultrastar_class.noteType[i] == 'F':
|
||||
continue
|
||||
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, i)
|
||||
duration = end_time - start_time
|
||||
step_size = 0.01 # todo: should be beat length ?
|
||||
parts = int(duration / step_size)
|
||||
parts = 1 if parts == 0 else parts
|
||||
|
||||
accurate_part_line_bonus_points = 0
|
||||
simple_part_line_bonus_points = 0
|
||||
|
||||
ultrastar_midi_note = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[i]))
|
||||
ultrastar_note = librosa.midi_to_note(ultrastar_midi_note)
|
||||
|
||||
for p in range(parts):
|
||||
st = start_time + step_size * p
|
||||
end = st + step_size
|
||||
pitch_note = create_midi_note_from_pitched_data(st, end, pitched_data)
|
||||
|
||||
if pitch_note[:-1] == ultrastar_note[:-1]:
|
||||
# Ignore octave high
|
||||
simple_points = add_point(ultrastar_class.noteType[i], simple_points)
|
||||
simple_part_line_bonus_points += 1
|
||||
|
||||
if pitch_note == ultrastar_note:
|
||||
# Octave high must be the same
|
||||
accurate_points = add_point(ultrastar_class.noteType[i], accurate_points)
|
||||
accurate_part_line_bonus_points += 1
|
||||
|
||||
accurate_points.parts += 1
|
||||
simple_points.parts += 1
|
||||
|
||||
if accurate_part_line_bonus_points >= parts:
|
||||
accurate_points.line_bonus += reachable_line_bonus_per_word
|
||||
|
||||
if simple_part_line_bonus_points >= parts:
|
||||
simple_points.line_bonus += reachable_line_bonus_per_word
|
||||
|
||||
return get_score(simple_points), get_score(accurate_points)
|
||||
|
||||
def print_score_calculation(simple_points, accurate_points):
|
||||
print(f"{PRINT_ULTRASTAR} {print_underlined_text('Simple (octave high ignored)')} points")
|
||||
print_score(simple_points)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} {print_underlined_text('Accurate (octave high matches)')} points:")
|
||||
print_score(accurate_points)
|
||||
@@ -1,17 +0,0 @@
|
||||
class UltrastarTxt:
|
||||
artist = ''
|
||||
title = ''
|
||||
mp3 = ''
|
||||
gap = ''
|
||||
bpm = ''
|
||||
cover = None
|
||||
creator = 'UltraSinger [GitHub]'
|
||||
fixer = 'YOUR NAME'
|
||||
comment = 'UltraSinger [GitHub]'
|
||||
startBeat = []
|
||||
startTimes = []
|
||||
endTimes = []
|
||||
durations = []
|
||||
pitches = []
|
||||
words = []
|
||||
noteType = []
|
||||
@@ -1,136 +0,0 @@
|
||||
from moduls.Ultrastar.ultrastar_converter import real_bpm_to_ultrastar_bpm, second_to_beat
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
import langcodes
|
||||
import re
|
||||
|
||||
|
||||
def get_multiplier(real_bpm):
|
||||
if real_bpm == 0:
|
||||
raise Exception("BPM is 0")
|
||||
|
||||
multiplier = 1
|
||||
result = 0
|
||||
while result < 400:
|
||||
result = real_bpm * multiplier
|
||||
multiplier += 1
|
||||
return multiplier - 2
|
||||
|
||||
|
||||
def get_language_name(language):
|
||||
return langcodes.Language.make(language=language).display_name()
|
||||
|
||||
|
||||
def create_ultrastar_txt_from_automation(transcribed_data, note_numbers, ultrastar_file_output, ultrastar_class,
|
||||
bpm=120):
|
||||
print(f"{PRINT_ULTRASTAR} Creating {ultrastar_file_output} from transcription.")
|
||||
|
||||
real_bpm = real_bpm_to_ultrastar_bpm(bpm)
|
||||
multiplication = get_multiplier(real_bpm)
|
||||
ultrastar_bpm = real_bpm * get_multiplier(real_bpm)
|
||||
|
||||
with open(ultrastar_file_output, 'w', encoding='utf8') as f:
|
||||
gap = transcribed_data[0].start
|
||||
|
||||
f.write(f'#ARTIST:{ultrastar_class.artist}\n')
|
||||
f.write(f'#TITLE:{ultrastar_class.title}\n')
|
||||
f.write(f'#CREATOR:{ultrastar_class.creator}\n')
|
||||
f.write(f'#FIXER:{ultrastar_class.fixer}\n')
|
||||
if ultrastar_class.language is not None:
|
||||
f.write(f'#LANGUAGE:{get_language_name(ultrastar_class.language)}\n')
|
||||
if ultrastar_class.cover is not None:
|
||||
f.write(f'#COVER:{ultrastar_class.cover}\n')
|
||||
f.write(f'#MP3:{ultrastar_class.mp3}\n')
|
||||
f.write(f'#VIDEO:{ultrastar_class.video}\n')
|
||||
f.write(f'#BPM:' + str(round(ultrastar_bpm, 2)) + '\n') # not the real BPM!
|
||||
f.write(f'#GAP:' + str(int(gap * 1000)) + '\n')
|
||||
f.write(f'#COMMENT:{ultrastar_class.comment}\n')
|
||||
|
||||
# Write the singing part
|
||||
previous_end_beat = 0
|
||||
for i in range(len(transcribed_data)):
|
||||
start_time = (transcribed_data[i].start - gap) * multiplication
|
||||
end_time = (transcribed_data[i].end - transcribed_data[i].start) * multiplication
|
||||
start_beat = round(second_to_beat(start_time, bpm))
|
||||
duration = round(second_to_beat(end_time, bpm))
|
||||
|
||||
# Fix the round issue, so the beats dont overlap
|
||||
if start_beat < previous_end_beat:
|
||||
start_beat = previous_end_beat
|
||||
previous_end_beat = start_beat + duration
|
||||
|
||||
# : 10 10 10 w
|
||||
# ':' start midi part
|
||||
# 'n1' start at real beat
|
||||
# 'n2' duration at real beat
|
||||
# 'n3' pitch where 0 == C4
|
||||
# 'w' lyric
|
||||
f.write(': ')
|
||||
f.write(str(start_beat) + ' ')
|
||||
f.write(str(duration) + ' ')
|
||||
f.write(str(note_numbers[i]) + ' ')
|
||||
f.write(transcribed_data[i].word)
|
||||
f.write('\n')
|
||||
|
||||
# detect silence between words
|
||||
if i < len(transcribed_data) - 1:
|
||||
silence = transcribed_data[i + 1].start - transcribed_data[i].end
|
||||
else:
|
||||
silence = 0
|
||||
|
||||
if silence > 0.3 and i != len(transcribed_data) - 1:
|
||||
# - 10
|
||||
# '-' end of current sing part
|
||||
# 'n1' show next at time in real beat
|
||||
f.write('- ')
|
||||
show_next = second_to_beat(transcribed_data[i].end - gap, bpm) * multiplication
|
||||
f.write(str(round(show_next)))
|
||||
f.write('\n')
|
||||
f.write('E')
|
||||
|
||||
|
||||
def create_repitched_txt_from_ultrastar_data(input_file, note_numbers, output_repitched_ultrastar):
|
||||
# todo: just add '_repitched' to input_file
|
||||
print("{PRINT_ULTRASTAR} Creating repitched ultrastar txt -> {input_file}_repitch.txt")
|
||||
|
||||
# todo: to reader
|
||||
file = open(input_file, 'r')
|
||||
txt = file.readlines()
|
||||
|
||||
i = 0
|
||||
# todo: just add '_repitched' to input_file
|
||||
with open(output_repitched_ultrastar, 'w', encoding='utf8') as f:
|
||||
for line in txt:
|
||||
if line.startswith(':'):
|
||||
parts = re.findall(r'\S+|\s+', line)
|
||||
# between are whitespaces
|
||||
# [0] :
|
||||
# [2] start beat
|
||||
# [4] duration
|
||||
# [6] pitch
|
||||
# [8] word
|
||||
parts[6] = str(note_numbers[i])
|
||||
delimiter = ''
|
||||
f.write(delimiter.join(parts))
|
||||
i += 1
|
||||
else:
|
||||
f.write(line)
|
||||
|
||||
def add_score_to_ultrastar_txt(ultrastar_file_output, score):
|
||||
with open(ultrastar_file_output, 'r', encoding='utf8') as f:
|
||||
text = f.read()
|
||||
text = text.split('\n')
|
||||
|
||||
for i in range(len(text)):
|
||||
if text[i].startswith("#COMMENT:"):
|
||||
text[i] = f"{text[i]} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
|
||||
break
|
||||
elif text[i].startswith(('F ', ': ', '* ', 'R ', 'G ')):
|
||||
text.insert(i, f"#COMMENT: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}")
|
||||
break
|
||||
|
||||
text = '\n'.join(text)
|
||||
|
||||
with open(ultrastar_file_output, 'w', encoding='utf8') as f:
|
||||
f.write(text)
|
||||
class UltraStarWriter:
|
||||
pass
|
||||
+4
-1
@@ -17,4 +17,7 @@ demucs~=4.0.0
|
||||
tqdm~=4.65.0
|
||||
langcodes~=3.3.0
|
||||
language-data~=1.1
|
||||
Pillow~=9.5.0
|
||||
Pillow~=9.5.0
|
||||
isort==5.12
|
||||
black==23.3
|
||||
pylint==2.17
|
||||
Reference in new issue
Block a user