This commit is contained in:
Vadim committed 2023-06-08 01:37:10 +02:00
commit 65d2dda38f
40 files changed
+1712 -1063

No files matched your search

+9 -9
View File
@@ -6,23 +6,23 @@ class Settings:
use_separated_vocal = True
create_karaoke = True
input_file_path = ''
output_file_path = ''
mono_audio_path = ''
input_file_path = ""
output_file_path = ""
mono_audio_path = ""
# Transcribe
audio_chunk_folder_name = 'audio-chunks'
audio_chunk_folder_name = "audio-chunks"
# Whisper
transcriber = 'whisper' # whisper|vosk
whisper_model = 'large-v2' # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
transcriber = "whisper" # whisper|vosk
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
# Vosk
vosk_model_path = '' # "models/vosk-model-small-en-us-0.15"
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
# Pitch
crepe_model_capacity = 'full' # tiny|small|medium|large|full
crepe_model_capacity = "full" # tiny|small|medium|large|full
crepe_step_size = 10
# Device
+449 -181
View File
@@ -1,73 +1,117 @@
import getopt
"""UltraSinger uses AI to automatically create UltraStar song files"""
import copy
import getopt
import os
import sys
from time import sleep
import Levenshtein
import librosa
from moduls import os_helper
from moduls.Audio.vocal_chunks import export_chunks_from_ultrastar_data, convert_audio_to_mono_wav, \
export_chunks_from_transcribed_data, remove_silence_from_transcribtion_data, convert_wav_to_mp3, \
export_transcribed_data_to_csv
from moduls.Audio.youtube import download_youtube_video, download_youtube_audio, get_youtube_title, download_youtube_thumbnail
from moduls.Audio.separation import separate_audio
from moduls.Audio.denoise import ffmpeg_reduce_noise
from moduls.Midi import midi_creator
from moduls.Midi.midi_creator import convert_frequencies_to_notes, most_frequent, create_midi_notes_from_pitched_data
from moduls.Pitcher.pitcher import get_frequency_with_high_confidence, get_pitch_with_crepe_file
from moduls.Ultrastar import ultrastar_parser, ultrastar_converter, ultrastar_writer, ultrastar_score_calculator
from moduls.Speech_Recognition.Vosk import transcribe_with_vosk
from moduls.Speech_Recognition.hyphenation import hyphenation, language_check
from moduls.Speech_Recognition.Whisper import transcribe_with_whisper
from moduls.Log import PRINT_ULTRASTAR, print_red_highlighted_text, print_blue_highlighted_text, \
print_gold_highlighted_text, \
print_light_blue_highlighted_text
from matplotlib import pyplot as plt
from moduls.Ultrastar.ultrastar_txt import UltrastarTxt
from Settings import Settings
from tqdm import tqdm
from moduls.DeviceDetection.device_detection import get_available_device
from time import sleep
from modules import os_helper
from modules.Audio.denoise import ffmpeg_reduce_noise
from modules.Audio.separation import separate_audio
from modules.Audio.vocal_chunks import (
export_chunks_from_transcribed_data,
export_chunks_from_ultrastar_data,
)
from modules.Audio.silence_processing import remove_silence_from_transcribtion_data
from modules.csv_export import export_transcribed_data_to_csv
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
from modules.Audio.youtube import (
download_youtube_audio,
download_youtube_thumbnail,
download_youtube_video,
get_youtube_title,
)
from modules.DeviceDetection.device_detection import get_available_device
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
gold_highlighted,
light_blue_highlighted,
red_highlighted,
)
from modules.Midi import midi_creator
from modules.Midi.midi_creator import (
convert_frequencies_to_notes,
create_midi_notes_from_pitched_data,
most_frequent,
)
from modules.Pitcher.pitcher import (
get_frequency_with_high_confidence,
get_pitch_with_crepe_file,
)
from modules.Speech_Recognition.hyphenation import hyphenation, language_check
from modules.Speech_Recognition.Vosk import transcribe_with_vosk
from modules.Speech_Recognition.Whisper import transcribe_with_whisper
from modules.Ultrastar import (
ultrastar_converter,
ultrastar_parser,
ultrastar_score_calculator,
ultrastar_writer,
)
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
from Settings import Settings
settings = Settings()
def get_confidence(pitched_data, threshold):
"""Docstring"""
# todo: replace get_frequency_with_high_conf from pitcher
conf_t = []
conf_f = []
conf_c = []
for i in range(len(pitched_data.times)):
for i in enumerate(pitched_data.times):
pos = i[0]
if pitched_data.confidence[i] > threshold:
conf_t.append(pitched_data.times[i])
conf_f.append(pitched_data.frequencies[i])
conf_c.append(pitched_data.confidence[i])
conf_t.append(pitched_data.times[pos])
conf_f.append(pitched_data.frequencies[pos])
conf_c.append(pitched_data.confidence[pos])
return conf_t, conf_f, conf_c
def convert_ultrastar_note_numbers(midi_notes):
print(f"{PRINT_ULTRASTAR} Creating Ultrastar notes from midi data")
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Creating Ultrastar notes from midi data")
ultrastar_note_numbers = []
for i in range(len(midi_notes)):
note_number_librosa = librosa.note_to_midi(midi_notes[i])
pitch = ultrastar_converter.midi_note_to_ultrastar_note(note_number_librosa)
for i, note in enumerate(midi_notes):
note_number_librosa = librosa.note_to_midi(note)
pitch = ultrastar_converter.midi_note_to_ultrastar_note(
note_number_librosa
)
ultrastar_note_numbers.append(pitch)
# todo: Progress?
# print("Note: " + midi_notes[i] + " midi_note: " + str(note_number_librosa) + ' pitch: ' + str(pitch))
# print(
# f"Note: {midi_notes[i]} midi_note: {str(note_number_librosa)} pitch: {str(pitch)}"
# )
return ultrastar_note_numbers
def pitch_each_chunk_with_crepe(directory):
print(f"{PRINT_ULTRASTAR} Pitching each chunk with {print_blue_highlighted_text('crepe')}")
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}"
)
midi_notes = []
for filename in sorted([f for f in os.listdir(directory) if f.endswith('.wav')],
key=lambda x: int(x.split("_")[1])):
for filename in sorted(
[f for f in os.listdir(directory) if f.endswith(".wav")],
key=lambda x: int(x.split("_")[1]),
):
filepath = os.path.join(directory, filename)
# todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes
pitched_data = get_pitch_with_crepe_file(filepath, settings.crepe_step_size, settings.crepe_model_capacity)
conf_f = get_frequency_with_high_confidence(pitched_data.frequencies, pitched_data.confidence)
pitched_data = get_pitch_with_crepe_file(
filepath, settings.crepe_step_size, settings.crepe_model_capacity
)
conf_f = get_frequency_with_high_confidence(
pitched_data.frequencies, pitched_data.confidence
)
notes = convert_frequencies_to_notes(conf_f)
note = most_frequent(notes)[0][0]
@@ -80,58 +124,69 @@ def pitch_each_chunk_with_crepe(directory):
def add_hyphen_to_data(transcribed_data, hyphen_words):
data = []
"""Docstring"""
new_data = []
for i in range(len(transcribed_data)):
for i, data in enumerate(transcribed_data):
if not hyphen_words[i]:
data.append(transcribed_data[i])
new_data.append(data)
else:
chunk_duration = transcribed_data[i].end - transcribed_data[i].start
chunk_duration = data.end - data.start
chunk_duration = chunk_duration / (len(hyphen_words[i]))
next_start = transcribed_data[i].start
for j in range(len(hyphen_words[i])):
dup = copy.copy(transcribed_data[i])
next_start = data.start
for j, hyphens in enumerate(hyphen_words[i]):
dup = copy.copy(data)
dup.start = next_start
next_start = transcribed_data[i].end - chunk_duration * (len(hyphen_words[i]) - 1 - j)
next_start = data.end - chunk_duration * (
len(hyphens) - 1 - j
)
dup.end = next_start
dup.word = hyphen_words[i][j]
dup.is_hyphen = True
data.append(dup)
new_data.append(dup)
return data
return new_data
def get_bpm_from_data(data, sr):
onset_env = librosa.onset.onset_strength(y=data, sr=sr)
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sr)
def get_bpm_from_data(data, sampling_rate):
"""Docstring"""
onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate)
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
print(f"{PRINT_ULTRASTAR} BPM is {print_blue_highlighted_text(str(round(wav_tempo[0], 2)))}")
print(
f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}"
)
return wav_tempo[0]
def get_bpm_from_file(wav_file):
data, sr = librosa.load(wav_file, sr=None)
return get_bpm_from_data(data, sr)
"""Docstring"""
data, sampling_rate = librosa.load(wav_file, sr=None)
return get_bpm_from_data(data, sampling_rate)
def correct_words(recognized_words, word_list_file):
with open(word_list_file, 'r') as f:
text = f.read()
"""Docstring"""
with open(word_list_file, "r", encoding="utf-8") as file:
text = file.read()
word_list = text.split()
for i, rec_word in enumerate(recognized_words):
if rec_word.word in word_list:
continue
else:
closest_word = min(word_list, key=lambda x: Levenshtein.distance(rec_word.word, x))
print(recognized_words[i].word + " - " + closest_word)
recognized_words[i].word = closest_word
closest_word = min(
word_list, key=lambda x: Levenshtein.distance(rec_word.word, x)
)
print(recognized_words[i].word + " - " + closest_word)
recognized_words[i].word = closest_word
return recognized_words
def print_help():
m = '''
"""Docstring"""
help_string = """
UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]
[opt]
@@ -167,72 +222,120 @@ def print_help():
[pitcher]
--crepe (default) tiny|small|medium|large|full
'''
print(m)
"""
print(help_string)
def plot(input_file, vosk_transcribed_data, midi_notes):
pitched_data = get_pitch_with_crepe_file(input_file, settings.crepe_step_size, settings.crepe_model_capacity)
t, f, c = get_confidence(pitched_data, 0.4)
"""Docstring"""
pitched_data = get_pitch_with_crepe_file(
input_file, settings.crepe_step_size, settings.crepe_model_capacity
)
conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4)
plt.ylim(0, 600)
plt.xlim(0, 50)
plt.plot(t, f, linewidth=0.1)
plt.plot(conf_t, conf_f, linewidth=0.1)
plt.savefig(os.path.join("test", "crepe_0.4.png"))
for i in range(len(vosk_transcribed_data)):
nf = librosa.note_to_hz(midi_notes[i])
plt.plot([vosk_transcribed_data[i].start, vosk_transcribed_data[i].end], [nf, nf], linewidth=1, alpha=0.5)
for i, data in enumerate(vosk_transcribed_data):
note_frequency = librosa.note_to_hz(midi_notes[i])
plt.plot(
[data.start, data.end],
[note_frequency, note_frequency],
linewidth=1,
alpha=0.5,
)
plt.savefig(os.path.join("test", "pit.png"), dpi=2000)
def remove_unecessary_punctuations(transcribed_data):
"""Docstring"""
punctuation = ".,"
for i in range(len(transcribed_data)):
transcribed_data[i].word = transcribed_data[i].word.translate({ord(i): None for i in punctuation})
for i, data in enumerate(transcribed_data):
data.word = data.word.translate(
{ord(i): None for i in punctuation}
)
def hyphenate_each_word(language, transcribed_data):
"""Docstring"""
hyphenated_word = []
lang_region = language_check(language)
if lang_region is None:
print(f"{PRINT_ULTRASTAR} {print_red_highlighted_text('Error in hyphenation for language ')} {print_blue_highlighted_text(language)} {print_red_highlighted_text(', maybe you want to disable it?')}")
print(
f"{ULTRASINGER_HEAD} {red_highlighted('Error in hyphenation for language ')} {blue_highlighted(language)} {red_highlighted(', maybe you want to disable it?')}"
)
return None
sleep(.1)
for i in tqdm(range(len(transcribed_data))):
hyphenated_word.append(hyphenation(transcribed_data[i].word, lang_region))
sleep(0.1)
for i, data in tqdm(enumerate(transcribed_data)):
hyphenated_word.append(
hyphenation(data.word, lang_region)
)
return hyphenated_word
def print_support():
"""Docstring"""
print()
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('Do you like UltraSinger? And want it to be even better? Then help with your')} {print_light_blue_highlighted_text('support')}{print_gold_highlighted_text('!')}")
print(f"{PRINT_ULTRASTAR} See project page -> https://github.com/rakuri255/UltraSinger")
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('This will help alot to keep this project alive and improved.')}")
print(
f"{ULTRASINGER_HEAD} {gold_highlighted('Do you like UltraSinger? And want it to be even better? Then help with your')} {light_blue_highlighted('support')}{gold_highlighted('!')}"
)
print(
f"{ULTRASINGER_HEAD} See project page -> https://github.com/rakuri255/UltraSinger"
)
print(
f"{ULTRASINGER_HEAD} {gold_highlighted('This will help alot to keep this project alive and improved.')}"
)
def run():
isAudio = ".txt" not in settings.input_file_path
"""Docstring"""
is_audio = ".txt" not in settings.input_file_path
ultrastar_class = None
real_bpm = None
if not isAudio: # Parse Ultrastar txt
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('re-pitch mode')}")
basename_without_ext, real_bpm, song_output, ultrastar_audio_input_path, ultrastar_class = parse_ultrastar_txt()
elif settings.input_file_path.startswith('https:'): # Youtube
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('full automatic mode')}")
basename_without_ext, song_output, ultrastar_audio_input_path = download_from_youtube()
if not is_audio: # Parse Ultrastar txt
print(
f"{ULTRASINGER_HEAD} {gold_highlighted('re-pitch mode')}"
)
(
basename_without_ext,
real_bpm,
song_output,
ultrastar_audio_input_path,
ultrastar_class,
) = parse_ultrastar_txt()
elif settings.input_file_path.startswith("https:"): # Youtube
print(
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
)
(
basename_without_ext,
song_output,
ultrastar_audio_input_path,
) = download_from_youtube()
else: # Audio File
print(f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('full automatic mode')}")
basename_without_ext, song_output, ultrastar_audio_input_path = setup_audio_input_file()
print(
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
)
(
basename_without_ext,
song_output,
ultrastar_audio_input_path,
) = setup_audio_input_file()
cache_path = os.path.join(song_output, 'cache')
settings.mono_audio_path = os.path.join(cache_path, basename_without_ext + '.wav')
cache_path = os.path.join(song_output, "cache")
settings.mono_audio_path = os.path.join(
cache_path, basename_without_ext + ".wav"
)
os_helper.create_folder(cache_path)
# Separate vocal from audio
audio_separation_path = separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_input_path)
audio_separation_path = separate_vocal_from_audio(
basename_without_ext, cache_path, ultrastar_audio_input_path
)
# Denoise vocal audio
denoise_vocal_audio(basename_without_ext, cache_path)
@@ -240,15 +343,19 @@ def run():
# Audio transcription
transcribed_data = None
language = None
if isAudio:
if is_audio:
language, transcribed_data = transcribe_audio(transcribed_data)
remove_unecessary_punctuations(transcribed_data)
transcribed_data = remove_silence_from_transcribtion_data(settings.mono_audio_path, transcribed_data)
transcribed_data = remove_silence_from_transcribtion_data(
settings.mono_audio_path, transcribed_data
)
if settings.hyphenation:
hyphen_words = hyphenate_each_word(language, transcribed_data)
if hyphen_words is not None:
transcribed_data = add_hyphen_to_data(transcribed_data, hyphen_words)
transcribed_data = add_hyphen_to_data(
transcribed_data, hyphen_words
)
# todo: do we need to correct words?
# lyric = 'input/faber_lyric.txt'
@@ -256,41 +363,59 @@ def run():
# Create audio chunks
if settings.create_audio_chunks:
create_audio_chunks(cache_path, isAudio, transcribed_data, ultrastar_audio_input_path, ultrastar_class)
create_audio_chunks(
cache_path,
is_audio,
transcribed_data,
ultrastar_audio_input_path,
ultrastar_class,
)
# Pitch the audio
midi_notes, pitched_data, ultrastar_note_numbers = pitch_audio(isAudio, transcribed_data, ultrastar_class)
midi_notes, pitched_data, ultrastar_note_numbers = pitch_audio(
is_audio, transcribed_data, ultrastar_class
)
# Create plot
if isAudio and settings.create_plot:
if is_audio and settings.create_plot:
plot(ultrastar_audio_input_path, transcribed_data, midi_notes)
# Write Ultrastar txt
if isAudio:
real_bpm, ultrastar_file_output = create_ultrastar_txt_from_automation(audio_separation_path,
basename_without_ext,
song_output, transcribed_data,
ultrastar_audio_input_path,
ultrastar_note_numbers, language)
if is_audio:
real_bpm, ultrastar_file_output = create_ultrastar_txt_from_automation(
audio_separation_path,
basename_without_ext,
song_output,
transcribed_data,
ultrastar_audio_input_path,
ultrastar_note_numbers,
language,
)
else:
ultrastar_file_output = create_ultrastar_txt_from_ultrastar_data(song_output, ultrastar_class,
ultrastar_note_numbers)
ultrastar_file_output = create_ultrastar_txt_from_ultrastar_data(
song_output, ultrastar_class, ultrastar_note_numbers
)
# Calc Points
ultrastar_class, simple_score, accurate_score = calculate_score_points(isAudio, pitched_data, ultrastar_class, ultrastar_file_output)
ultrastar_class, simple_score, accurate_score = calculate_score_points(
is_audio, pitched_data, ultrastar_class, ultrastar_file_output
)
# Add calculated score to Ultrastar txt
ultrastar_writer.add_score_to_ultrastar_txt(ultrastar_file_output, simple_score)
ultrastar_writer.add_score_to_ultrastar_txt(
ultrastar_file_output, simple_score
)
# Midi
if settings.create_midi:
create_midi_file(isAudio, real_bpm, song_output, ultrastar_class)
create_midi_file(is_audio, real_bpm, song_output, ultrastar_class)
# Print Support
print_support()
def get_unused_song_output_dir(path):
"""Docstring"""
# check if dir exists and add (i) if it does
i = 1
if os_helper.check_if_folder_exists(path):
@@ -302,25 +427,35 @@ def get_unused_song_output_dir(path):
path = path.replace(f"({i - 1})", f"({i})")
i += 1
if i > 999:
print(f"{PRINT_ULTRASTAR} {print_red_highlighted_text('Error: Could not create output folder! (999) is the maximum number of tries.')}")
print(
f"{ULTRASINGER_HEAD} {red_highlighted('Error: Could not create output folder! (999) is the maximum number of tries.')}"
)
sys.exit(1)
return path
def transcribe_audio(transcribed_data):
"""Docstring"""
if settings.transcriber == "whisper":
device = "cpu" if settings.force_whisper_cpu else settings.device
transcribed_data, language = transcribe_with_whisper(settings.mono_audio_path, settings.whisper_model,
device)
transcribed_data, language = transcribe_with_whisper(
settings.mono_audio_path, settings.whisper_model, device)
else: # vosk
transcribed_data = transcribe_with_vosk(settings.mono_audio_path, settings.vosk_model_path)
transcribed_data = transcribe_with_vosk(
settings.mono_audio_path, settings.vosk_model_path
)
# todo: make language selectable
language = 'en'
language = "en"
return language, transcribed_data
def separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_input_path):
audio_separation_path = os.path.join(cache_path, "separated", "htdemucs", basename_without_ext)
def separate_vocal_from_audio(
basename_without_ext, cache_path, ultrastar_audio_input_path
):
"""Docstring"""
audio_separation_path = os.path.join(
cache_path, "separated", "htdemucs", basename_without_ext
)
device = "cpu" if settings.force_separation_cpu else settings.device
if settings.use_separated_vocal or settings.create_karaoke:
separate_audio(ultrastar_audio_input_path, cache_path, device)
@@ -328,36 +463,85 @@ def separate_vocal_from_audio(basename_without_ext, cache_path, ultrastar_audio_
vocals_path = os.path.join(audio_separation_path, "vocals.wav")
convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path)
else:
convert_audio_to_mono_wav(ultrastar_audio_input_path, settings.mono_audio_path)
convert_audio_to_mono_wav(
ultrastar_audio_input_path, settings.mono_audio_path
)
return audio_separation_path
def calculate_score_points(isAudio, pitched_data, ultrastar_class, ultrastar_file_output):
if isAudio:
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output)
simple_score, accurate_score = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
def calculate_score_points(
is_audio, pitched_data, ultrastar_class, ultrastar_file_output
):
"""Docstring"""
if is_audio:
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
ultrastar_file_output
)
(
simple_score,
accurate_score,
) = ultrastar_score_calculator.calculate_score(
pitched_data, ultrastar_class
)
ultrastar_score_calculator.print_score_calculation(
simple_score, accurate_score
)
else:
print(f"{PRINT_ULTRASTAR} {print_blue_highlighted_text('Score of original Ultrastar txt')}")
simple_score, accurate_score = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
print(f"{PRINT_ULTRASTAR} {print_blue_highlighted_text('Score of re-pitched Ultrastar txt')}")
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output)
simple_score, accurate_score = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
print(
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of original Ultrastar txt')}"
)
(
simple_score,
accurate_score,
) = ultrastar_score_calculator.calculate_score(
pitched_data, ultrastar_class
)
ultrastar_score_calculator.print_score_calculation(
simple_score, accurate_score
)
print(
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of re-pitched Ultrastar txt')}"
)
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
ultrastar_file_output
)
(
simple_score,
accurate_score,
) = ultrastar_score_calculator.calculate_score(
pitched_data, ultrastar_class
)
ultrastar_score_calculator.print_score_calculation(
simple_score, accurate_score
)
return ultrastar_class, simple_score, accurate_score
def create_ultrastar_txt_from_ultrastar_data(song_output, ultrastar_class, ultrastar_note_numbers):
output_repitched_ultrastar = os.path.join(song_output, ultrastar_class.title + '.txt')
ultrastar_writer.create_repitched_txt_from_ultrastar_data(settings.input_file_path, ultrastar_note_numbers,
output_repitched_ultrastar)
def create_ultrastar_txt_from_ultrastar_data(
song_output, ultrastar_class, ultrastar_note_numbers
):
"""Docstring"""
output_repitched_ultrastar = os.path.join(
song_output, ultrastar_class.title + ".txt"
)
ultrastar_writer.create_repitched_txt_from_ultrastar_data(
settings.input_file_path,
ultrastar_note_numbers,
output_repitched_ultrastar,
)
return output_repitched_ultrastar
def create_ultrastar_txt_from_automation(audio_separation_path, basename_without_ext, song_output,
transcribed_data, ultrastar_audio_input_path, ultrastar_note_numbers,
language):
def create_ultrastar_txt_from_automation(
audio_separation_path,
basename_without_ext,
song_output,
transcribed_data,
ultrastar_audio_input_path,
ultrastar_note_numbers,
language,
):
"""Docstring"""
ultrastar_header = UltrastarTxt()
ultrastar_header.title = basename_without_ext
ultrastar_header.artist = basename_without_ext
@@ -365,13 +549,23 @@ def create_ultrastar_txt_from_automation(audio_separation_path, basename_without
ultrastar_header.video = basename_without_ext + ".mp4"
ultrastar_header.language = language
cover = basename_without_ext + " [CO].jpg"
ultrastar_header.cover = cover if os_helper.check_file_exists(os.path.join(song_output, cover)) else None
ultrastar_header.cover = (
cover
if os_helper.check_file_exists(os.path.join(song_output, cover))
else None
)
real_bpm = get_bpm_from_file(ultrastar_audio_input_path)
ultrastar_file_output = os.path.join(song_output, basename_without_ext + '.txt')
ultrastar_writer.create_ultrastar_txt_from_automation(transcribed_data, ultrastar_note_numbers,
ultrastar_file_output,
ultrastar_header, real_bpm)
ultrastar_file_output = os.path.join(
song_output, basename_without_ext + ".txt"
)
ultrastar_writer.create_ultrastar_txt_from_automation(
transcribed_data,
ultrastar_note_numbers,
ultrastar_file_output,
ultrastar_header,
real_bpm,
)
if settings.create_karaoke:
no_vocals_path = os.path.join(audio_separation_path, "no_vocals.wav")
title = basename_without_ext + " [Karaoke]"
@@ -381,13 +575,18 @@ def create_ultrastar_txt_from_automation(audio_separation_path, basename_without
karaoke_audio_output_path = karaoke_output_path + ".mp3"
convert_wav_to_mp3(no_vocals_path, karaoke_audio_output_path)
karaoke_txt_output_path = karaoke_output_path + ".txt"
ultrastar_writer.create_ultrastar_txt_from_automation(transcribed_data, ultrastar_note_numbers,
karaoke_txt_output_path,
ultrastar_header, real_bpm)
ultrastar_writer.create_ultrastar_txt_from_automation(
transcribed_data,
ultrastar_note_numbers,
karaoke_txt_output_path,
ultrastar_header,
real_bpm,
)
return real_bpm, ultrastar_file_output
def setup_audio_input_file():
"""Docstring"""
basename = os.path.basename(settings.input_file_path)
basename_without_ext = os.path.splitext(basename)[0]
song_output = os.path.join(settings.output_file_path, basename_without_ext)
@@ -402,6 +601,7 @@ FILENAME_REPLACEMENTS = (('?:"', ""), ("<", "("), (">", ")"), ("/\\|*", "-"))
def sanitize_filename(fname: str) -> str:
"""Docstring"""
for old, new in FILENAME_REPLACEMENTS:
for char in old:
fname = fname.replace(char, new)
@@ -411,82 +611,138 @@ def sanitize_filename(fname: str) -> str:
def download_from_youtube():
"""Docstring"""
title = get_youtube_title(settings.input_file_path)
basename_without_ext = sanitize_filename(title)
basename = basename_without_ext + '.mp3'
basename = basename_without_ext + ".mp3"
song_output = os.path.join(settings.output_file_path, basename_without_ext)
song_output = get_unused_song_output_dir(song_output)
os_helper.create_folder(song_output)
download_youtube_audio(settings.input_file_path, basename_without_ext, song_output)
download_youtube_video(settings.input_file_path, basename_without_ext, song_output)
download_youtube_thumbnail(settings.input_file_path, basename_without_ext, song_output)
download_youtube_audio(
settings.input_file_path, basename_without_ext, song_output
)
download_youtube_video(
settings.input_file_path, basename_without_ext, song_output
)
download_youtube_thumbnail(
settings.input_file_path, basename_without_ext, song_output
)
ultrastar_audio_input_path = os.path.join(song_output, basename)
return basename_without_ext, song_output, ultrastar_audio_input_path
def parse_ultrastar_txt():
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(settings.input_file_path)
real_bpm = ultrastar_converter.ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(',', '.')))
"""Docstring"""
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
settings.input_file_path
)
real_bpm = ultrastar_converter.ultrastar_bpm_to_real_bpm(
float(ultrastar_class.bpm.replace(",", "."))
)
ultrastar_mp3_name = ultrastar_class.mp3
basename_without_ext = os.path.splitext(ultrastar_mp3_name)[0]
dirname = os.path.dirname(settings.input_file_path)
ultrastar_audio_input_path = os.path.join(dirname, ultrastar_mp3_name)
song_output = os.path.join(settings.output_file_path, ultrastar_class.artist + ' - ' + ultrastar_class.title)
song_output = os.path.join(
settings.output_file_path,
ultrastar_class.artist + " - " + ultrastar_class.title,
)
song_output = get_unused_song_output_dir(song_output)
os_helper.create_folder(song_output)
return basename_without_ext, real_bpm, song_output, ultrastar_audio_input_path, ultrastar_class
return (
basename_without_ext,
real_bpm,
song_output,
ultrastar_audio_input_path,
ultrastar_class,
)
def create_midi_file(isAudio, real_bpm, song_output, ultrastar_class):
print(f"{PRINT_ULTRASTAR} Creating Midi with {print_blue_highlighted_text('pretty_midi')}")
if isAudio:
voice_instrument = [midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)]
midi_output = os.path.join(song_output, ultrastar_class.title + '.mid')
midi_creator.instruments_to_midi(voice_instrument, real_bpm, midi_output)
def create_midi_file(is_audio, real_bpm, song_output, ultrastar_class):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}"
)
if is_audio:
voice_instrument = [
midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)
]
midi_output = os.path.join(song_output, ultrastar_class.title + ".mid")
midi_creator.instruments_to_midi(
voice_instrument, real_bpm, midi_output
)
else:
voice_instrument = [midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)]
midi_output = os.path.join(song_output, ultrastar_class.title + '.mid')
midi_creator.instruments_to_midi(voice_instrument, real_bpm, midi_output)
voice_instrument = [
midi_creator.convert_ultrastar_to_midi_instrument(ultrastar_class)
]
midi_output = os.path.join(song_output, ultrastar_class.title + ".mid")
midi_creator.instruments_to_midi(
voice_instrument, real_bpm, midi_output
)
def pitch_audio(isAudio, transcribed_data, ultrastar_class):
def pitch_audio(is_audio, transcribed_data, ultrastar_class):
"""Docstring"""
# todo: chunk pitching as option?
# midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name)
pitched_data = get_pitch_with_crepe_file(settings.mono_audio_path, settings.crepe_step_size,
settings.crepe_model_capacity)
if isAudio:
pitched_data = get_pitch_with_crepe_file(
settings.mono_audio_path,
settings.crepe_step_size,
settings.crepe_model_capacity,
)
if is_audio:
start_times = []
end_times = []
for i in range(len(transcribed_data)):
start_times.append(transcribed_data[i].start)
end_times.append(transcribed_data[i].end)
midi_notes = create_midi_notes_from_pitched_data(start_times, end_times, pitched_data)
for i, data in enumerate(transcribed_data):
start_times.append(data.start)
end_times.append(data.end)
midi_notes = create_midi_notes_from_pitched_data(
start_times, end_times, pitched_data
)
else:
midi_notes = create_midi_notes_from_pitched_data(ultrastar_class.startTimes, ultrastar_class.endTimes,
pitched_data)
midi_notes = create_midi_notes_from_pitched_data(
ultrastar_class.startTimes, ultrastar_class.endTimes, pitched_data
)
ultrastar_note_numbers = convert_ultrastar_note_numbers(midi_notes)
return midi_notes, pitched_data, ultrastar_note_numbers
def create_audio_chunks(cache_path, isAudio, transcribed_data, ultrastar_audio_input_path, ultrastar_class):
audio_chunks_path = os.path.join(cache_path, settings.audio_chunk_folder_name)
def create_audio_chunks(
cache_path,
is_audio,
transcribed_data,
ultrastar_audio_input_path,
ultrastar_class,
):
"""Docstring"""
audio_chunks_path = os.path.join(
cache_path, settings.audio_chunk_folder_name
)
os_helper.create_folder(audio_chunks_path)
if isAudio: # and csv
if is_audio: # and csv
csv_filename = os.path.join(audio_chunks_path, "_chunks.csv")
export_chunks_from_transcribed_data(settings.mono_audio_path, transcribed_data, audio_chunks_path)
export_chunks_from_transcribed_data(
settings.mono_audio_path, transcribed_data, audio_chunks_path
)
export_transcribed_data_to_csv(transcribed_data, csv_filename)
else:
export_chunks_from_ultrastar_data(ultrastar_audio_input_path, ultrastar_class, audio_chunks_path)
export_chunks_from_ultrastar_data(
ultrastar_audio_input_path, ultrastar_class, audio_chunks_path
)
def denoise_vocal_audio(basename_without_ext, cache_path):
denoised_path = os.path.join(cache_path, basename_without_ext + '_denoised.wav')
"""Docstring"""
denoised_path = os.path.join(
cache_path, basename_without_ext + "_denoised.wav"
)
ffmpeg_reduce_noise(settings.mono_audio_path, denoised_path)
settings.mono_audio_path = denoised_path
def main(argv):
"""Docstring"""
init_settings(argv)
run()
# todo: cleanup
@@ -494,15 +750,27 @@ def main(argv):
def init_settings(argv):
"""Docstring"""
short = "hi:o:amv:"
long = ["ifile=", "ofile=", "crepe=", "vosk=", "whisper=", "hyphenation=", "disable_separation=",
"disable_karaoke=", "create_audio_chunks=", "force_whisper_cpu=", "force_separation_cpu="]
long = [
"ifile=",
"ofile=",
"crepe=",
"vosk=",
"whisper=",
"hyphenation=",
"disable_separation=",
"disable_karaoke=",
"create_audio_chunks=",
"force_whisper_cpu=",
"force_separation_cpu="
]
opts, args = getopt.getopt(argv, short, long)
if len(opts) == 0:
print_help()
sys.exit()
for opt, arg in opts:
if opt == '-h':
if opt == "-h":
print_help()
sys.exit()
elif opt in ("-i", "--ifile"):
@@ -513,7 +781,7 @@ def init_settings(argv):
settings.transcriber = "whisper"
settings.whisper_model = arg
elif opt in ("--vosk"):
settings.transcriber = 'vosk'
settings.transcriber = "vosk"
settings.vosk_model_path = arg
elif opt in ("--crepe"):
settings.crepe_model_capacity = arg
@@ -529,12 +797,12 @@ def init_settings(argv):
settings.force_whisper_cpu = arg
elif opt in ("--force_separation_cpu"):
settings.force_separation_cpu = arg
if settings.output_file_path == '':
if settings.input_file_path.startswith('https:'):
if settings.output_file_path == "":
if settings.input_file_path.startswith("https:"):
dirname = os.getcwd()
else:
dirname = os.path.dirname(settings.input_file_path)
settings.output_file_path = os.path.join(dirname, 'output')
settings.output_file_path = os.path.join(dirname, "output")
settings.device = get_available_device()
+28
View File
@@ -0,0 +1,28 @@
"""Convert audio to other formats"""
from pydub import AudioSegment
from modules.console_colors import ULTRASINGER_HEAD
def convert_audio_to_mono_wav(input_file: str, output_file: str) -> None:
"""Convert audio to mono wav"""
print(f"{ULTRASINGER_HEAD} Converting audio for AI")
if ".mp3" in input_file:
sound = AudioSegment.from_mp3(input_file)
elif ".wav" in input_file:
sound = AudioSegment.from_wav(input_file)
else:
raise ValueError("data format not supported")
sound = sound.set_channels(1)
sound.export(output_file, format="wav")
def convert_wav_to_mp3(input_file: str, output_file: str) -> None:
"""Convert wav to mp3"""
print(f"{ULTRASINGER_HEAD} Converting wav to mp3")
sound = AudioSegment.from_wav(input_file)
sound.export(output_file, format="mp3")
+30
View File
@@ -0,0 +1,30 @@
"""Reduce noise from audio"""
import ffmpeg
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None:
"""Reduce noise from vocal audio with ffmpeg."""
# Denoise audio samples with FFT.
# A description of the accepted parameters follows.
# noise_reduction, nr
# Set the noise reduction in dB, allowed range is 0.01 to 97. Default value is 12 dB.
# noise_floor, nf
# Set the noise floor in dB, allowed range is -80 to -20. Default value is -50 dB.
# track_noise, tn
# Enable noise floor tracking. By default is disabled.
# With this enabled, noise floor is automatically adjusted.
print(
f"{ULTRASINGER_HEAD} Reduce noise from vocal audio with {blue_highlighted('ffmpeg')}."
)
(
ffmpeg.input(input_file_path)
.output(output_file, af="afftdn=nr=70:nf=-50:tn=1")
.overwrite_output()
.run()
)
+27
View File
@@ -0,0 +1,27 @@
"""Separate vocals from audio"""
import subprocess
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.os_helper import current_executor_path, move, path_join
def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None:
"""Separate vocals from audio with demucs."""
print(
f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker."
)
# Model selection?
# -n mdx_q
# -n htdemucs_ft
device = "cpu" if device != "cuda" else "cuda"
subprocess.run(
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
)
separated_folder = path_join(current_executor_path(), "separated")
move(separated_folder, output_file)
+53
View File
@@ -0,0 +1,53 @@
"""Silence processing module"""
import librosa
from modules.console_colors import ULTRASINGER_HEAD
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: []) -> []:
"""Remove silence from given transcribtion data"""
print(
f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data"
)
audio, sample_rate = librosa.load(audio_path, sr=None)
for i, data in enumerate(transcribed_data):
start_time = data.start
end_time = data.end
start_sample = int(start_time * sample_rate)
end_sample = int(end_time * sample_rate)
chunk = audio[start_sample:end_sample]
# todo: why 5 works good? It should be 40db ?!?
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
silence_threshold = 5
onsets = librosa.effects.split(
chunk, top_db=silence_threshold, frame_length=2048, hop_length=100
)
# Get the duration of the first and last silent intervals
if len(onsets) > 0:
first_silence = onsets[0][0]
last_silence = len(chunk) - onsets[-1][1]
first_silence_duration = librosa.samples_to_time(
first_silence, sr=sample_rate
)
last_silence_duration = librosa.samples_to_time(
last_silence, sr=sample_rate
)
else:
first_silence_duration = 0
last_silence_duration = 0
data.start = (
data.start + first_silence_duration
)
data.end = (
data.end - last_silence_duration
)
return transcribed_data
+89
View File
@@ -0,0 +1,89 @@
"""Vocal chunks module."""
import os
import re
import wave
from modules.console_colors import ULTRASINGER_HEAD
from modules.os_helper import create_folder
from modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
)
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
class AudioManipulation:
"""Docstring"""
def export_chunks_from_transcribed_data(
audio_filename: str, transcribed_data: [], output_folder_name: str
) -> None:
"""Export transcribed_data as vocal chunks wav files"""
print(
f"{ULTRASINGER_HEAD} Export transcribed data as vocal chunks wav files"
)
wave_file = wave.open(audio_filename, "rb")
sample_rate, n_channels = wave_file.getparams()[2], wave_file.getparams()[0]
for i, data in enumerate(transcribed_data):
start_byte = int(data.start * sample_rate * n_channels)
end_byte = int(data.end * sample_rate * n_channels)
chunk = get_chunk(end_byte, start_byte, wave_file)
export_chunk_to_wav_file(
chunk, output_folder_name, i, data.word, wave_file
)
wave_file.close()
def export_chunks_from_ultrastar_data(
audio_filename: str, ultrastar_data: UltrastarTxt, folder_name: str
) -> None:
"""Export ultrastar data as vocal chunks wav files"""
print(f"{ULTRASINGER_HEAD} Export Ultrastar data as vocal chunks wav files")
create_folder(folder_name)
wave_file = wave.open(audio_filename, "rb")
sample_rate, n_channels = wave_file.getparams()[2], wave_file.getparams()[0]
for i, word in enumerate(ultrastar_data.words):
start_time = get_start_time_from_ultrastar(ultrastar_data, i)
end_time = get_end_time_from_ultrastar(ultrastar_data, i)
start_byte = int(start_time * sample_rate * n_channels)
end_byte = int(end_time * sample_rate * n_channels)
chunk = get_chunk(end_byte, start_byte, wave_file)
export_chunk_to_wav_file(
chunk, folder_name, i, word, wave_file
)
def export_chunk_to_wav_file(chunk, folder_name: str, i: int, word: str, wave_file) -> None:
"""Export vocal chunks to wav file"""
clean_word = re.sub("[^A-Za-z0-9]+", "", word)
# todo: Progress?
# print(f"{str(i)} {clean_word}")
with wave.open(
os.path.join(folder_name, f"chunk_{i}_{clean_word}.wav"), "wb"
) as chunk_file:
chunk_file.setparams(wave_file.getparams())
chunk_file.writeframes(chunk)
def get_chunk(end_byte: int, start_byte: int, wave_file):
"""
Gets the chunk from wave file.
Returns chunk as n frames of audio, as a bytes object.
"""
# todo: get out of position error message
wave_file.setpos(start_byte) # ({:.2f})
chunk = wave_file.readframes(end_byte - start_byte)
return chunk
+83
View File
@@ -0,0 +1,83 @@
"""YouTube Downloader"""
import io
import os
import yt_dlp
from PIL import Image
from modules.console_colors import ULTRASINGER_HEAD
def get_youtube_title(url: str) -> str:
"""Get the title of the YouTube video"""
ydl_opts = {}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
result = ydl.extract_info(
url, download=False # We just want to extract the info
)
return result["title"]
def download_youtube_audio(url: str, clear_filename: str, output_path: str):
"""Download audio from YouTube"""
print(f"{ULTRASINGER_HEAD} Downloading Audio")
ydl_opts = {
"format": "bestaudio/best",
"outtmpl": output_path + "/" + clear_filename,
"postprocessors": [
{"key": "FFmpegExtractAudio", "preferredcodec": "mp3"}
],
}
start_download(ydl_opts, url)
def download_youtube_thumbnail(url: str, clear_filename: str, output_path: str):
"""Download thumbnail from YouTube"""
print(f"{ULTRASINGER_HEAD} Downloading thumbnail")
ydl_opts = {
"skip_download": True,
"writethumbnail": True,
}
download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path)
def download_and_convert_thumbnail(ydl_opts, url: str, clear_filename: str, output_path: str) -> None:
"""Download and convert thumbnail from YouTube"""
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info_dict = ydl.extract_info(url, download=False)
thumbnail_url = info_dict.get("thumbnail")
if thumbnail_url:
response = ydl.urlopen(thumbnail_url)
image_data = response.read()
image = Image.open(io.BytesIO(image_data))
image.save(
os.path.join(output_path, clear_filename + " [CO].jpg"), "JPEG"
)
def download_youtube_video(url: str, clear_filename: str, output_path: str) -> None:
"""Download video from YouTube"""
print(f"{ULTRASINGER_HEAD} Downloading Video")
ydl_opts = {
"format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4",
"outtmpl": output_path + "/" + clear_filename + ".mp4",
}
start_download(ydl_opts, url)
def start_download(ydl_opts, url: str) -> None:
"""Start the download the ydl_opts"""
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
errors = ydl.download(url)
if errors:
raise Exception("Download failed with error: " + str(errors))
@@ -0,0 +1,21 @@
"""Docstring"""
import torch
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted
def get_available_device():
"""Docstring"""
isCuda = torch.cuda.is_available()
if not isCuda:
print(
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
)
return "cpu"
else:
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
return "cuda"
+127
View File
@@ -0,0 +1,127 @@
"""Docstring"""
import math
from collections import Counter
import librosa
import numpy as np
import pretty_midi
from modules.Pitcher.pitcher import get_frequency_with_high_confidence
from modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
ultrastar_note_to_midi_note,
)
from modules.console_colors import (
ULTRASINGER_HEAD,
red_highlighted,
)
def convert_ultrastar_to_midi_instrument(ultrastar_class):
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Creating midi instrument from Ultrastar txt")
instrument = pretty_midi.Instrument(program=0)
velocity = 100
for i in enumerate(ultrastar_class.words):
pos = i[0]
start_time = get_start_time_from_ultrastar(ultrastar_class, pos)
end_time = get_end_time_from_ultrastar(ultrastar_class, pos)
pitch = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[pos]))
note = pretty_midi.Note(velocity, pitch, start_time, end_time)
instrument.notes.append(note)
return instrument
def instruments_to_midi(instruments, bpm, midi_output):
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Creating midi file -> {midi_output}")
midi_data = pretty_midi.PrettyMIDI(initial_tempo=bpm)
for instrument in instruments:
midi_data.instruments.append(instrument)
midi_data.write(midi_output)
class MidiCreator:
"""Docstring"""
def convert_frequencies_to_notes(frequency):
"""Docstring"""
notes = []
for freq in frequency:
notes.append(librosa.hz_to_note(float(freq)))
return notes
def most_frequent(array):
"""Docstring"""
return Counter(array).most_common(1)
def find_nearest_index(array, value):
"""Docstring"""
idx = np.searchsorted(array, value, side="left")
if idx > 0 and (
idx == len(array)
or math.fabs(value - array[idx - 1]) < math.fabs(value - array[idx])
):
return idx - 1
return idx
def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Creating midi notes from pitched data")
midi_notes = []
for i in enumerate(start_times):
pos = i[0]
start_time = start_times[pos]
end_time = end_times[pos]
note = create_midi_note_from_pitched_data(
start_time, end_time, pitched_data
)
midi_notes.append(note)
# todo: Progress?
# print(filename + " f: " + str(mean))
return midi_notes
def create_midi_note_from_pitched_data(start_time, end_time, pitched_data):
"""Docstring"""
start = find_nearest_index(pitched_data.times, start_time)
end = find_nearest_index(pitched_data.times, end_time)
# fixme: why is end smaller as start?
if end < start:
print(f"{ULTRASINGER_HEAD} {red_highlighted('Error: end time is smaller as start time. The note is C0 now ...')}")
return "C0"
if start == end:
freqs = [pitched_data.frequencies[start]]
confs = [pitched_data.confidence[start]]
else:
freqs = pitched_data.frequencies[start:end]
confs = pitched_data.confidence[start:end]
conf_f = get_frequency_with_high_confidence(freqs, confs)
notes = convert_frequencies_to_notes(conf_f)
note = most_frequent(notes)[0][0]
return note
@@ -1,4 +1,9 @@
"""Docstring"""
class PitchedData:
"""Docstring"""
times = []
frequencies = []
confidence = []
+84
View File
@@ -0,0 +1,84 @@
"""Docstring"""
import csv
import crepe
from scipy.io import wavfile
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.Pitcher.pitched_data import PitchedData
def get_pitch_with_crepe_file(filename, step_size, model_capacity):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
)
# Todo: add GPU support by using torchcrepe
sample_rate, audio = wavfile.read(filename)
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
def get_pitch_with_crepe(audio, sample_rate, step_size, model_capacity):
"""Docstring"""
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
def write_lists_to_csv(times, frequencies, confidences, filename):
"""Docstring"""
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile)
header = ["time", "frequency", "confidence"]
writer.writerow(header)
for i in enumerate(times):
pos = i[0]
writer.writerow([times[pos], frequencies[pos], confidences[pos]])
def read_data_from_csv(filename):
"""Docstring"""
csv_data = []
with open(filename, "r", encoding="utf-8") as csv_file:
csv_reader = csv.reader(csv_file)
for line in csv_reader:
csv_data.append(line)
headless_data = csv_data[1:]
return headless_data
def get_frequency_with_high_confidence(freqs, confs, threshold=0.4):
"""Docstring"""
conf_f = []
for i, conf in enumerate(confs):
if conf > threshold:
conf_f.append(freqs[i])
if not conf_f:
conf_f = freqs
return conf_f
class Pitcher:
"""Docstring"""
@@ -0,0 +1,18 @@
"""Docstring"""
class TranscribedData:
"""Transcribed data from json file"""
def __init__(self, transcribed_json):
# Vosk = conf, Whisper = confidence
self.conf = transcribed_json.get(
"conf", transcribed_json.get("confidence", None)
)
# Vosk = word, Whisper = text
self.word = transcribed_json.get(
"word", transcribed_json.get("text", None)
)
self.end = transcribed_json["end"]
self.start = transcribed_json["start"]
self.is_hyphen = None
@@ -1,21 +1,26 @@
import wave
"""Docstring"""
import json
import wave
from vosk import Model, KaldiRecognizer
from moduls.Speech_Recognition.TranscribedData import TranscribedData
from moduls.Log import PRINT_ULTRASTAR
from vosk import KaldiRecognizer, Model
from modules.console_colors import ULTRASINGER_HEAD
from modules.Speech_Recognition.TranscribedData import TranscribedData
# todo: Rename to Transcoder?
def transcribe_with_vosk(audio_filename, model_path):
"""Docstring"""
# Code from here: https://towardsdatascience.com/speech-recognition-with-timestamps-934ede4234b2
print(f"{PRINT_ULTRASTAR} Transcribing {audio_filename} with vosk and model {model_path}")
print(
f"{ULTRASINGER_HEAD} Transcribing {audio_filename} with vosk and model {model_path}"
)
model = Model(model_path)
wf = wave.open(audio_filename, "rb")
recognizer = KaldiRecognizer(model, wf.getframerate())
wave_file = wave.open(audio_filename, "rb")
recognizer = KaldiRecognizer(model, wave_file.getframerate())
recognizer.SetWords(True)
@@ -23,13 +28,13 @@ def transcribe_with_vosk(audio_filename, model_path):
results = []
# recognize speech using vosk model
while True:
data = wf.readframes(4000)
data = wave_file.readframes(4000)
if len(data) == 0:
break
if recognizer.AcceptWaveform(data):
part_result = json.loads(recognizer.Result())
results.append(part_result)
wf.close() # close audiofile
wave_file.close() # close audiofile
part_result = json.loads(recognizer.FinalResult())
results.append(part_result)
@@ -41,9 +46,9 @@ def transcribe_with_vosk(audio_filename, model_path):
# and it returns an empty dictionary
# {'text': ''}
continue
for obj in sentence['result']:
for obj in sentence["result"]:
vtd = TranscribedData(obj) # create custom Word object
vtd.word = vtd.word + ' '
vtd.word = vtd.word + " "
transcribed_data.append(vtd) # and add it to list
# Todo: remove silent part from each word
@@ -57,4 +62,4 @@ def transcribe_with_vosk(audio_filename, model_path):
class SpeechToText:
pass
"""Docstring"""
+64
View File
@@ -0,0 +1,64 @@
"""Docstring"""
import whisperx
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.Speech_Recognition.TranscribedData import TranscribedData
def transcribe_with_whisper(audio_path, model, device="cpu"):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
)
batch_size = 16 # reduce if low on GPU mem
compute_type = (
"float16" if device == "cuda" else "int8"
) # change to "int8" if low on GPU mem (may reduce accuracy)
# transcribe with original whisper
loaded_whisper_model = whisperx.load_model(
model, device=device, compute_type=compute_type
)
audio = whisperx.load_audio(audio_path)
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
language = result["language"]
# load alignment model and metadata
model_a, metadata = whisperx.load_align_model(
language_code=language, device=device
)
# align whisper output
result_aligned = whisperx.align(
result["segments"],
model_a,
metadata,
audio,
device,
return_char_alignments=False,
)
transcribed_data = []
for segment in result_aligned["segments"]:
for obj in segment["words"]:
if len(obj) < 4:
print(
f"{red_highlighted('Error: Skipping Word {}, because of missing timings'.format(obj['word']))}"
)
continue
vtd = TranscribedData(obj) # create custom Word object
vtd.word = vtd.word + " "
transcribed_data.append(vtd) # and add it to list
return transcribed_data, language
@@ -1,11 +1,18 @@
from hyphen import Hyphenator
from hyphen import dictools
from moduls.Log import PRINT_ULTRASTAR
from moduls.Log import print_blue_highlighted_text, print_gold_highlighted_text
"""Docstring"""
import string
from hyphen import Hyphenator, dictools
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
)
def language_check(language="en"):
"""Docstring"""
def language_check(language='en'):
dict_langs = dictools.LANGUAGES
lang_region = None
@@ -17,20 +24,26 @@ def language_check(language='en'):
if lang_region is None:
return None
print(f"{PRINT_ULTRASTAR} Hyphenate using language code: {print_blue_highlighted_text(lang_region)}")
print(
f"{ULTRASINGER_HEAD} Hyphenate using language code: {blue_highlighted(lang_region)}"
)
return lang_region
def contains_punctuation(word):
"""Docstring"""
return any(elem in word for elem in string.punctuation)
def hyphenation(word, lang_region):
"""Docstring"""
if contains_punctuation(word):
return None
hy = Hyphenator(lang_region)
syllabus = hy.syllables(word)
hyphenator = Hyphenator(lang_region)
syllabus = hyphenator.syllables(word)
length = len(syllabus)
if length > 1:
@@ -1,22 +1,28 @@
"""Docstring"""
import os
import speech_recognition as sr
from pydub import AudioSegment
from pydub.silence import split_on_silence
from moduls.Log import PRINT_ULTRASTAR
import os
from modules.console_colors import ULTRASINGER_HEAD
# todo: Code from here: https://www.thepythoncode.com/article/using-speech-recognition-to-convert-speech-to-text-python
def PrintText(wav_file):
# English speach!
r = sr.Recognizer()
def print_text(wav_file):
"""Docstring"""
# English speech!
recognizer = sr.Recognizer()
# open the file
with sr.AudioFile(wav_file) as source:
# listen for the data (load audio to memory)
audio_data = r.record(source)
audio_data = recognizer.record(source)
# recognize (convert from speech to text)
text = r.recognize_google(audio_data)
text = recognizer.recognize_google(audio_data)
print(text)
@@ -30,14 +36,15 @@ def get_large_audio_transcription(wav_file):
sound = AudioSegment.from_wav(wav_file)
# split audio sound where silence is 700 miliseconds or more and get chunks
chunks = split_on_silence(sound,
# experiment with this value for your target audio file
min_silence_len=500,
# adjust this per requirement
silence_thresh=sound.dBFS - 14,
# keep the silence for 1 second, adjustable as well
keep_silence=500,
)
chunks = split_on_silence(
sound,
# experiment with this value for your target audio file
min_silence_len=500,
# adjust this per requirement
silence_thresh=sound.dBFS - 14,
# keep the silence for 1 second, adjustable as well
keep_silence=500,
)
folder_name = "audio-chunks"
# create a directory to store the audio chunks
@@ -45,7 +52,7 @@ def get_large_audio_transcription(wav_file):
os.mkdir(folder_name)
whole_text = ""
r = sr.Recognizer()
recognizer = sr.Recognizer()
# process each chunk
for i, audio_chunk in enumerate(chunks, start=1):
@@ -55,12 +62,12 @@ def get_large_audio_transcription(wav_file):
audio_chunk.export(chunk_filename, format="wav")
# recognize the chunk
with sr.AudioFile(chunk_filename) as source:
audio_listened = r.record(source)
audio_listened = recognizer.record(source)
# try converting it to text
try:
text = r.recognize_google(audio_listened)
except sr.UnknownValueError as e:
print("Error:", str(e))
text = recognizer.recognize_google(audio_listened)
except sr.UnknownValueError as error:
print("Error:", str(error))
else:
text = f"{text.capitalize()}. "
print(chunk_filename, ":", text)
@@ -70,19 +77,29 @@ def get_large_audio_transcription(wav_file):
def transcribe_audio(audio_file):
"""Docstring"""
recognizer = sr.Recognizer()
with sr.AudioFile(audio_file) as source:
audio = recognizer.record(source)
try:
transcript = recognizer.recognize_google(audio, show_all=True)
start_time = transcript['result'][0]['alternative'][0]['words'][0]['startTime']
end_time = transcript['result'][0]['alternative'][0]['words'][-1]['endTime']
return transcript['result'][0]['alternative'][0]['transcript'], start_time, end_time
start_time = transcript["result"][0]["alternative"][0]["words"][0][
"startTime"
]
end_time = transcript["result"][0]["alternative"][0]["words"][-1][
"endTime"
]
return (
transcript["result"][0]["alternative"][0]["transcript"],
start_time,
end_time,
)
except sr.UnknownValueError:
print(f"{PRINT_ULTRASTAR} Could not understand audio")
except sr.RequestError as e:
print(f"{PRINT_ULTRASTAR} Error with recognizing service; {e}")
print(f"{ULTRASINGER_HEAD} Could not understand audio")
except sr.RequestError as error:
print(f"{ULTRASINGER_HEAD} Error with recognizing service; {error}")
class SpeechToText:
pass
"""Docstring"""
@@ -1,16 +1,22 @@
"""Docstring"""
def real_bpm_to_ultrastar_bpm(real_bpm):
"""Docstring"""
# The UltraStar BPM info is a fifth beat of the real BPM
ultrastar_bpm = real_bpm / 4
return ultrastar_bpm
def ultrastar_bpm_to_real_bpm(ultrastar_bpm):
"""Docstring"""
# The UltraStar BPM info is a fifth beat of the real BPM
bpm = ultrastar_bpm * 4
return bpm
def second_to_beat(seconds, real_bpm):
"""Docstring"""
# BPM = 60 * beat / T
# T * BPM = 60 * beat
# beat = T * BPM / 60
@@ -19,32 +25,54 @@ def second_to_beat(seconds, real_bpm):
def beat_to_second(beat, real_bpm):
"""Docstring"""
seconds = beat * 60 / real_bpm
return seconds
def midi_note_to_ultrastar_note(midi_note):
"""Docstring"""
# C4 == 48 in Midi
ultrastar_note = midi_note - 48
return ultrastar_note
def ultrastar_note_to_midi_note(ultrastar_note):
"""Docstring"""
# C4 == 48 in Midi
midi_note = ultrastar_note + 48
return midi_note
def get_start_time_from_ultrastar(ultrastar_class, pos):
"""Docstring"""
gap = int(ultrastar_class.gap) / 1000
real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(',', '.')))
start_time = beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap
real_bpm = ultrastar_bpm_to_real_bpm(
float(ultrastar_class.bpm.replace(",", "."))
)
start_time = (
beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap
)
return start_time
def get_end_time_from_ultrastar(ultrastar_class, pos):
"""Docstring"""
gap = int(ultrastar_class.gap) / 1000
real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(',', '.')))
end_time = beat_to_second(int(ultrastar_class.startBeat[pos]) + int(ultrastar_class.durations[pos]),
real_bpm) + gap
real_bpm = ultrastar_bpm_to_real_bpm(
float(ultrastar_class.bpm.replace(",", "."))
)
end_time = (
beat_to_second(
int(ultrastar_class.startBeat[pos])
+ int(ultrastar_class.durations[pos]),
real_bpm,
)
+ gap
)
return end_time
+59
View File
@@ -0,0 +1,59 @@
"""Docstring"""
from modules.console_colors import ULTRASINGER_HEAD
from modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
)
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
def parse_ultrastar_txt(input_file):
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Parse ultrastar txt -> {input_file}")
with open(input_file, "r", encoding="utf-8") as file:
txt = file.readlines()
ultrastar_class = UltrastarTxt()
count = 0
# Strips the newline character
for line in txt:
count += 1
if line.startswith("#"):
if line.startswith("#ARTIST"):
ultrastar_class.artist = line.split(":")[1].replace("\n", "")
elif line.startswith("#TITLE"):
ultrastar_class.title = line.split(":")[1].replace("\n", "")
elif line.startswith("#MP3"):
ultrastar_class.mp3 = line.split(":")[1].replace("\n", "")
elif line.startswith("#GAP"):
ultrastar_class.gap = line.split(":")[1].replace("\n", "")
elif line.startswith("#BPM"):
ultrastar_class.bpm = line.split(":")[1].replace("\n", "")
elif line.startswith(("F ", ": ", "* ", "R ", "G ")):
parts = line.split()
# [0] F : * R G
# [1] start beat
# [2] duration
# [3] pitch
# [4] word
ultrastar_class.noteType.append(parts[0])
ultrastar_class.startBeat.append(parts[1])
ultrastar_class.durations.append(parts[2])
ultrastar_class.pitches.append(parts[3])
ultrastar_class.words.append(parts[4] if len(parts) > 4 else "")
# do always as last
pos = len(ultrastar_class.startBeat) - 1
ultrastar_class.startTimes.append(
get_start_time_from_ultrastar(ultrastar_class, pos)
)
ultrastar_class.endTimes.append(
get_end_time_from_ultrastar(ultrastar_class, pos)
)
# todo: Progress?
return ultrastar_class
@@ -0,0 +1,164 @@
"""Docstring"""
import librosa
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
cyan_highlighted,
gold_highlighted,
light_blue_highlighted,
underlined,
)
from modules.Midi.midi_creator import create_midi_note_from_pitched_data
from modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
ultrastar_note_to_midi_note,
)
MAX_SONG_SCORE = 10000
MAX_SONG_LINE_BONUS = 1000
class Points:
"""Docstring"""
notes = 0
golden_notes = 0
rap = 0
golden_rap = 0
line_bonus = 0
parts = 0
def add_point(note_type, points):
"""Docstring"""
if note_type == ":":
points.notes += 1
elif note_type == "*":
points.golden_notes += 2
elif note_type == "R":
points.rap += 1
elif note_type == "G":
points.golden_rap += 2
return points
class Score:
"""Docstring"""
max_score = 0
notes = 0
golden = 0
line_bonus = 0
score = 0
def get_score(points):
"""Docstring"""
score = Score()
score.max_score = (
MAX_SONG_SCORE
if points.line_bonus == 0
else MAX_SONG_SCORE - MAX_SONG_LINE_BONUS
)
score.notes = round(
score.max_score * (points.notes + points.rap) / points.parts
)
score.golden = round(points.golden_notes + points.golden_rap)
score.score = round(score.notes + points.line_bonus + score.golden)
score.line_bonus = round(points.line_bonus)
return score
def print_score(score):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Total: {cyan_highlighted(str(score.score))}, notes: {blue_highlighted(str(score.notes))}, line bonus: {light_blue_highlighted(str(score.line_bonus))}, golden notes: {gold_highlighted(str(score.golden))}"
)
def calculate_score(pitched_data, ultrastar_class):
"""Docstring"""
print(ULTRASINGER_HEAD + " Calculating Ultrastar Points")
simple_points = Points()
accurate_points = Points()
reachable_line_bonus_per_word = MAX_SONG_LINE_BONUS / len(
ultrastar_class.words
)
for i in enumerate(ultrastar_class.words):
pos = i[0]
if ultrastar_class.words == "":
continue
if ultrastar_class.noteType[pos] == "F":
continue
start_time = get_start_time_from_ultrastar(ultrastar_class, pos)
end_time = get_end_time_from_ultrastar(ultrastar_class, pos)
duration = end_time - start_time
step_size = 0.01 # todo: should be beat length ?
parts = int(duration / step_size)
parts = 1 if parts == 0 else parts
accurate_part_line_bonus_points = 0
simple_part_line_bonus_points = 0
ultrastar_midi_note = ultrastar_note_to_midi_note(
int(ultrastar_class.pitches[pos])
)
ultrastar_note = librosa.midi_to_note(ultrastar_midi_note)
for part in range(parts):
start = start_time + step_size * part
end = start + step_size
pitch_note = create_midi_note_from_pitched_data(
start, end, pitched_data
)
if pitch_note[:-1] == ultrastar_note[:-1]:
# Ignore octave high
simple_points = add_point(
ultrastar_class.noteType[pos], simple_points
)
simple_part_line_bonus_points += 1
if pitch_note == ultrastar_note:
# Octave high must be the same
accurate_points = add_point(
ultrastar_class.noteType[pos], accurate_points
)
accurate_part_line_bonus_points += 1
accurate_points.parts += 1
simple_points.parts += 1
if accurate_part_line_bonus_points >= parts:
accurate_points.line_bonus += reachable_line_bonus_per_word
if simple_part_line_bonus_points >= parts:
simple_points.line_bonus += reachable_line_bonus_per_word
return get_score(simple_points), get_score(accurate_points)
def print_score_calculation(simple_points, accurate_points):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} {underlined('Simple (octave high ignored)')} points"
)
print_score(simple_points)
print(
f"{ULTRASINGER_HEAD} {underlined('Accurate (octave high matches)')} points:"
)
print_score(accurate_points)
+22
View File
@@ -0,0 +1,22 @@
"""Docstring"""
class UltrastarTxt:
"""Docstring"""
artist = ""
title = ""
mp3 = ""
gap = ""
bpm = ""
cover = None
creator = "UltraSinger [GitHub]"
fixer = "YOUR NAME" # Only for UltraStar World Party
comment = "UltraSinger [GitHub]"
startBeat = []
startTimes = []
endTimes = []
durations = []
pitches = []
words = []
noteType = []
+179
View File
@@ -0,0 +1,179 @@
"""Docstring"""
import re
import langcodes
from modules.console_colors import ULTRASINGER_HEAD
from modules.Ultrastar.ultrastar_converter import (
real_bpm_to_ultrastar_bpm,
second_to_beat,
)
def get_multiplier(real_bpm):
"""Docstring"""
if real_bpm == 0:
raise Exception("BPM is 0")
multiplier = 1
result = 0
while result < 400:
result = real_bpm * multiplier
multiplier += 1
return multiplier - 2
def get_language_name(language):
"""Docstring"""
return langcodes.Language.make(language=language).display_name()
def create_ultrastar_txt_from_automation(
transcribed_data,
note_numbers,
ultrastar_file_output,
ultrastar_class,
bpm=120,
):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Creating {ultrastar_file_output} from transcription."
)
real_bpm = real_bpm_to_ultrastar_bpm(bpm)
multiplication = get_multiplier(real_bpm)
ultrastar_bpm = real_bpm * get_multiplier(real_bpm)
with open(ultrastar_file_output, "w", encoding="utf-8") as file:
gap = transcribed_data[0].start
file.write(f"#ARTIST:{ultrastar_class.artist}\n")
file.write(f"#TITLE:{ultrastar_class.title}\n")
file.write(f"#CREATOR:{ultrastar_class.creator}\n")
file.write(f"#FIXER:{ultrastar_class.fixer}\n")
if ultrastar_class.language is not None:
file.write(
f"#LANGUAGE:{get_language_name(ultrastar_class.language)}\n"
)
if ultrastar_class.cover is not None:
file.write(f"#COVER:{ultrastar_class.cover}\n")
file.write(f"#MP3:{ultrastar_class.mp3}\n")
file.write(f"#VIDEO:{ultrastar_class.video}\n")
file.write(f"#BPM:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
file.write(f"#GAP:{int(gap * 1000)}\n")
file.write(f"#COMMENT:{ultrastar_class.comment}\n")
# Write the singing part
previous_end_beat = 0
for i, data in enumerate(transcribed_data):
start_time = (data.start - gap) * multiplication
end_time = (
data.end - data.start
) * multiplication
start_beat = round(second_to_beat(start_time, bpm))
duration = round(second_to_beat(end_time, bpm))
# Fix the round issue, so the beats don’t overlap
start_beat = max(start_beat, previous_end_beat)
previous_end_beat = start_beat + duration
# : 10 10 10 w
# ':' start midi part
# 'n1' start at real beat
# 'n2' duration at real beat
# 'n3' pitch where 0 == C4
# 'w' lyric
file.write(": ")
file.write(str(start_beat) + " ")
file.write(str(duration) + " ")
file.write(str(note_numbers[i]) + " ")
file.write(data.word)
file.write("\n")
# detect silence between words
if i < len(transcribed_data) - 1:
silence = (
transcribed_data[i + 1].start - data.end
)
else:
silence = 0
if silence > 0.3 and i != len(transcribed_data) - 1:
# - 10
# '-' end of current sing part
# 'n1' show next at time in real beat
file.write("- ")
show_next = (
second_to_beat(data.end - gap, bpm)
* multiplication
)
file.write(str(round(show_next)))
file.write("\n")
file.write("E")
def create_repitched_txt_from_ultrastar_data(
input_file, note_numbers, output_repitched_ultrastar
):
"""Docstring"""
# todo: just add '_repitched' to input_file
print(
"{PRINT_ULTRASTAR} Creating repitched ultrastar txt -> {input_file}_repitch.txt"
)
# todo: to reader
with open(input_file, "r", encoding="utf-8") as file:
txt = file.readlines()
i = 0
# todo: just add '_repitched' to input_file
with open(output_repitched_ultrastar, "w", encoding="utf-8") as file:
for line in txt:
if line.startswith(":"):
parts = re.findall(r"\S+|\s+", line)
# between are whitespaces
# [0] :
# [2] start beat
# [4] duration
# [6] pitch
# [8] word
parts[6] = str(note_numbers[i])
delimiter = ""
file.write(delimiter.join(parts))
i += 1
else:
file.write(line)
def add_score_to_ultrastar_txt(ultrastar_file_output, score):
"""Docstring"""
with open(ultrastar_file_output, "r", encoding="utf-8") as file:
text = file.read()
text = text.split("\n")
for i, line in enumerate(text):
if line.startswith("#COMMENT:"):
text[
i
] = f"{line} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
break
if line.startswith(("F ", ": ", "* ", "R ", "G ")):
text.insert(
i,
f"#COMMENT: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
)
break
text = "\n".join(text)
with open(ultrastar_file_output, "w", encoding="utf-8") as file:
file.write(text)
class UltraStarWriter:
"""Docstring"""
+45
View File
@@ -0,0 +1,45 @@
"""Colors for the console"""
ULTRASINGER_HEAD = "\033[92m[UltraSinger]\033[0m"
def blue_highlighted(text: str) -> str:
"""Returns a blue highlighted text"""
return f"{Bcolors.blue}{text}{Bcolors.endc}"
def gold_highlighted(text: str) -> str:
"""Returns a gold highlighted text"""
return f"{Bcolors.gold}{text}{Bcolors.endc}"
def light_blue_highlighted(text: str) -> str:
"""Returns a light blue highlighted text"""
return f"{Bcolors.light_blue}{text}{Bcolors.endc}"
def underlined(text: str) -> str:
"""Returns an underlined text"""
return f"{Bcolors.underline}{text}{Bcolors.endc}"
def red_highlighted(text: str) -> str:
"""Returns a red highlighted text"""
return f"{Bcolors.red}{text}{Bcolors.endc}"
def cyan_highlighted(text: str) -> str:
"""Returns a cyan highlighted text"""
return f"{Bcolors.cyan}{text}{Bcolors.endc}"
class Bcolors:
"""Colors for the console"""
blue = "\033[94m"
red = "\033[91m"
light_blue = "\033[96m"
cyan = "\033[36m"
gold = "\033[93m"
underline = "\033[4m"
endc = "\033[0m"
+24
View File
@@ -0,0 +1,24 @@
"""CSV export module"""
import csv
from modules.console_colors import ULTRASINGER_HEAD
def export_transcribed_data_to_csv(transcribed_data: [], filename: str) -> None:
"""Export transcribed data to csv"""
print(f"{ULTRASINGER_HEAD} Exporting transcribed data to CSV")
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile)
header = ["word", "start", "end", "confidence"]
writer.writerow(header)
for i, data in enumerate(transcribed_data):
writer.writerow(
[
data.word,
data.start,
data.end,
data.conf,
]
)
+13 -3
View File
@@ -1,34 +1,44 @@
"""Docstring"""
import os
import shutil
from moduls.Log import PRINT_ULTRASTAR
from modules.console_colors import ULTRASINGER_HEAD
def create_folder(folder_name):
print(f"{PRINT_ULTRASTAR} Creating output folder. -> {folder_name}")
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Creating output folder. -> {folder_name}")
is_exist = os.path.exists(folder_name)
if not is_exist:
os.makedirs(folder_name)
def move(original, target):
"""Docstring"""
shutil.move(original, target)
def copy(original, target):
"""Docstring"""
shutil.copy(original, target)
def current_executor_path():
"""Docstring"""
return os.getcwd()
def path_join(path1, path2):
"""Docstring"""
return os.path.join(path1, path2)
def check_file_exists(file_path):
"""Docstring"""
return os.path.isfile(file_path)
def check_if_folder_exists(song_output):
return os.path.isdir(song_output)
"""Docstring"""
return os.path.isdir(song_output)
-23
View File
@@ -1,23 +0,0 @@
import ffmpeg
from moduls.Log import PRINT_ULTRASTAR, print_blue_highlighted_text
def ffmpeg_reduce_noise(input_file_path, output_file):
# Denoise audio samples with FFT.
# A description of the accepted parameters follows.
# noise_reduction, nr
# Set the noise reduction in dB, allowed range is 0.01 to 97. Default value is 12 dB.
# noise_floor, nf
# Set the noise floor in dB, allowed range is -80 to -20. Default value is -50 dB.
# track_noise, tn
# Enable noise floor tracking. By default is disabled. With this enabled, noise floor is automatically adjusted.
print(f"{PRINT_ULTRASTAR} Reduce noise from vocal audio with {print_blue_highlighted_text('ffmpeg')}.")
(
ffmpeg
.input(input_file_path)
.output(output_file, af='afftdn=nr=70:nf=-50:tn=1')
.overwrite_output()
.run()
)
-15
View File
@@ -1,15 +0,0 @@
import subprocess
from moduls.os_helper import move, current_executor_path, path_join
from moduls.Log import PRINT_ULTRASTAR, print_blue_highlighted_text, print_red_highlighted_text
def separate_audio(input_file_path, output_file, device="cpu"):
print(f"{PRINT_ULTRASTAR} Separating vocals from audio with {print_blue_highlighted_text('demucs')} and {print_red_highlighted_text(device)} as worker.")
# Model selection?
# -n mdx_q
# -n htdemucs_ft
device = "cpu" if device != "cuda" else "cuda"
subprocess.run(["demucs", "-d", device, "--two-stems=vocals", input_file_path])
separated_folder = path_join(current_executor_path(), "separated")
move(separated_folder, output_file)
-144
View File
@@ -1,144 +0,0 @@
import csv
import os
import wave
import re
import librosa
from moduls.os_helper import create_folder
from moduls.Ultrastar.ultrastar_converter import get_start_time_from_ultrastar, get_end_time_from_ultrastar
from pydub import AudioSegment
from moduls.Log import PRINT_ULTRASTAR
def convert_audio_to_mono_wav(input_file, output_file):
"""Convert audio to mono wav"""
print(f"{PRINT_ULTRASTAR} Converting audio for AI")
if '.mp3' in input_file:
sound = AudioSegment.from_mp3(input_file)
elif '.wav' in input_file:
sound = AudioSegment.from_wav(input_file)
else:
raise ValueError('data format not supported')
sound = sound.set_channels(1)
sound.export(output_file, format="wav")
def convert_wav_to_mp3(input_file, output_file):
sound = AudioSegment.from_wav(input_file)
sound.export(output_file, format="mp3")
class AudioManipulation:
pass
def export_chunks_from_transcribed_data(audio_filename, transcribed_data, output_folder_name):
"""Export transcribed_data as vocal chunks wav files"""
print(f"{PRINT_ULTRASTAR} Export transcribed data as vocal chunks wav files")
wf = wave.open(audio_filename, "rb")
sr, n_channels = wf.getparams()[2], wf.getparams()[0]
for i in range(len(transcribed_data)):
start_byte = int(transcribed_data[i].start * sr * n_channels)
end_byte = int(transcribed_data[i].end * sr * n_channels)
chunk = get_chunk(end_byte, start_byte, wf)
export_chunk_to_wav_file(chunk, output_folder_name, i, transcribed_data[i].word, wf)
wf.close()
def remove_silence_from_transcribtion_data(audio_path, transcribed_data):
print(f"{PRINT_ULTRASTAR} Removing silent start and ending, from transcription data")
y, sr = librosa.load(audio_path, sr=None)
for i in range(len(transcribed_data)):
start_time = transcribed_data[i].start
end_time = transcribed_data[i].end
start_sample = int(start_time * sr)
end_sample = int(end_time * sr)
chunk = y[start_sample:end_sample]
# todo: why 5 works good? It should be 40db ?!?
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
silence_threshold = 5
onsets = librosa.effects.split(chunk, top_db=silence_threshold, frame_length=2048, hop_length=100)
# Get the duration of the first and last silent intervals
if len(onsets) > 0:
first_silence = onsets[0][0]
last_silence = len(chunk) - onsets[-1][1]
first_silence_duration = librosa.samples_to_time(first_silence, sr=sr)
last_silence_duration = librosa.samples_to_time(last_silence, sr=sr)
else:
first_silence_duration = 0
last_silence_duration = 0
transcribed_data[i].start = transcribed_data[i].start + first_silence_duration
transcribed_data[i].end = transcribed_data[i].end - last_silence_duration
return transcribed_data
def export_chunks_from_ultrastar_data(audio_filename, ultrastar_data, folder_name):
"""Export ultrastar data as vocal chunks wav files"""
print(f"{PRINT_ULTRASTAR} Export Ultrastar data as vocal chunks wav files")
create_folder(folder_name)
wf = wave.open(audio_filename, "rb")
sr, n_channels = wf.getparams()[2], wf.getparams()[0]
for i in range(len(ultrastar_data.words)):
start_time = get_start_time_from_ultrastar(ultrastar_data, i)
end_time = get_end_time_from_ultrastar(ultrastar_data, i)
start_byte = int(start_time * sr * n_channels)
end_byte = int(end_time * sr * n_channels)
chunk = get_chunk(end_byte, start_byte, wf)
export_chunk_to_wav_file(chunk, folder_name, i, ultrastar_data.words[i], wf)
def export_chunk_to_wav_file(chunk, folder_name, i, word, wf):
"""Export vocal chunks to wav file"""
clean_word = re.sub('[^A-Za-z0-9]+', '', word)
# todo: Progress?
# print(str(i) + ' ' + clean_word)
with wave.open(os.path.join(folder_name, f"chunk_{i}_{clean_word}.wav"),
"wb") as chunk_file:
chunk_file.setparams(wf.getparams())
chunk_file.writeframes(chunk)
def get_chunk(end_byte, start_byte, wf):
"""
Gets the chunk from wave file.
Returns chunk as n frames of audio, as a bytes object.
"""
# todo: get out of position error message
wf.setpos(start_byte) # ({:.2f})
chunk = wf.readframes(end_byte - start_byte)
return chunk
def export_transcribed_data_to_csv(transcribed_data, filename):
"""Export transcribed data to csv"""
print(f"{PRINT_ULTRASTAR} Exporting transcribed data to CSV")
with open(filename, 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
header = ["word", "start", "end", "confidence"]
writer.writerow(header)
for i in range(len(transcribed_data)):
writer.writerow(
[transcribed_data[i].word, transcribed_data[i].start, transcribed_data[i].end,
transcribed_data[i].conf])
-67
View File
@@ -1,67 +0,0 @@
import yt_dlp
from moduls.Log import PRINT_ULTRASTAR
from PIL import Image
import io
import os
def get_youtube_title(url):
ydl_opts = {}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
result = ydl.extract_info(
url,
download=False # We just want to extract the info
)
return result['title']
def download_youtube_audio(url, clear_filename, output_path):
print(f"{PRINT_ULTRASTAR} Downloading Audio")
ydl_opts = {
'format': 'bestaudio/best',
'outtmpl': output_path + '/' + clear_filename,
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3'
}]
}
start_download(ydl_opts, url)
def download_youtube_thumbnail(url, clear_filename, output_path):
print(f"{PRINT_ULTRASTAR} Downloading thumbnail")
ydl_opts = {
'skip_download': True,
'writethumbnail': True,
}
download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path)
def download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path):
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info_dict = ydl.extract_info(url, download=False)
thumbnail_url = info_dict.get('thumbnail')
if thumbnail_url:
response = ydl.urlopen(thumbnail_url)
image_data = response.read()
image = Image.open(io.BytesIO(image_data))
image.save(os.path.join(output_path, clear_filename + " [CO].jpg"), "JPEG")
def download_youtube_video(url, clear_filename, output_path):
print(f"{PRINT_ULTRASTAR} Downloading Video")
ydl_opts = {
'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4',
'outtmpl': output_path + '/' + clear_filename + '.mp4'
}
start_download(ydl_opts, url)
def start_download(ydl_opts, url):
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
errors = ydl.download(url)
if errors:
raise Exception("Download failed with error: " + str(errors))
@@ -1,13 +0,0 @@
from moduls.Log import PRINT_ULTRASTAR, print_red_highlighted_text
import torch
def get_available_device():
isCuda = torch.cuda.is_available()
if not isCuda:
print(f"{PRINT_ULTRASTAR} There are no {print_red_highlighted_text('cuda')} devices available. -> Using {print_red_highlighted_text('cpu')}.")
return "cpu"
else:
print(f"{PRINT_ULTRASTAR} Using {print_red_highlighted_text('cuda')} GPU.")
return "cuda"
-35
View File
@@ -1,35 +0,0 @@
PRINT_ULTRASTAR = "\033[92m[UltraSinger]\033[0m"
def print_blue_highlighted_text(text):
return Bcolors.BLUE + text + Bcolors.ENDC
def print_gold_highlighted_text(text):
return Bcolors.Gold + text + Bcolors.ENDC
def print_light_blue_highlighted_text(text):
return Bcolors.LightBlue + text + Bcolors.ENDC
def print_underlined_text(text):
return Bcolors.Underline + text + Bcolors.ENDC
def print_red_highlighted_text(text):
return Bcolors.RED + text + Bcolors.ENDC
def print_cyan_highlighted_text(text):
return Bcolors.Cyan + text + Bcolors.ENDC
class Bcolors:
BLUE = '\033[94m'
RED = '\033[91m'
LightBlue = '\033[96m'
Cyan = '\033[36m'
Gold = '\033[93m'
Underline = '\033[4m'
ENDC = '\033[0m'
-96
View File
@@ -1,96 +0,0 @@
import math
from collections import Counter
import librosa
import numpy as np
import pretty_midi
from moduls.Pitcher.pitcher import get_frequency_with_high_confidence
from moduls.Ultrastar.ultrastar_converter import ultrastar_note_to_midi_note, get_start_time_from_ultrastar, \
get_end_time_from_ultrastar
from moduls.Log import PRINT_ULTRASTAR
def convert_ultrastar_to_midi_instrument(ultrastar_class):
print(f"{PRINT_ULTRASTAR} Creating midi instrument from Ultrastar txt")
instrument = pretty_midi.Instrument(program=0)
velocity = 100
for i in range(len(ultrastar_class.words)):
start_time = get_start_time_from_ultrastar(ultrastar_class, i)
end_time = get_end_time_from_ultrastar(ultrastar_class, i)
pitch = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[i]))
note = pretty_midi.Note(velocity, pitch, start_time, end_time)
instrument.notes.append(note)
return instrument
def instruments_to_midi(instruments, bpm, midi_output):
print(f"{PRINT_ULTRASTAR} Creating midi file -> {midi_output}")
midi_data = pretty_midi.PrettyMIDI(initial_tempo=bpm)
for instrument in instruments:
midi_data.instruments.append(instrument)
midi_data.write(midi_output)
class MidiCreator:
pass
def convert_frequencies_to_notes(frequency):
notes = []
for f in frequency:
notes.append(librosa.hz_to_note(float(f)))
return notes
def most_frequent(ar):
return Counter(ar).most_common(1)
def find_nearest_index(array, value):
idx = np.searchsorted(array, value, side="left")
if idx > 0 and (idx == len(array) or math.fabs(value - array[idx - 1]) < math.fabs(value - array[idx])):
return idx - 1
else:
return idx
def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
print(f"{PRINT_ULTRASTAR} Creating midi notes from pitched data")
midi_notes = []
for i in range(len(start_times)):
start_time = start_times[i]
end_time = end_times[i]
note = create_midi_note_from_pitched_data(start_time, end_time, pitched_data)
midi_notes.append(note)
# todo: Progress?
# print(filename + " f: " + str(mean))
return midi_notes
def create_midi_note_from_pitched_data(start_time, end_time, pitched_data):
s = find_nearest_index(pitched_data.times, start_time)
e = find_nearest_index(pitched_data.times, end_time)
if s == e:
f = [pitched_data.frequencies[s]]
c = [pitched_data.confidence[s]]
else:
f = pitched_data.frequencies[s:e]
c = pitched_data.confidence[s:e]
conf_f = get_frequency_with_high_confidence(f, c)
notes = convert_frequencies_to_notes(conf_f)
note = most_frequent(notes)[0][0]
return note
-61
View File
@@ -1,61 +0,0 @@
import csv
import crepe
from moduls.Pitcher.pitched_data import PitchedData
from scipy.io import wavfile
from moduls.Log import PRINT_ULTRASTAR
from moduls.Log import print_blue_highlighted_text, print_red_highlighted_text
def get_pitch_with_crepe_file(filename, step_size, model_capacity):
print(f"{PRINT_ULTRASTAR} Pitching with {print_blue_highlighted_text('crepe')} and model {print_blue_highlighted_text(model_capacity)} and {print_red_highlighted_text('cpu')} as worker")
# Todo: add GPU support by using torchcrepe
sr, audio = wavfile.read(filename)
pitched_data = PitchedData()
pitched_data.times, pitched_data.frequencies, pitched_data.confidence, activation = crepe.predict(audio, sr,
model_capacity,
step_size=step_size,
viterbi=True)
return pitched_data
def get_pitch_with_crepe(y, sr, step_size, model_capacity):
pitched_data = PitchedData()
pitched_data.times, pitched_data.frequencies, pitched_data.confidence, activation = crepe.predict(y, sr,
model_capacity,
step_size=step_size,
viterbi=True)
return pitched_data
def write_lists_to_csv(time, frequency, confidence, filename):
with open(filename, 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
header = ["time", "frequency", "confidence"]
writer.writerow(header)
for i in range(len(time)):
writer.writerow([time[i], frequency[i], confidence[i]])
def read_data_from_csv(filename):
csv_data = []
with open(filename, 'r') as csv_file:
csv_reader = csv.reader(csv_file)
for line in csv_reader:
csv_data.append(line)
headless_data = csv_data[1:]
return headless_data
def get_frequency_with_high_confidence(f, c, threshold=0.4):
conf_f = []
for i in range(len(c)):
if c[i] > threshold:
conf_f.append(f[i])
if not conf_f:
conf_f = f
return conf_f
class Pitcher:
pass
@@ -1,11 +0,0 @@
class TranscribedData:
""" Transcribed data from json file """
def __init__(self, transcribed_json):
# Vosk = conf, Whisper = confidence
self.conf = transcribed_json.get("conf", transcribed_json.get("confidence", None))
# Vosk = word, Whisper = text
self.word = transcribed_json.get("word", transcribed_json.get("text", None))
self.end = transcribed_json["end"]
self.start = transcribed_json["start"]
self.is_hyphen = None
-39
View File
@@ -1,39 +0,0 @@
import whisperx
from moduls.Speech_Recognition.TranscribedData import TranscribedData
from moduls.Log import PRINT_ULTRASTAR
from moduls.Log import print_blue_highlighted_text, print_red_highlighted_text
def transcribe_with_whisper(audio_path, model, device="cpu"):
print(f"{PRINT_ULTRASTAR} Loading {print_blue_highlighted_text('whisper')} with model {print_blue_highlighted_text(model)} and {print_red_highlighted_text(device)} as worker")
batch_size = 16 # reduce if low on GPU mem
compute_type = "float16" if device == "cuda" else "int8" # change to "int8" if low on GPU mem (may reduce accuracy)
# transcribe with original whisper
loaded_whisper_model = whisperx.load_model(model, device=device, compute_type=compute_type)
audio = whisperx.load_audio(audio_path)
print(f"{PRINT_ULTRASTAR} Transcribing {audio_path}")
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
language = result["language"]
# load alignment model and metadata
model_a, metadata = whisperx.load_align_model(language_code=language, device=device)
# align whisper output
result_aligned = whisperx.align(result["segments"], model_a, metadata, audio, device, return_char_alignments=False)
transcribed_data = []
for segment in result_aligned["segments"]:
for obj in segment["words"]:
if len(obj) < 4:
print(f"{print_red_highlighted_text('Error: Skipping Word {}, because of missing timings'.format(obj['word']))}")
continue
vtd = TranscribedData(obj) # create custom Word object
vtd.word = vtd.word + ' '
transcribed_data.append(vtd) # and add it to list
return transcribed_data, language
-49
View File
@@ -1,49 +0,0 @@
from moduls.Ultrastar.ultrastar_txt import UltrastarTxt
from moduls.Ultrastar.ultrastar_converter import get_end_time_from_ultrastar, get_start_time_from_ultrastar
from moduls.Log import PRINT_ULTRASTAR
def parse_ultrastar_txt(input_file):
print(f"{PRINT_ULTRASTAR} Parse ultrastar txt -> {input_file}")
file = open(input_file, 'r', encoding='utf8')
txt = file.readlines()
ultrastar_class = UltrastarTxt()
count = 0
# Strips the newline character
for line in txt:
count += 1
if line.startswith('#'):
if line.startswith('#ARTIST'):
ultrastar_class.artist = line.split(':')[1].replace('\n', '')
elif line.startswith('#TITLE'):
ultrastar_class.title = line.split(':')[1].replace('\n', '')
elif line.startswith('#MP3'):
ultrastar_class.mp3 = line.split(':')[1].replace('\n', '')
elif line.startswith('#GAP'):
ultrastar_class.gap = line.split(':')[1].replace('\n', '')
elif line.startswith('#BPM'):
ultrastar_class.bpm = line.split(':')[1].replace('\n', '')
elif line.startswith(('F ', ': ', '* ', 'R ', 'G ')):
parts = line.split()
# [0] F : * R G
# [1] start beat
# [2] duration
# [3] pitch
# [4] word
ultrastar_class.noteType.append(parts[0])
ultrastar_class.startBeat.append(parts[1])
ultrastar_class.durations.append(parts[2])
ultrastar_class.pitches.append(parts[3])
ultrastar_class.words.append(parts[4] if len(parts) > 4 else '')
# do always as last
pos = len(ultrastar_class.startBeat) - 1
ultrastar_class.startTimes.append(get_start_time_from_ultrastar(ultrastar_class, pos))
ultrastar_class.endTimes.append(get_end_time_from_ultrastar(ultrastar_class, pos))
# todo: Progress?
return ultrastar_class
@@ -1,111 +0,0 @@
from moduls.Midi.midi_creator import create_midi_note_from_pitched_data
from moduls.Ultrastar.ultrastar_converter import get_start_time_from_ultrastar, get_end_time_from_ultrastar, \
ultrastar_note_to_midi_note
from moduls.Log import PRINT_ULTRASTAR, print_blue_highlighted_text, print_gold_highlighted_text, \
print_light_blue_highlighted_text, print_underlined_text, print_cyan_highlighted_text
import librosa
MAX_SONG_SCORE = 10000
MAX_SONG_LINE_BONUS = 1000
class Points:
notes = 0
golden_notes = 0
rap = 0
golden_rap = 0
line_bonus = 0
parts = 0
def add_point(noteType, points):
if noteType == ':':
points.notes += 1
elif noteType == '*':
points.golden_notes += 2
elif noteType == 'R':
points.rap += 1
elif noteType == 'G':
points.golden_rap += 2
return points
class Score:
max_score = 0
notes = 0
golden = 0
line_bonus = 0
score = 0
def get_score(points):
score = Score()
score.max_score = MAX_SONG_SCORE if points.line_bonus == 0 else MAX_SONG_SCORE - MAX_SONG_LINE_BONUS
score.notes = round(score.max_score * (points.notes + points.rap) / points.parts)
score.golden = round(points.golden_notes + points.golden_rap)
score.score = round(score.notes + points.line_bonus + score.golden)
score.line_bonus = round(points.line_bonus)
return score
def print_score(score):
print(f"{PRINT_ULTRASTAR} Total: {print_cyan_highlighted_text(str(score.score))}, notes: {print_blue_highlighted_text(str(score.notes))}, line bonus: {print_light_blue_highlighted_text(str(score.line_bonus))}, golden notes: {print_gold_highlighted_text(str(score.golden))}")
def calculate_score(pitched_data, ultrastar_class):
print(PRINT_ULTRASTAR + " Calculating Ultrastar Points")
simple_points = Points()
accurate_points = Points()
reachable_line_bonus_per_word = MAX_SONG_LINE_BONUS / len(ultrastar_class.words)
for i in range(len(ultrastar_class.words)):
if ultrastar_class.words == "":
continue
if ultrastar_class.noteType[i] == 'F':
continue
start_time = get_start_time_from_ultrastar(ultrastar_class, i)
end_time = get_end_time_from_ultrastar(ultrastar_class, i)
duration = end_time - start_time
step_size = 0.01 # todo: should be beat length ?
parts = int(duration / step_size)
parts = 1 if parts == 0 else parts
accurate_part_line_bonus_points = 0
simple_part_line_bonus_points = 0
ultrastar_midi_note = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[i]))
ultrastar_note = librosa.midi_to_note(ultrastar_midi_note)
for p in range(parts):
st = start_time + step_size * p
end = st + step_size
pitch_note = create_midi_note_from_pitched_data(st, end, pitched_data)
if pitch_note[:-1] == ultrastar_note[:-1]:
# Ignore octave high
simple_points = add_point(ultrastar_class.noteType[i], simple_points)
simple_part_line_bonus_points += 1
if pitch_note == ultrastar_note:
# Octave high must be the same
accurate_points = add_point(ultrastar_class.noteType[i], accurate_points)
accurate_part_line_bonus_points += 1
accurate_points.parts += 1
simple_points.parts += 1
if accurate_part_line_bonus_points >= parts:
accurate_points.line_bonus += reachable_line_bonus_per_word
if simple_part_line_bonus_points >= parts:
simple_points.line_bonus += reachable_line_bonus_per_word
return get_score(simple_points), get_score(accurate_points)
def print_score_calculation(simple_points, accurate_points):
print(f"{PRINT_ULTRASTAR} {print_underlined_text('Simple (octave high ignored)')} points")
print_score(simple_points)
print(f"{PRINT_ULTRASTAR} {print_underlined_text('Accurate (octave high matches)')} points:")
print_score(accurate_points)
-17
View File
@@ -1,17 +0,0 @@
class UltrastarTxt:
artist = ''
title = ''
mp3 = ''
gap = ''
bpm = ''
cover = None
creator = 'UltraSinger [GitHub]'
fixer = 'YOUR NAME'
comment = 'UltraSinger [GitHub]'
startBeat = []
startTimes = []
endTimes = []
durations = []
pitches = []
words = []
noteType = []
-136
View File
@@ -1,136 +0,0 @@
from moduls.Ultrastar.ultrastar_converter import real_bpm_to_ultrastar_bpm, second_to_beat
from moduls.Log import PRINT_ULTRASTAR
import langcodes
import re
def get_multiplier(real_bpm):
if real_bpm == 0:
raise Exception("BPM is 0")
multiplier = 1
result = 0
while result < 400:
result = real_bpm * multiplier
multiplier += 1
return multiplier - 2
def get_language_name(language):
return langcodes.Language.make(language=language).display_name()
def create_ultrastar_txt_from_automation(transcribed_data, note_numbers, ultrastar_file_output, ultrastar_class,
bpm=120):
print(f"{PRINT_ULTRASTAR} Creating {ultrastar_file_output} from transcription.")
real_bpm = real_bpm_to_ultrastar_bpm(bpm)
multiplication = get_multiplier(real_bpm)
ultrastar_bpm = real_bpm * get_multiplier(real_bpm)
with open(ultrastar_file_output, 'w', encoding='utf8') as f:
gap = transcribed_data[0].start
f.write(f'#ARTIST:{ultrastar_class.artist}\n')
f.write(f'#TITLE:{ultrastar_class.title}\n')
f.write(f'#CREATOR:{ultrastar_class.creator}\n')
f.write(f'#FIXER:{ultrastar_class.fixer}\n')
if ultrastar_class.language is not None:
f.write(f'#LANGUAGE:{get_language_name(ultrastar_class.language)}\n')
if ultrastar_class.cover is not None:
f.write(f'#COVER:{ultrastar_class.cover}\n')
f.write(f'#MP3:{ultrastar_class.mp3}\n')
f.write(f'#VIDEO:{ultrastar_class.video}\n')
f.write(f'#BPM:' + str(round(ultrastar_bpm, 2)) + '\n') # not the real BPM!
f.write(f'#GAP:' + str(int(gap * 1000)) + '\n')
f.write(f'#COMMENT:{ultrastar_class.comment}\n')
# Write the singing part
previous_end_beat = 0
for i in range(len(transcribed_data)):
start_time = (transcribed_data[i].start - gap) * multiplication
end_time = (transcribed_data[i].end - transcribed_data[i].start) * multiplication
start_beat = round(second_to_beat(start_time, bpm))
duration = round(second_to_beat(end_time, bpm))
# Fix the round issue, so the beats dont overlap
if start_beat < previous_end_beat:
start_beat = previous_end_beat
previous_end_beat = start_beat + duration
# : 10 10 10 w
# ':' start midi part
# 'n1' start at real beat
# 'n2' duration at real beat
# 'n3' pitch where 0 == C4
# 'w' lyric
f.write(': ')
f.write(str(start_beat) + ' ')
f.write(str(duration) + ' ')
f.write(str(note_numbers[i]) + ' ')
f.write(transcribed_data[i].word)
f.write('\n')
# detect silence between words
if i < len(transcribed_data) - 1:
silence = transcribed_data[i + 1].start - transcribed_data[i].end
else:
silence = 0
if silence > 0.3 and i != len(transcribed_data) - 1:
# - 10
# '-' end of current sing part
# 'n1' show next at time in real beat
f.write('- ')
show_next = second_to_beat(transcribed_data[i].end - gap, bpm) * multiplication
f.write(str(round(show_next)))
f.write('\n')
f.write('E')
def create_repitched_txt_from_ultrastar_data(input_file, note_numbers, output_repitched_ultrastar):
# todo: just add '_repitched' to input_file
print("{PRINT_ULTRASTAR} Creating repitched ultrastar txt -> {input_file}_repitch.txt")
# todo: to reader
file = open(input_file, 'r')
txt = file.readlines()
i = 0
# todo: just add '_repitched' to input_file
with open(output_repitched_ultrastar, 'w', encoding='utf8') as f:
for line in txt:
if line.startswith(':'):
parts = re.findall(r'\S+|\s+', line)
# between are whitespaces
# [0] :
# [2] start beat
# [4] duration
# [6] pitch
# [8] word
parts[6] = str(note_numbers[i])
delimiter = ''
f.write(delimiter.join(parts))
i += 1
else:
f.write(line)
def add_score_to_ultrastar_txt(ultrastar_file_output, score):
with open(ultrastar_file_output, 'r', encoding='utf8') as f:
text = f.read()
text = text.split('\n')
for i in range(len(text)):
if text[i].startswith("#COMMENT:"):
text[i] = f"{text[i]} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
break
elif text[i].startswith(('F ', ': ', '* ', 'R ', 'G ')):
text.insert(i, f"#COMMENT: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}")
break
text = '\n'.join(text)
with open(ultrastar_file_output, 'w', encoding='utf8') as f:
f.write(text)
class UltraStarWriter:
pass
+4 -1
View File
@@ -17,4 +17,7 @@ demucs~=4.0.0
tqdm~=4.65.0
langcodes~=3.3.0
language-data~=1.1
Pillow~=9.5.0
Pillow~=9.5.0
isort==5.12
black==23.3
pylint==2.17