mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Cherry refactorings from evaluation PR
This commit is contained in:
1 parent
398027b724
commit
4b86e7a86b
19 files changed
+421
-259
No files matched your search
@@ -123,6 +123,7 @@ _Not all options working now!_
|
||||
--hyphenation True|False >> ((default) is True)
|
||||
--disable_separation True|False >> ((default) is False)
|
||||
--disable_karaoke True|False >> ((default) is False)
|
||||
--ignore_audio True|False >> ((default) is False)
|
||||
--create_audio_chunks True|False >> ((default) is False)
|
||||
--keep_cache True|False >> ((default) is False)
|
||||
--plot True|False >> ((default) is False)
|
||||
|
||||
@@ -30,14 +30,13 @@ class ConvertToTranscribedDataTest(unittest.TestCase):
|
||||
|
||||
# Words should have space at the end
|
||||
expected_output = [
|
||||
TranscribedData(
|
||||
{"word": "UltraSinger ", "start": 1.23, "end": 2.34, "is_hyphen": None, "confidence": 0.95}),
|
||||
TranscribedData({"word": "is ", "start": 2.34, "end": 3.45, "is_hyphen": None, "confidence": 0.9}),
|
||||
TranscribedData({"word": "cool! ", "start": 3.45, "end": 4.56, "is_hyphen": None, "confidence": 0.85}),
|
||||
TranscribedData({"word": "And ", "start": 4.56, "end": 5.67, "is_hyphen": None, "confidence": 0.95}),
|
||||
TranscribedData({"word": "will ", "start": 5.67, "end": 6.78, "is_hyphen": None, "confidence": 0.9}),
|
||||
TranscribedData({"word": "be ", "start": 6.78, "end": 7.89, "is_hyphen": None, "confidence": 0.85}),
|
||||
TranscribedData({"word": "better! ", "start": 7.89, "end": 9.01, "is_hyphen": None, "confidence": 0.8}),
|
||||
TranscribedData.from_dict({"word": "UltraSinger ", "start": 1.23, "end": 2.34, "is_hyphen": None, "confidence": 0.95}),
|
||||
TranscribedData.from_dict({"word": "is ", "start": 2.34, "end": 3.45, "is_hyphen": None, "confidence": 0.9}),
|
||||
TranscribedData.from_dict({"word": "cool! ", "start": 3.45, "end": 4.56, "is_hyphen": None, "confidence": 0.85}),
|
||||
TranscribedData.from_dict({"word": "And ", "start": 4.56, "end": 5.67, "is_hyphen": None, "confidence": 0.95}),
|
||||
TranscribedData.from_dict({"word": "will ", "start": 5.67, "end": 6.78, "is_hyphen": None, "confidence": 0.9}),
|
||||
TranscribedData.from_dict({"word": "be ", "start": 6.78, "end": 7.89, "is_hyphen": None, "confidence": 0.85}),
|
||||
TranscribedData.from_dict({"word": "better! ", "start": 7.89, "end": 9.01, "is_hyphen": None, "confidence": 0.8}),
|
||||
]
|
||||
|
||||
# Act
|
||||
|
||||
@@ -68,20 +68,20 @@ class TestCreateUltrastarTxt(unittest.TestCase):
|
||||
def arrange(self):
|
||||
# Arrange
|
||||
transcribed_data = [
|
||||
TranscribedData({
|
||||
"conf": 0.95,
|
||||
TranscribedData.from_dict({
|
||||
"confidence": 0.95,
|
||||
"word": "UltraSinger ",
|
||||
"end": 2.5,
|
||||
"start": 0.5
|
||||
}),
|
||||
TranscribedData({
|
||||
"conf": 0.9,
|
||||
TranscribedData.from_dict({
|
||||
"confidence": 0.9,
|
||||
"word": "is ",
|
||||
"end": 4.5,
|
||||
"start": 3.0
|
||||
}),
|
||||
TranscribedData({
|
||||
"conf": 0.85,
|
||||
TranscribedData.from_dict({
|
||||
"confidence": 0.85,
|
||||
"word": "cool! ",
|
||||
"end": 7.5,
|
||||
"start": 5.5
|
||||
@@ -110,20 +110,27 @@ class TestCreateUltrastarTxt(unittest.TestCase):
|
||||
expected_calls = []
|
||||
if version.parse(ver) >= version.parse("1.0.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VERSION}:{default_ultrastar_class.version}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n")
|
||||
expected_calls += [
|
||||
f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n",
|
||||
f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n",
|
||||
f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n"
|
||||
]
|
||||
if version.parse(ver) >= version.parse("1.1.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n") # todo: video is optional
|
||||
expected_calls.append(f"#{UltrastarTxtTag.BPM}:390.0\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.GAP}:500\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n")
|
||||
expected_calls.append(": 0 52 1 UltraSinger \n")
|
||||
expected_calls.append(": 65 39 2 is \n")
|
||||
expected_calls.append(": 130 52 3 cool! \n")
|
||||
expected_calls.append("E")
|
||||
expected_calls += [f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n"]
|
||||
if default_ultrastar_class.video is not None:
|
||||
expected_calls += [
|
||||
f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n"
|
||||
]
|
||||
expected_calls += [
|
||||
f"#{UltrastarTxtTag.BPM}:390.0\n",
|
||||
f"#{UltrastarTxtTag.GAP}:500\n",
|
||||
f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n",
|
||||
f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n",
|
||||
": 0 52 1 UltraSinger \n",
|
||||
": 65 39 2 is \n",
|
||||
": 130 52 3 cool! \n",
|
||||
"E"
|
||||
]
|
||||
|
||||
return expected_calls
|
||||
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from dataclasses_json import dataclass_json
|
||||
|
||||
|
||||
@dataclass_json
|
||||
@dataclass
|
||||
class Settings:
|
||||
APP_VERSION = "0.0.11-dev4"
|
||||
|
||||
@@ -7,6 +14,8 @@ class Settings:
|
||||
hyphenation = True
|
||||
use_separated_vocal = True
|
||||
create_karaoke = True
|
||||
ignore_audio = False
|
||||
input_file_is_ultrastar_txt = False
|
||||
keep_cache = False
|
||||
|
||||
input_file_path = ""
|
||||
@@ -37,3 +46,12 @@ class Settings:
|
||||
force_cpu = False
|
||||
force_whisper_cpu = False
|
||||
force_crepe_cpu = False
|
||||
|
||||
# UltraSinger Evaluation Configuration
|
||||
test_songs_input_folder = None
|
||||
cache_override_path = None
|
||||
skip_cache_vocal_separation = False
|
||||
skip_cache_denoise_vocal_audio = False
|
||||
skip_cache_transcription = False
|
||||
skip_cache_pitch_detection = False
|
||||
calculate_score = True
|
||||
+201
-133
@@ -22,6 +22,8 @@ from modules.Audio.vocal_chunks import (
|
||||
export_chunks_from_ultrastar_data,
|
||||
)
|
||||
from modules.Audio.silence_processing import remove_silence_from_transcription_data, get_silence_sections
|
||||
from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult
|
||||
from modules.Ultrastar.ultrastar_score_calculator import Score
|
||||
from modules.csv_handler import export_transcribed_data_to_csv
|
||||
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
|
||||
from modules.Audio.youtube import (
|
||||
@@ -37,6 +39,7 @@ from modules.console_colors import (
|
||||
gold_highlighted,
|
||||
light_blue_highlighted,
|
||||
red_highlighted,
|
||||
green_highlighted,
|
||||
)
|
||||
from modules.Midi.midi_creator import (
|
||||
convert_frequencies_to_notes,
|
||||
@@ -52,12 +55,22 @@ from modules.Pitcher.pitcher import (
|
||||
get_pitch_with_crepe_file,
|
||||
)
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
from modules.Speech_Recognition.hyphenation import hyphenation, language_check, create_hyphenator
|
||||
from modules.Speech_Recognition.hyphenation import (
|
||||
hyphenation,
|
||||
language_check,
|
||||
create_hyphenator,
|
||||
)
|
||||
from modules.Speech_Recognition.Whisper import transcribe_with_whisper
|
||||
from modules.Ultrastar import ultrastar_score_calculator, ultrastar_writer, ultrastar_converter, ultrastar_parser
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
|
||||
from modules.Ultrastar import (
|
||||
ultrastar_score_calculator,
|
||||
ultrastar_writer,
|
||||
ultrastar_converter,
|
||||
ultrastar_parser,
|
||||
)
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, FILE_ENCODING
|
||||
from Settings import Settings
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
from modules.os_helper import check_file_exists
|
||||
from modules.plot import plot, plot_spectrogram
|
||||
from modules.musicbrainz_client import get_music_infos
|
||||
|
||||
@@ -66,9 +79,7 @@ settings = Settings()
|
||||
|
||||
def pitch_each_chunk_with_crepe(directory: str) -> list[str]:
|
||||
"""Pitch each chunk with crepe and return midi notes"""
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}"
|
||||
)
|
||||
print(f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}")
|
||||
|
||||
midi_notes = []
|
||||
for filename in sorted(
|
||||
@@ -97,7 +108,9 @@ def pitch_each_chunk_with_crepe(directory: str) -> list[str]:
|
||||
return midi_notes
|
||||
|
||||
|
||||
def add_hyphen_to_data(transcribed_data: list[TranscribedData], hyphen_words: list[list[str]]):
|
||||
def add_hyphen_to_data(
|
||||
transcribed_data: list[TranscribedData], hyphen_words: list[list[str]]
|
||||
):
|
||||
"""Add hyphen to transcribed data return new data list"""
|
||||
new_data = []
|
||||
|
||||
@@ -133,9 +146,7 @@ def get_bpm_from_data(data, sampling_rate):
|
||||
onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate)
|
||||
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}"
|
||||
)
|
||||
print(f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}")
|
||||
return wav_tempo[0]
|
||||
|
||||
|
||||
@@ -226,12 +237,12 @@ def remove_unecessary_punctuations(transcribed_data: list[TranscribedData]) -> N
|
||||
"""Remove unecessary punctuations from transcribed data"""
|
||||
punctuation = ".,"
|
||||
for i, data in enumerate(transcribed_data):
|
||||
data.word = data.word.translate(
|
||||
{ord(i): None for i in punctuation}
|
||||
)
|
||||
data.word = data.word.translate({ord(i): None for i in punctuation})
|
||||
|
||||
|
||||
def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData]) -> list[list[str]] | None:
|
||||
def hyphenate_each_word(
|
||||
language: str, transcribed_data: list[TranscribedData]
|
||||
) -> list[list[str]] | None:
|
||||
"""Hyphenate each word in the transcribed data."""
|
||||
lang_region = language_check(language)
|
||||
if lang_region is None:
|
||||
@@ -281,17 +292,15 @@ def print_version() -> None:
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('*****************************')}"
|
||||
)
|
||||
|
||||
def run() -> None:
|
||||
def run() -> tuple[str, Score, Score]:
|
||||
"""The processing function of this program"""
|
||||
is_audio = ".txt" not in settings.input_file_path
|
||||
settings.input_file_is_ultrastar_txt = settings.input_file_path.endswith(".txt")
|
||||
|
||||
ultrastar_class = None
|
||||
real_bpm = None
|
||||
(title, artist, year, genre) = (None, None, None, None)
|
||||
|
||||
if not is_audio: # Parse Ultrastar txt
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('re-pitch mode')}"
|
||||
)
|
||||
if settings.input_file_is_ultrastar_txt: # Parse Ultrastar txt
|
||||
(
|
||||
basename_without_ext,
|
||||
real_bpm,
|
||||
@@ -299,28 +308,35 @@ def run() -> None:
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_class,
|
||||
) = parse_ultrastar_txt()
|
||||
|
||||
if not ultrastar_class.mp3:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error!')} The provided text file does not have a reference to "
|
||||
f"an audio file."
|
||||
)
|
||||
exit(1)
|
||||
elif settings.input_file_path.startswith("https:"): # Youtube
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
|
||||
)
|
||||
print(f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}")
|
||||
(
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
(title, artist, year, genre)
|
||||
(title, artist, year, genre),
|
||||
) = download_from_youtube()
|
||||
else: # Audio File
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
|
||||
)
|
||||
print(f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}")
|
||||
(
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
(title, artist, year, genre)
|
||||
(title, artist, year, genre),
|
||||
) = infos_from_audio_input_file()
|
||||
|
||||
cache_path = os.path.join(song_output, "cache")
|
||||
cache_path = (
|
||||
os.path.join(song_output, "cache")
|
||||
if settings.cache_override_path is None
|
||||
else settings.cache_override_path
|
||||
)
|
||||
settings.processing_audio_path = os.path.join(
|
||||
cache_path, basename_without_ext + ".wav"
|
||||
)
|
||||
@@ -373,11 +389,12 @@ def run() -> None:
|
||||
# Audio transcription
|
||||
transcribed_data = None
|
||||
language = settings.language
|
||||
if is_audio:
|
||||
detected_language, transcribed_data = transcribe_audio()
|
||||
if not settings.ignore_audio:
|
||||
transcription_result = transcribe_audio(cache_path)
|
||||
if language is None:
|
||||
language = detected_language
|
||||
language = transcription_result.detected_language
|
||||
|
||||
transcribed_data = transcription_result.transcribed_data
|
||||
remove_unecessary_punctuations(transcribed_data)
|
||||
|
||||
if settings.hyphenation:
|
||||
@@ -397,7 +414,6 @@ def run() -> None:
|
||||
if settings.create_audio_chunks:
|
||||
create_audio_chunks(
|
||||
cache_path,
|
||||
is_audio,
|
||||
transcribed_data,
|
||||
ultrastar_audio_input_path,
|
||||
ultrastar_class,
|
||||
@@ -405,7 +421,7 @@ def run() -> None:
|
||||
|
||||
# Pitch the audio
|
||||
midi_segments, pitched_data, ultrastar_note_numbers, transcribed_data = pitch_audio(
|
||||
is_audio, transcribed_data, ultrastar_class
|
||||
transcribed_data, ultrastar_class, cache_path
|
||||
)
|
||||
|
||||
# Create plot
|
||||
@@ -416,7 +432,7 @@ def run() -> None:
|
||||
plot(pitched_data, song_output, midi_segments)
|
||||
|
||||
# Write Ultrastar txt
|
||||
if is_audio:
|
||||
if not settings.ignore_audio:
|
||||
real_bpm, ultrastar_file_output = create_ultrastar_txt_from_automation(
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
@@ -434,15 +450,16 @@ def run() -> None:
|
||||
song_output, ultrastar_class, ultrastar_note_numbers
|
||||
)
|
||||
|
||||
# Calc Points
|
||||
ultrastar_class, simple_score, accurate_score = calculate_score_points(
|
||||
is_audio, pitched_data, ultrastar_class, ultrastar_file_output
|
||||
)
|
||||
simple_score = None
|
||||
accurate_score = None
|
||||
if settings.calculate_score:
|
||||
# Calc Points
|
||||
ultrastar_class, simple_score, accurate_score = calculate_score_points(
|
||||
pitched_data, ultrastar_class, ultrastar_file_output
|
||||
)
|
||||
|
||||
# Add calculated score to Ultrastar txt #Todo: Missing Karaoke
|
||||
ultrastar_writer.add_score_to_ultrastar_txt(
|
||||
ultrastar_file_output, simple_score
|
||||
)
|
||||
ultrastar_writer.add_score_to_ultrastar_txt(ultrastar_file_output, simple_score)
|
||||
|
||||
# Midi
|
||||
if settings.create_midi:
|
||||
@@ -454,6 +471,7 @@ def run() -> None:
|
||||
|
||||
# Print Support
|
||||
print_support()
|
||||
return ultrastar_file_output, simple_score, accurate_score
|
||||
|
||||
|
||||
def mute_no_singing_parts(mono_output_path, mute_output_path):
|
||||
@@ -493,58 +511,85 @@ def get_unused_song_output_dir(path: str) -> str:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error: Could not create output folder! (999) is the maximum number of tries.')}"
|
||||
)
|
||||
sys.exit(1)
|
||||
raise ValueError("Could not create output folder! (999) is the maximum number of tries.")
|
||||
return path
|
||||
|
||||
|
||||
def transcribe_audio() -> (str, list[TranscribedData]):
|
||||
def transcribe_audio(cache_path: str) -> TranscriptionResult:
|
||||
"""Transcribe audio with AI"""
|
||||
transcription_result = None
|
||||
if settings.transcriber == "whisper":
|
||||
device = "cpu" if settings.force_whisper_cpu else settings.pytorch_device
|
||||
transcribed_data, detected_language = transcribe_with_whisper(
|
||||
settings.processing_audio_path,
|
||||
settings.whisper_model,
|
||||
device,
|
||||
settings.whisper_align_model,
|
||||
settings.whisper_batch_size,
|
||||
settings.whisper_compute_type,
|
||||
settings.language,
|
||||
)
|
||||
transcription_config = f"{settings.transcriber}_{settings.whisper_model}_{settings.pytorch_device}_{settings.whisper_align_model}_{settings.whisper_align_model}_{settings.whisper_batch_size}_{settings.whisper_compute_type}_{settings.language}"
|
||||
transcription_path = os.path.join(cache_path, f"{transcription_config}.json")
|
||||
cached_transcription_available = check_file_exists(transcription_path)
|
||||
if settings.skip_cache_transcription or not cached_transcription_available:
|
||||
transcription_result = transcribe_with_whisper(
|
||||
settings.processing_audio_path,
|
||||
settings.whisper_model,
|
||||
settings.pytorch_device,
|
||||
settings.whisper_align_model,
|
||||
settings.whisper_batch_size,
|
||||
settings.whisper_compute_type,
|
||||
settings.language,
|
||||
)
|
||||
with open(transcription_path, "w", encoding=FILE_ENCODING) as file:
|
||||
file.write(transcription_result.to_json())
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached transcribed data")
|
||||
with open(transcription_path) as file:
|
||||
json = file.read()
|
||||
transcription_result = TranscriptionResult.from_json(json)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
return detected_language, transcribed_data
|
||||
return transcription_result
|
||||
|
||||
|
||||
def separate_vocal_from_audio(
|
||||
basename_without_ext: str, cache_path: str, ultrastar_audio_input_path: str
|
||||
basename_without_ext: str, cache_path: str, ultrastar_audio_input_path: str
|
||||
) -> str:
|
||||
"""Separate vocal from audio"""
|
||||
demucs_output_folder = os.path.splitext(
|
||||
os.path.basename(ultrastar_audio_input_path)
|
||||
)[0]
|
||||
audio_separation_path = os.path.join(
|
||||
cache_path, "separated", "htdemucs", basename_without_ext
|
||||
cache_path, "separated", "htdemucs", demucs_output_folder
|
||||
)
|
||||
|
||||
vocals_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
instrumental_path = os.path.join(audio_separation_path, "no_vocals.wav")
|
||||
if settings.use_separated_vocal or settings.create_karaoke:
|
||||
separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device)
|
||||
cache_available = check_file_exists(vocals_path) and check_file_exists(
|
||||
instrumental_path
|
||||
)
|
||||
if settings.skip_cache_vocal_separation or not cache_available:
|
||||
separate_audio(
|
||||
ultrastar_audio_input_path, cache_path, settings.pytorch_device
|
||||
)
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached separated vocals")
|
||||
|
||||
if settings.use_separated_vocal:
|
||||
input_path = vocals_path
|
||||
else:
|
||||
input_path = ultrastar_audio_input_path
|
||||
|
||||
convert_audio_to_mono_wav(input_path, settings.processing_audio_path)
|
||||
|
||||
return audio_separation_path
|
||||
|
||||
def calculate_score_points(
|
||||
is_audio: bool, pitched_data: PitchedData, ultrastar_class: UltrastarTxtValue, ultrastar_file_output: str
|
||||
pitched_data: PitchedData,
|
||||
ultrastar_class: UltrastarTxtValue,
|
||||
ultrastar_file_output: str,
|
||||
):
|
||||
"""Calculate score points"""
|
||||
if is_audio:
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
ultrastar_file_output
|
||||
)
|
||||
if not settings.ignore_audio:
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output)
|
||||
(
|
||||
simple_score,
|
||||
accurate_score,
|
||||
) = ultrastar_score_calculator.calculate_score(
|
||||
pitched_data, ultrastar_class
|
||||
)
|
||||
ultrastar_score_calculator.print_score_calculation(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
) = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
|
||||
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
|
||||
else:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of original Ultrastar txt')}"
|
||||
@@ -552,32 +597,24 @@ def calculate_score_points(
|
||||
(
|
||||
simple_score,
|
||||
accurate_score,
|
||||
) = ultrastar_score_calculator.calculate_score(
|
||||
pitched_data, ultrastar_class
|
||||
)
|
||||
ultrastar_score_calculator.print_score_calculation(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
) = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
|
||||
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of re-pitched Ultrastar txt')}"
|
||||
)
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
ultrastar_file_output
|
||||
)
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output)
|
||||
(
|
||||
simple_score,
|
||||
accurate_score,
|
||||
) = ultrastar_score_calculator.calculate_score(
|
||||
pitched_data, ultrastar_class
|
||||
)
|
||||
ultrastar_score_calculator.print_score_calculation(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
) = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class)
|
||||
ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score)
|
||||
return ultrastar_class, simple_score, accurate_score
|
||||
|
||||
|
||||
def create_ultrastar_txt_from_ultrastar_data(
|
||||
song_output: str, ultrastar_class: UltrastarTxtValue, ultrastar_note_numbers: list[int]
|
||||
song_output: str,
|
||||
ultrastar_class: UltrastarTxtValue,
|
||||
ultrastar_note_numbers: list[int],
|
||||
) -> str:
|
||||
"""Create Ultrastar txt from Ultrastar data"""
|
||||
output_repitched_ultrastar = os.path.join(
|
||||
@@ -616,9 +653,7 @@ def create_ultrastar_txt_from_automation(
|
||||
ultrastar_header.language = language
|
||||
cover = basename_without_ext + " [CO].jpg"
|
||||
ultrastar_header.cover = (
|
||||
cover
|
||||
if os_helper.check_file_exists(os.path.join(song_output, cover))
|
||||
else None
|
||||
cover if os_helper.check_file_exists(os.path.join(song_output, cover)) else None
|
||||
)
|
||||
ultrastar_header.creator = f"{ultrastar_header.creator} {Settings.APP_VERSION}"
|
||||
ultrastar_header.comment = f"{ultrastar_header.comment} {Settings.APP_VERSION}"
|
||||
@@ -634,9 +669,7 @@ def create_ultrastar_txt_from_automation(
|
||||
ultrastar_header.genre = format_separated_string(genre)
|
||||
|
||||
real_bpm = get_bpm_from_file(ultrastar_audio_input_path)
|
||||
ultrastar_file_output = os.path.join(
|
||||
song_output, basename_without_ext + ".txt"
|
||||
)
|
||||
ultrastar_file_output = os.path.join(song_output, basename_without_ext + ".txt")
|
||||
ultrastar_writer.create_ultrastar_txt_from_automation(
|
||||
transcribed_data,
|
||||
ultrastar_note_numbers,
|
||||
@@ -715,9 +748,17 @@ def infos_from_audio_input_file() -> tuple[str, str, str, tuple[str, str, str, s
|
||||
song_output = get_unused_song_output_dir(song_output)
|
||||
os_helper.create_folder(song_output)
|
||||
os_helper.copy(settings.input_file_path, song_output)
|
||||
os_helper.rename(os.path.join(song_output, os.path.basename(settings.input_file_path)), os.path.join(song_output, basename))
|
||||
os_helper.rename(
|
||||
os.path.join(song_output, os.path.basename(settings.input_file_path)),
|
||||
os.path.join(song_output, basename),
|
||||
)
|
||||
ultrastar_audio_input_path = os.path.join(song_output, basename)
|
||||
return basename_without_ext, song_output, ultrastar_audio_input_path, (title, artist, year_info, genre_info)
|
||||
return (
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
(title, artist, year_info, genre_info),
|
||||
)
|
||||
|
||||
|
||||
FILENAME_REPLACEMENTS = (('?:"', ""), ("<", "("), (">", ")"), ("/\\|*", "-"))
|
||||
@@ -738,7 +779,9 @@ def download_from_youtube() -> tuple[str, str, str, tuple[str, str, str, str]]:
|
||||
(artist, title) = get_youtube_title(settings.input_file_path)
|
||||
|
||||
# Get additional data for song
|
||||
(title_info, artist_info, year_info, genre_info) = get_music_infos(f"{artist} - {title}")
|
||||
(title_info, artist_info, year_info, genre_info) = get_music_infos(
|
||||
f"{artist} - {title}"
|
||||
)
|
||||
|
||||
if title_info is not None:
|
||||
title = title_info
|
||||
@@ -749,29 +792,29 @@ def download_from_youtube() -> tuple[str, str, str, tuple[str, str, str, str]]:
|
||||
song_output = os.path.join(settings.output_file_path, basename_without_ext)
|
||||
song_output = get_unused_song_output_dir(song_output)
|
||||
os_helper.create_folder(song_output)
|
||||
download_youtube_audio(
|
||||
settings.input_file_path, basename_without_ext, song_output
|
||||
)
|
||||
download_youtube_video(
|
||||
settings.input_file_path, basename_without_ext, song_output
|
||||
)
|
||||
download_youtube_audio(settings.input_file_path, basename_without_ext, song_output)
|
||||
download_youtube_video(settings.input_file_path, basename_without_ext, song_output)
|
||||
download_youtube_thumbnail(
|
||||
settings.input_file_path, basename_without_ext, song_output
|
||||
)
|
||||
ultrastar_audio_input_path = os.path.join(song_output, basename)
|
||||
return basename_without_ext, song_output, ultrastar_audio_input_path, (title, artist, year_info, genre_info)
|
||||
return (
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
ultrastar_audio_input_path,
|
||||
(title, artist, year_info, genre_info),
|
||||
)
|
||||
|
||||
|
||||
def parse_ultrastar_txt() -> tuple[str, float, str, str, UltrastarTxtValue]:
|
||||
"""Parse Ultrastar txt"""
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
settings.input_file_path
|
||||
)
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(settings.input_file_path)
|
||||
real_bpm = ultrastar_converter.ultrastar_bpm_to_real_bpm(
|
||||
float(ultrastar_class.bpm.replace(",", "."))
|
||||
)
|
||||
ultrastar_mp3_name = ultrastar_class.mp3
|
||||
basename_without_ext = os.path.splitext(ultrastar_mp3_name)[0]
|
||||
|
||||
basename_without_ext = f"{ultrastar_class.artist} - {ultrastar_class.title}"
|
||||
dirname = os.path.dirname(settings.input_file_path)
|
||||
ultrastar_audio_input_path = os.path.join(dirname, ultrastar_mp3_name)
|
||||
song_output = os.path.join(
|
||||
@@ -790,37 +833,53 @@ def parse_ultrastar_txt() -> tuple[str, float, str, str, UltrastarTxtValue]:
|
||||
)
|
||||
|
||||
|
||||
def create_midi_file(real_bpm: float,
|
||||
song_output: str,
|
||||
ultrastar_class: UltrastarTxtValue,
|
||||
basename_without_ext: str) -> None:
|
||||
def create_midi_file(
|
||||
real_bpm: float,
|
||||
song_output: str,
|
||||
ultrastar_class: UltrastarTxtValue,
|
||||
basename_without_ext: str,
|
||||
) -> None:
|
||||
"""Create midi file"""
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}"
|
||||
)
|
||||
print(f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}")
|
||||
|
||||
voice_instrument = [
|
||||
convert_ultrastar_to_midi_instrument(ultrastar_class)
|
||||
]
|
||||
midi_output = os.path.join(song_output, f"{basename_without_ext}.mid")
|
||||
instruments_to_midi(
|
||||
voice_instrument, real_bpm, midi_output
|
||||
)
|
||||
instruments_to_midi(voice_instrument, real_bpm, midi_output)
|
||||
|
||||
|
||||
def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrastar_class: UltrastarTxtValue) -> tuple[
|
||||
list[MidiSegment], PitchedData, list[int], list[TranscribedData]]:
|
||||
def pitch_audio(
|
||||
transcribed_data: list[TranscribedData],
|
||||
ultrastar_class: UltrastarTxtValue,
|
||||
cache_path: str
|
||||
) -> tuple[list[MidiSegment], PitchedData, list[int], list[TranscribedData]]:
|
||||
"""Pitch audio"""
|
||||
# todo: chunk pitching as option?
|
||||
# midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name)
|
||||
device = "cpu" if settings.force_crepe_cpu else settings.tensorflow_device
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
settings.processing_audio_path,
|
||||
settings.crepe_model_capacity,
|
||||
settings.crepe_step_size,
|
||||
device,
|
||||
)
|
||||
if is_audio:
|
||||
|
||||
pitching_config = f"crepe_{settings.ignore_audio}_{settings.crepe_model_capacity}_{settings.crepe_step_size}_{settings.tensorflow_device}"
|
||||
pitched_data_path = os.path.join(cache_path, f"{pitching_config}.json")
|
||||
cache_available = check_file_exists(pitched_data_path)
|
||||
|
||||
if settings.skip_cache_transcription or not cache_available:
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
settings.processing_audio_path,
|
||||
settings.crepe_model_capacity,
|
||||
settings.crepe_step_size,
|
||||
settings.tensorflow_device,
|
||||
)
|
||||
|
||||
pitched_data_json = pitched_data.to_json()
|
||||
with open(pitched_data_path, "w", encoding=FILE_ENCODING) as file:
|
||||
file.write(pitched_data_json)
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached pitch data")
|
||||
with open(pitched_data_path) as file:
|
||||
json = file.read()
|
||||
pitched_data = PitchedData.from_json(json)
|
||||
|
||||
if not settings.ignore_audio:
|
||||
start_times = []
|
||||
end_times = []
|
||||
words = []
|
||||
@@ -828,34 +887,36 @@ def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrast
|
||||
start_times.append(midi_segment.start)
|
||||
end_times.append(midi_segment.end)
|
||||
words.append(midi_segment.word)
|
||||
midi_segments = create_midi_notes_from_pitched_data(start_times, end_times, words, pitched_data)
|
||||
midi_segments = create_midi_notes_from_pitched_data(
|
||||
start_times, end_times, words, pitched_data
|
||||
)
|
||||
|
||||
else:
|
||||
midi_segments = create_midi_notes_from_pitched_data(
|
||||
ultrastar_class.startTimes, ultrastar_class.endTimes, ultrastar_class.words, pitched_data
|
||||
)
|
||||
|
||||
ultrastar_note_numbers = convert_midi_notes_to_ultrastar_notes(midi_segments)
|
||||
|
||||
new_transcribed_data = []
|
||||
for i, midi_segment in enumerate(midi_segments):
|
||||
new_transcribed_data.append(TranscribedData({"word": midi_segment.word, "start": midi_segment.start, "end": midi_segment.end, "is_hyphen": None, "confidence": 1}))
|
||||
new_transcribed_data.append(TranscribedData(
|
||||
{"word": midi_segment.word, "start": midi_segment.start, "end": midi_segment.end, "is_hyphen": None,
|
||||
"confidence": 1}))
|
||||
|
||||
return midi_segments, pitched_data, ultrastar_note_numbers, new_transcribed_data
|
||||
|
||||
|
||||
def create_audio_chunks(
|
||||
cache_path: str,
|
||||
is_audio: bool,
|
||||
transcribed_data: list[TranscribedData],
|
||||
ultrastar_audio_input_path: str,
|
||||
ultrastar_class: UltrastarTxtValue
|
||||
) -> None:
|
||||
"""Create audio chunks"""
|
||||
audio_chunks_path = os.path.join(
|
||||
cache_path, settings.audio_chunk_folder_name
|
||||
)
|
||||
audio_chunks_path = os.path.join(cache_path, settings.audio_chunk_folder_name)
|
||||
os_helper.create_folder(audio_chunks_path)
|
||||
if is_audio: # and csv
|
||||
if not settings.ignore_audio: # and csv
|
||||
csv_filename = os.path.join(audio_chunks_path, "_chunks.csv")
|
||||
export_chunks_from_transcribed_data(
|
||||
settings.processing_audio_path, transcribed_data, audio_chunks_path
|
||||
@@ -868,7 +929,11 @@ def create_audio_chunks(
|
||||
|
||||
def denoise_vocal_audio(input_path: str, output_path: str) -> None:
|
||||
"""Denoise vocal audio"""
|
||||
ffmpeg_reduce_noise(input_path, output_path)
|
||||
cache_available = check_file_exists(output_path)
|
||||
if settings.skip_cache_denoise_vocal_audio or not cache_available:
|
||||
ffmpeg_reduce_noise(input_path, output_path)
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached denoised audio")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> None:
|
||||
@@ -924,6 +989,8 @@ def init_settings(argv: list[str]) -> None:
|
||||
settings.create_karaoke = not arg
|
||||
elif opt in ("--create_audio_chunks"):
|
||||
settings.create_audio_chunks = arg
|
||||
elif opt in ("--ignore_audio"):
|
||||
settings.ignore_audio = arg in ["True", "true"]
|
||||
elif opt in ("--force_cpu"):
|
||||
settings.force_cpu = arg
|
||||
if settings.force_cpu:
|
||||
@@ -970,6 +1037,7 @@ def arg_options():
|
||||
"disable_separation=",
|
||||
"disable_karaoke=",
|
||||
"create_audio_chunks=",
|
||||
"ignore_audio=",
|
||||
"force_cpu=",
|
||||
"force_whisper_cpu=",
|
||||
"force_crepe_cpu=",
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
"""Separate vocals from audio"""
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
import demucs.separate
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
@@ -10,16 +13,18 @@ from modules.console_colors import (
|
||||
from modules.os_helper import current_executor_path, move, path_join
|
||||
|
||||
|
||||
def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None:
|
||||
def separate_audio(input_file_path: str, output_folder: str, device="cpu") -> None:
|
||||
"""Separate vocals from audio with demucs."""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker."
|
||||
)
|
||||
# Model selection?
|
||||
# -n htdemucs_ft
|
||||
subprocess.run(
|
||||
["demucs", "-d", device, "--two-stems=vocals", "--float32", input_file_path]
|
||||
)
|
||||
separated_folder = path_join(current_executor_path(), "separated")
|
||||
move(separated_folder, output_file)
|
||||
|
||||
# subprocess.run(
|
||||
# ["demucs", "-d", device, "--two-stems=vocals", "--float32", input_file_path]
|
||||
# )
|
||||
# separated_folder = path_join(current_executor_path(), "separated")
|
||||
# move(separated_folder, output_file)
|
||||
|
||||
# fixme "--float32" is missing
|
||||
demucs.separate.main(shlex.split(f'--two-stems vocals -d {device} --out "{os.path.join(output_folder, "separated")}" "{input_file_path}"'))
|
||||
@@ -63,7 +63,7 @@ def remove_silence(silence_parts_list: list[tuple[float, float]], transcribed_da
|
||||
split_word = "~ "
|
||||
is_word_end = True
|
||||
|
||||
split_data = TranscribedData({"conf": data.conf, "word": split_word, "end": split_end, "start": silence_end, "is_word_end": is_word_end})
|
||||
split_data = TranscribedData({"confidence": data.confidence, "word": split_word, "end": split_end, "start": silence_end, "is_word_end": is_word_end})
|
||||
|
||||
if not was_split:
|
||||
data.end = silence_start
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
"""Pitched data"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from dataclasses_json import dataclass_json
|
||||
|
||||
|
||||
@dataclass_json
|
||||
@dataclass
|
||||
class PitchedData:
|
||||
"""Pitched data from crepe"""
|
||||
|
||||
@@ -30,6 +30,10 @@ def get_pitch_with_crepe(
|
||||
times, frequencies, confidence, activation = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
|
||||
# convert to native float for serialization
|
||||
confidence = [float(x) for x in confidence]
|
||||
|
||||
return PitchedData(times, frequencies, confidence)
|
||||
|
||||
|
||||
|
||||
@@ -1,21 +1,29 @@
|
||||
"""Docstring"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from dataclasses_json import dataclass_json
|
||||
|
||||
|
||||
@dataclass_json
|
||||
@dataclass
|
||||
class TranscribedData:
|
||||
"""Transcribed data from json file"""
|
||||
|
||||
def __init__(self, transcribed_json = None):
|
||||
if transcribed_json is None:
|
||||
return
|
||||
# Vosk = conf, Whisper = confidence
|
||||
self.conf = transcribed_json.get(
|
||||
"conf", transcribed_json.get("confidence", None)
|
||||
)
|
||||
# Vosk = word, Whisper = text
|
||||
self.word = transcribed_json.get(
|
||||
"word", transcribed_json.get("text", None)
|
||||
)
|
||||
self.end = transcribed_json.get("end", None)
|
||||
self.start = transcribed_json.get("start", None)
|
||||
self.is_hyphen = transcribed_json.get("is_hyphen", None)
|
||||
self.is_word_end = transcribed_json.get("is_word_end", True)
|
||||
confidence: float = 0
|
||||
word: str = ""
|
||||
start: float = 0
|
||||
end: float = 0
|
||||
is_hyphen: bool = False
|
||||
is_word_end: bool = True
|
||||
|
||||
|
||||
def from_whisper(whisper_dict) -> TranscribedData:
|
||||
transcribed_data = TranscribedData()
|
||||
if "score" in whisper_dict:
|
||||
transcribed_data.confidence = whisper_dict["score"]
|
||||
if "word" in whisper_dict:
|
||||
transcribed_data.word = whisper_dict["word"]
|
||||
if "start" in whisper_dict:
|
||||
transcribed_data.start = whisper_dict["start"]
|
||||
if "end" in whisper_dict:
|
||||
transcribed_data.end = whisper_dict["end"]
|
||||
return transcribed_data
|
||||
@@ -0,0 +1,14 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from dataclasses_json import dataclass_json
|
||||
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
@dataclass_json
|
||||
@dataclass
|
||||
class TranscriptionResult:
|
||||
"""Transcription result"""
|
||||
|
||||
transcribed_data: list[TranscribedData]
|
||||
detected_language: str
|
||||
@@ -2,11 +2,16 @@
|
||||
|
||||
import sys
|
||||
|
||||
import torch
|
||||
import whisperx
|
||||
from torch.cuda import OutOfMemoryError
|
||||
|
||||
from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData, from_whisper
|
||||
|
||||
|
||||
MEMORY_ERROR_MESSAGE = f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu"
|
||||
|
||||
|
||||
def transcribe_with_whisper(
|
||||
@@ -17,7 +22,7 @@ def transcribe_with_whisper(
|
||||
batch_size: int = 16,
|
||||
compute_type: str = None,
|
||||
language: str = None,
|
||||
) -> (list[TranscribedData], str):
|
||||
) -> TranscriptionResult:
|
||||
"""Transcribe with whisper"""
|
||||
|
||||
# Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work.
|
||||
@@ -32,9 +37,50 @@ def transcribe_with_whisper(
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
|
||||
try:
|
||||
torch.cuda.empty_cache()
|
||||
loaded_whisper_model = whisperx.load_model(
|
||||
model, language=language, device=device, compute_type=compute_type
|
||||
)
|
||||
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(
|
||||
audio, batch_size=batch_size, language=language
|
||||
)
|
||||
|
||||
detected_language = result["language"]
|
||||
if language is None:
|
||||
language = detected_language
|
||||
|
||||
# load alignment model and metadata
|
||||
try:
|
||||
model_a, metadata = whisperx.load_align_model(
|
||||
language_code=language, device=device, model_name=model_name
|
||||
)
|
||||
except ValueError as ve:
|
||||
print(
|
||||
f"{red_highlighted(f'{ve}')}"
|
||||
f"\n"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
|
||||
f"Try add it with --align_model [huggingface]."
|
||||
)
|
||||
raise ve
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(
|
||||
result["segments"],
|
||||
model_a,
|
||||
metadata,
|
||||
audio,
|
||||
device,
|
||||
return_char_alignments=False,
|
||||
)
|
||||
|
||||
transcribed_data = convert_to_transcribed_data(result_aligned)
|
||||
|
||||
return TranscriptionResult(transcribed_data, detected_language)
|
||||
except ValueError as value_error:
|
||||
if (
|
||||
"Requested float16 compute type, but the target device or backend do not support efficient float16 computation."
|
||||
@@ -44,68 +90,27 @@ def transcribe_with_whisper(
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Your GPU does not support efficient float16 computation; run UltraSinger with '--whisper_compute_type int8'"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
raise value_error
|
||||
except OutOfMemoryError as oom_exception:
|
||||
print(oom_exception)
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(
|
||||
audio, batch_size=batch_size, language=language
|
||||
)
|
||||
|
||||
detected_language = result["language"]
|
||||
if language is None:
|
||||
language = detected_language
|
||||
|
||||
# load alignment model and metadata
|
||||
try:
|
||||
model_a, metadata = whisperx.load_align_model(
|
||||
language_code=language, device=device, model_name=model_name
|
||||
)
|
||||
except ValueError as ve:
|
||||
print(
|
||||
f"{red_highlighted(f'{ve}')}"
|
||||
f"\n"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
|
||||
f"Try add it with --align_model [huggingface]."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(
|
||||
result["segments"],
|
||||
model_a,
|
||||
metadata,
|
||||
audio,
|
||||
device,
|
||||
return_char_alignments=False,
|
||||
)
|
||||
|
||||
transcribed_data = convert_to_transcribed_data(result_aligned)
|
||||
|
||||
return transcribed_data, detected_language
|
||||
print(MEMORY_ERROR_MESSAGE)
|
||||
raise oom_exception
|
||||
except Exception as exception:
|
||||
if "CUDA failed with error out of memory" in str(exception.args[0]):
|
||||
print(exception)
|
||||
print(MEMORY_ERROR_MESSAGE)
|
||||
raise exception
|
||||
|
||||
|
||||
def convert_to_transcribed_data(result_aligned):
|
||||
transcribed_data = []
|
||||
for segment in result_aligned["segments"]:
|
||||
for obj in segment["words"]:
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
vtd = from_whisper(obj) # create custom Word object
|
||||
vtd.word = vtd.word + " " # add space to end of word
|
||||
if len(obj) < 4:
|
||||
previous = transcribed_data[-1]
|
||||
if not previous:
|
||||
previous.end = 0
|
||||
previous.end = ""
|
||||
previous = transcribed_data[-1] if len(transcribed_data) != 0 else TranscribedData()
|
||||
vtd.start = previous.end + 0.1
|
||||
vtd.end = previous.end + 0.2
|
||||
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
|
||||
|
||||
@@ -48,30 +48,25 @@ def ultrastar_note_to_midi_note(ultrastar_note: int) -> int:
|
||||
return midi_note
|
||||
|
||||
|
||||
def get_start_time_from_ultrastar(ultrastar_class: UltrastarTxtValue, pos: int) -> float:
|
||||
def get_start_time_from_ultrastar(
|
||||
ultrastar_class: UltrastarTxtValue, pos: int
|
||||
) -> float:
|
||||
"""Calculates the start time from the Ultrastar txt"""
|
||||
|
||||
gap = int(ultrastar_class.gap) / 1000
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(
|
||||
float(ultrastar_class.bpm.replace(",", "."))
|
||||
)
|
||||
start_time = (
|
||||
beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap
|
||||
)
|
||||
gap = int(float(ultrastar_class.gap.replace(",", ".")) / 1000)
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(",", ".")))
|
||||
start_time = beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap
|
||||
return start_time
|
||||
|
||||
|
||||
def get_end_time_from_ultrastar(ultrastar_class: UltrastarTxtValue, pos: int) -> float:
|
||||
"""Calculates the end time from the Ultrastar txt"""
|
||||
|
||||
gap = int(ultrastar_class.gap) / 1000
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(
|
||||
float(ultrastar_class.bpm.replace(",", "."))
|
||||
)
|
||||
gap = int(float(ultrastar_class.gap.replace(",", ".")) / 1000)
|
||||
real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(",", ".")))
|
||||
end_time = (
|
||||
beat_to_second(
|
||||
int(ultrastar_class.startBeat[pos])
|
||||
+ int(ultrastar_class.durations[pos]),
|
||||
int(ultrastar_class.startBeat[pos]) + int(ultrastar_class.durations[pos]),
|
||||
real_bpm,
|
||||
)
|
||||
+ gap
|
||||
|
||||
@@ -5,7 +5,13 @@ from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
)
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, UltrastarTxtTag, UltrastarTxtNoteTypeTag, FILE_ENCODING
|
||||
from modules.Ultrastar.ultrastar_txt import (
|
||||
UltrastarTxtValue,
|
||||
UltrastarTxtTag,
|
||||
UltrastarTxtNoteTypeTag,
|
||||
FILE_ENCODING,
|
||||
)
|
||||
|
||||
|
||||
def parse_ultrastar_txt(input_file: str) -> UltrastarTxtValue:
|
||||
"""Parse ultrastar txt file to UltrastarTxt class"""
|
||||
@@ -35,12 +41,23 @@ def parse_ultrastar_txt(input_file: str) -> UltrastarTxtValue:
|
||||
ultrastar_class.gap = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.BPM}"):
|
||||
ultrastar_class.bpm = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith((
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEO}"):
|
||||
ultrastar_class.video = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEOGAP}"):
|
||||
ultrastar_class.videoGap = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.COVER}"):
|
||||
ultrastar_class.cover = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.BACKGROUND}"):
|
||||
ultrastar_class.background = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(
|
||||
(
|
||||
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
|
||||
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
|
||||
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ")):
|
||||
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ",
|
||||
)
|
||||
):
|
||||
parts = line.split()
|
||||
# [0] F : * R G
|
||||
# [1] start beat
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
"""Ultrastar score calculator."""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from dataclasses_json import dataclass_json
|
||||
|
||||
import librosa
|
||||
|
||||
@@ -48,6 +51,8 @@ def add_point(note_type: str, points: Points) -> Points:
|
||||
return points
|
||||
|
||||
|
||||
@dataclass_json
|
||||
@dataclass
|
||||
class Score:
|
||||
"""Docstring"""
|
||||
|
||||
|
||||
@@ -19,9 +19,11 @@ class UltrastarTxtTag(str, Enum):
|
||||
GENRE = 'GENRE' # Multi-language support since v1.1.0
|
||||
YEAR = 'YEAR' # Multi-language support since v1.1.0
|
||||
COVER = 'COVER' # Path to cover. Should end with `*[CO].jpg`
|
||||
BACKGROUND = 'BACKGROUND' # Path to background. Is shown when there is no video. Should end with `*[BG].jpg`
|
||||
CREATOR = 'CREATOR' # Multi-language support since v1.1.0
|
||||
COMMENT = 'COMMENT'
|
||||
VIDEO = 'VIDEO'
|
||||
VIDEOGAP = 'VIDEOGAP'
|
||||
FILE_END = 'E'
|
||||
LINEBREAK = '-'
|
||||
|
||||
@@ -32,8 +34,6 @@ class UltrastarTxtTag(str, Enum):
|
||||
TAGS = 'TAGS' # Tags for the song. Can be used for filtering
|
||||
|
||||
# Unused 0.2.0
|
||||
BACKGROUND = 'BACKGROUND' # Path to background. Is shown when there is no video. Should end with `*[BG].jpg`
|
||||
VIDEOGAP = 'VIDEOGAP'
|
||||
EDITION = 'EDITION' # Multi-language support since v1.1.0
|
||||
START = 'START'
|
||||
END = 'END'
|
||||
@@ -84,10 +84,12 @@ class UltrastarTxtValue:
|
||||
mp3 = ""
|
||||
audio = ""
|
||||
video = None
|
||||
videoGap = None
|
||||
gap = ""
|
||||
bpm = ""
|
||||
language = None
|
||||
cover = None
|
||||
background = None
|
||||
vocals = None
|
||||
instrumental = None
|
||||
tags = None
|
||||
|
||||
@@ -99,6 +99,8 @@ def create_ultrastar_txt_from_automation(
|
||||
file.write(f"#{UltrastarTxtTag.GENRE}:{ultrastar_class.genre}\n")
|
||||
if ultrastar_class.cover is not None:
|
||||
file.write(f"#{UltrastarTxtTag.COVER}:{ultrastar_class.cover}\n")
|
||||
if ultrastar_class.background is not None:
|
||||
file.write(f"#{UltrastarTxtTag.BACKGROUND}:{ultrastar_class.background}\n")
|
||||
file.write(f"#{UltrastarTxtTag.MP3}:{ultrastar_class.mp3}\n")
|
||||
if version.parse(ultrastar_class.version) >= version.parse("1.1.0"):
|
||||
file.write(f"#{UltrastarTxtTag.AUDIO}:{ultrastar_class.audio}\n")
|
||||
@@ -108,7 +110,10 @@ def create_ultrastar_txt_from_automation(
|
||||
file.write(f"#{UltrastarTxtTag.INSTRUMENTAL}:{ultrastar_class.instrumental}\n")
|
||||
if ultrastar_class.tags is not None:
|
||||
file.write(f"#{UltrastarTxtTag.TAGS}:{ultrastar_class.tags}\n")
|
||||
file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n")
|
||||
if ultrastar_class.video is not None:
|
||||
file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n")
|
||||
if ultrastar_class.videoGap is not None:
|
||||
file.write(f"#{UltrastarTxtTag.VIDEOGAP}:{ultrastar_class.videoGap}\n")
|
||||
file.write(f"#{UltrastarTxtTag.BPM}:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
|
||||
file.write(f"#{UltrastarTxtTag.GAP}:{int(gap * 1000)}\n")
|
||||
file.write(f"#{UltrastarTxtTag.CREATOR}:{ultrastar_class.creator}\n")
|
||||
|
||||
@@ -8,6 +8,11 @@ def blue_highlighted(text: str) -> str:
|
||||
return f"{Bcolors.blue}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def green_highlighted(text: str) -> str:
|
||||
"""Returns a blue highlighted text"""
|
||||
return f"{Bcolors.dark_green}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def gold_highlighted(text: str) -> str:
|
||||
"""Returns a gold highlighted text"""
|
||||
return f"{Bcolors.gold}{text}{Bcolors.endc}"
|
||||
@@ -37,6 +42,7 @@ class Bcolors:
|
||||
"""Colors for the console"""
|
||||
|
||||
blue = "\033[94m"
|
||||
dark_green = "\033[32m"
|
||||
red = "\033[91m"
|
||||
light_blue = "\033[96m"
|
||||
cyan = "\033[36m"
|
||||
|
||||
@@ -20,7 +20,7 @@ def export_transcribed_data_to_csv(transcribed_data: list[TranscribedData], file
|
||||
data.word,
|
||||
data.start,
|
||||
data.end,
|
||||
data.conf,
|
||||
data.confidence,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user