diff --git a/README.md b/README.md index 0e9e4e6..ef8db11 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ _Not all options working now!_ --hyphenation True|False >> ((default) is True) --disable_separation True|False >> ((default) is False) --disable_karaoke True|False >> ((default) is False) + --ignore_audio True|False >> ((default) is False) --create_audio_chunks True|False >> ((default) is False) --keep_cache True|False >> ((default) is False) --plot True|False >> ((default) is False) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 0150e6e..4d7b05b 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,10 +1,12 @@ # Version: 0.0.12 -Date: 2024.07.13 +Date: 2024.07.07 - Changes: + - Reduce Memory usage by clearing cache in whisper + - Some bug fixes and improved error handling and logs - Fix hypen language download # Version: 0.0.11 -Date: 2024.06.02 +Date: 2024.07.06 - Changes: - Better linebreak calculation - Remove cache folder when finished diff --git a/pytest/modules/Audio/test_silence_processing.py b/pytest/modules/Audio/test_silence_processing.py index 04641c6..d3cd380 100644 --- a/pytest/modules/Audio/test_silence_processing.py +++ b/pytest/modules/Audio/test_silence_processing.py @@ -21,42 +21,12 @@ class SilenceProcessingTest(unittest.TestCase): silence_parts_list = [(2.0, 9.0)] transcribed_data = [ - TranscribedData({ - "conf": 0.95, - "word": "Before silence ", - "end": 2.0, - "start": 1.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "End is in silence ", - "end": 5.0, - "start": 1.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "Is in silence ", - "end": 4.0, - "start": 3.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "Start is in silence ", - "end": 10.0, - "start": 5.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "After silence ", - "end": 10.0, - "start": 9.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "Silence is inside ", - "end": 12.0, - "start": 1.0 - }), + TranscribedData(word="Before silence ", confidence=0.95, start=1.0, end=2.0), + TranscribedData(word="End is in silence ", confidence=0.95, start=1.0, end=5.0), + TranscribedData(word="Is in silence ", confidence=0.95, start=3.0, end=4.0), + TranscribedData(word="Start is in silence ", confidence=0.95, start=5.0, end=10.0), + TranscribedData(word="After silence ", confidence=0.95, start=9.0, end=10.0), + TranscribedData(word="Silence is inside ", confidence=0.95, start=1.0, end=12.0), ] # Act @@ -106,12 +76,7 @@ class SilenceProcessingTest(unittest.TestCase): silence_parts_list = [(2.0, 3.0), (4.0, 5.0), (6.0, 7.0), (8.0, 9.0)] transcribed_data = [ - TranscribedData({ - "conf": 0.95, - "word": "Silence is inside ", - "end": 12.0, - "start": 1.0 - }), + TranscribedData(word="Silence is inside ", confidence=0.95, start=1.0, end=12.0), ] # Act @@ -154,12 +119,7 @@ class SilenceProcessingTest(unittest.TestCase): silence_parts_list = [(2.0, 3.0), (4.0, 5.0), (6.0, 7.0), (8.0, 9.0)] transcribed_data = [ - TranscribedData({ - "conf": 0.95, - "word": "Silence is inside ", - "end": 9.0, - "start": 1.0 - }), + TranscribedData(word="Silence is inside ", confidence=0.95, start=1.0, end=9.0), ] # Act @@ -196,24 +156,9 @@ class SilenceProcessingTest(unittest.TestCase): silence_parts_list = [(2.0, 2.1), (3.09, 3.1), (4.0, 4.09), (5.0, 5.09), (6.0, 6.09), (7.0, 7.09)] transcribed_data = [ - TranscribedData({ - "conf": 0.95, - "word": "Remove split ", - "end": 2.19, - "start": 1.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "Is split ", - "end": 4.1, - "start": 3.0 - }), - TranscribedData({ - "conf": 0.95, - "word": "Is split2 ", - "end": 7.1, - "start": 5.0 - }), + TranscribedData(word="Remove split ", confidence=0.95, start=1.0, end=2.19), + TranscribedData(word="Is split ", confidence=0.95, start=3.0, end=4.1), + TranscribedData(word="Is split2 ", confidence=0.95, start=5.0, end=7.1), ] # Act diff --git a/pytest/modules/Speech_Recognition/test_Whisper.py b/pytest/modules/Speech_Recognition/test_Whisper.py index 8f78701..51c3580 100644 --- a/pytest/modules/Speech_Recognition/test_Whisper.py +++ b/pytest/modules/Speech_Recognition/test_Whisper.py @@ -30,14 +30,13 @@ class ConvertToTranscribedDataTest(unittest.TestCase): # Words should have space at the end expected_output = [ - TranscribedData( - {"word": "UltraSinger ", "start": 1.23, "end": 2.34, "is_hyphen": None, "confidence": 0.95}), - TranscribedData({"word": "is ", "start": 2.34, "end": 3.45, "is_hyphen": None, "confidence": 0.9}), - TranscribedData({"word": "cool! ", "start": 3.45, "end": 4.56, "is_hyphen": None, "confidence": 0.85}), - TranscribedData({"word": "And ", "start": 4.56, "end": 5.67, "is_hyphen": None, "confidence": 0.95}), - TranscribedData({"word": "will ", "start": 5.67, "end": 6.78, "is_hyphen": None, "confidence": 0.9}), - TranscribedData({"word": "be ", "start": 6.78, "end": 7.89, "is_hyphen": None, "confidence": 0.85}), - TranscribedData({"word": "better! ", "start": 7.89, "end": 9.01, "is_hyphen": None, "confidence": 0.8}), + TranscribedData(word="UltraSinger ", start=1.23, end=2.34, is_hyphen=False, confidence=0.95), + TranscribedData(word="is ", start=2.34, end=3.45, is_hyphen=False, confidence=0.9), + TranscribedData(word="cool! ", start=3.45, end=4.56, is_hyphen=False, confidence=0.85), + TranscribedData(word="And ", start=4.56, end=5.67, is_hyphen=False, confidence=0.95), + TranscribedData(word="will ", start=5.67, end=6.78, is_hyphen=False, confidence=0.9), + TranscribedData(word="be ", start=6.78, end=7.89, is_hyphen=False, confidence=0.85), + TranscribedData(word="better! ", start=7.89, end=9.01, is_hyphen=False, confidence=0.8), ] # Act diff --git a/pytest/modules/UltraStar/test_ultrastar_writer.py b/pytest/modules/UltraStar/test_ultrastar_writer.py index 514da00..5252ae4 100644 --- a/pytest/modules/UltraStar/test_ultrastar_writer.py +++ b/pytest/modules/UltraStar/test_ultrastar_writer.py @@ -68,20 +68,20 @@ class TestCreateUltrastarTxt(unittest.TestCase): def arrange(self): # Arrange transcribed_data = [ - TranscribedData({ - "conf": 0.95, + TranscribedData.from_dict({ + "confidence": 0.95, "word": "UltraSinger ", "end": 2.5, "start": 0.5 }), - TranscribedData({ - "conf": 0.9, + TranscribedData.from_dict({ + "confidence": 0.9, "word": "is ", "end": 4.5, "start": 3.0 }), - TranscribedData({ - "conf": 0.85, + TranscribedData.from_dict({ + "confidence": 0.85, "word": "cool! ", "end": 7.5, "start": 5.5 @@ -110,20 +110,27 @@ class TestCreateUltrastarTxt(unittest.TestCase): expected_calls = [] if version.parse(ver) >= version.parse("1.0.0"): expected_calls.append(f"#{UltrastarTxtTag.VERSION}:{default_ultrastar_class.version}\n") - expected_calls.append(f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n") - expected_calls.append(f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n") - expected_calls.append(f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n") + expected_calls += [ + f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n", + f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n", + f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n" + ] if version.parse(ver) >= version.parse("1.1.0"): - expected_calls.append(f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n") - expected_calls.append(f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n") # todo: video is optional - expected_calls.append(f"#{UltrastarTxtTag.BPM}:390.0\n") - expected_calls.append(f"#{UltrastarTxtTag.GAP}:500\n") - expected_calls.append(f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n") - expected_calls.append(f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n") - expected_calls.append(": 0 52 1 UltraSinger \n") - expected_calls.append(": 65 39 2 is \n") - expected_calls.append(": 130 52 3 cool! \n") - expected_calls.append("E") + expected_calls += [f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n"] + if default_ultrastar_class.video is not None: + expected_calls += [ + f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n" + ] + expected_calls += [ + f"#{UltrastarTxtTag.BPM}:390.0\n", + f"#{UltrastarTxtTag.GAP}:500\n", + f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n", + f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n", + ": 0 52 1 UltraSinger \n", + ": 65 39 2 is \n", + ": 130 52 3 cool! \n", + "E" + ] return expected_calls diff --git a/pytest/test_UltraSinger.py b/pytest/test_UltraSinger.py index ca0b576..9146127 100644 --- a/pytest/test_UltraSinger.py +++ b/pytest/test_UltraSinger.py @@ -47,7 +47,7 @@ class TestUltraSinger(unittest.TestCase): result = parse_ultrastar_txt() # Assert - self.assertEqual(result, ("test", + self.assertEqual(result, ("Test Artist - Test Title", 120.0, os.path.join("path", "to", "output", "Test Artist - Test Title"), os.path.join("path", "to", "input", "test.mp3"), diff --git a/src/Settings.py b/src/Settings.py index d0cfc30..66f124a 100644 --- a/src/Settings.py +++ b/src/Settings.py @@ -1,3 +1,10 @@ +from dataclasses import dataclass + +from dataclasses_json import dataclass_json + + +@dataclass_json +@dataclass class Settings: APP_VERSION = "0.0.12-dev2" @@ -7,6 +14,8 @@ class Settings: hyphenation = True use_separated_vocal = True create_karaoke = True + ignore_audio = False + input_file_is_ultrastar_txt = False keep_cache = False # Process data Paths @@ -43,3 +52,12 @@ class Settings: # MuseScore musescore_path = None + + # UltraSinger Evaluation Configuration + test_songs_input_folder = None + cache_override_path = None + skip_cache_vocal_separation = False + skip_cache_denoise_vocal_audio = False + skip_cache_transcription = False + skip_cache_pitch_detection = False + calculate_score = True \ No newline at end of file diff --git a/src/UltraSinger.py b/src/UltraSinger.py index 36ee0c9..e5e52e7 100644 --- a/src/UltraSinger.py +++ b/src/UltraSinger.py @@ -22,6 +22,8 @@ from modules.Audio.vocal_chunks import ( export_chunks_from_ultrastar_data, ) from modules.Audio.silence_processing import remove_silence_from_transcription_data, get_silence_sections +from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult +from modules.Ultrastar.ultrastar_score_calculator import Score from modules.csv_handler import export_transcribed_data_to_csv from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3 from modules.Audio.youtube import ( @@ -37,6 +39,7 @@ from modules.console_colors import ( gold_highlighted, light_blue_highlighted, red_highlighted, + green_highlighted, ) from modules.Midi.midi_creator import ( convert_frequencies_to_notes, @@ -52,12 +55,22 @@ from modules.Pitcher.pitcher import ( get_pitch_with_crepe_file, ) from modules.Pitcher.pitched_data import PitchedData -from modules.Speech_Recognition.hyphenation import hyphenation, language_check, create_hyphenator +from modules.Speech_Recognition.hyphenation import ( + hyphenation, + language_check, + create_hyphenator, +) from modules.Speech_Recognition.Whisper import transcribe_with_whisper -from modules.Ultrastar import ultrastar_score_calculator, ultrastar_writer, ultrastar_converter, ultrastar_parser -from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue +from modules.Ultrastar import ( + ultrastar_score_calculator, + ultrastar_writer, + ultrastar_converter, + ultrastar_parser, +) +from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, FILE_ENCODING from Settings import Settings from modules.Speech_Recognition.TranscribedData import TranscribedData +from modules.os_helper import check_file_exists from modules.plot import plot, plot_spectrogram from modules.musicbrainz_client import get_music_infos from modules.sheet import create_sheet @@ -67,9 +80,7 @@ settings = Settings() def pitch_each_chunk_with_crepe(directory: str) -> list[str]: """Pitch each chunk with crepe and return midi notes""" - print( - f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}" - ) + print(f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}") midi_notes = [] for filename in sorted( @@ -98,7 +109,9 @@ def pitch_each_chunk_with_crepe(directory: str) -> list[str]: return midi_notes -def add_hyphen_to_data(transcribed_data: list[TranscribedData], hyphen_words: list[list[str]]): +def add_hyphen_to_data( + transcribed_data: list[TranscribedData], hyphen_words: list[list[str]] +): """Add hyphen to transcribed data return new data list""" new_data = [] @@ -134,9 +147,7 @@ def get_bpm_from_data(data, sampling_rate): onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate) wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate) - print( - f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}" - ) + print(f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}") return wav_tempo[0] @@ -228,12 +239,12 @@ def remove_unecessary_punctuations(transcribed_data: list[TranscribedData]) -> N """Remove unecessary punctuations from transcribed data""" punctuation = ".," for i, data in enumerate(transcribed_data): - data.word = data.word.translate( - {ord(i): None for i in punctuation} - ) + data.word = data.word.translate({ord(i): None for i in punctuation}) -def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData]) -> list[list[str]] | None: +def hyphenate_each_word( + language: str, transcribed_data: list[TranscribedData] +) -> list[list[str]] | None: """Hyphenate each word in the transcribed data.""" lang_region = language_check(language) if lang_region is None: @@ -284,17 +295,15 @@ def print_version() -> None: f"{ULTRASINGER_HEAD} {gold_highlighted('*****************************')}" ) -def run() -> None: +def run() -> tuple[str, Score, Score]: """The processing function of this program""" - is_audio = ".txt" not in settings.input_file_path + settings.input_file_is_ultrastar_txt = settings.input_file_path.endswith(".txt") + ultrastar_class = None real_bpm = None (title, artist, year, genre) = (None, None, None, None) - if not is_audio: # Parse Ultrastar txt - print( - f"{ULTRASINGER_HEAD} {gold_highlighted('re-pitch mode')}" - ) + if settings.input_file_is_ultrastar_txt: # Parse Ultrastar txt ( basename_without_ext, real_bpm, @@ -302,28 +311,35 @@ def run() -> None: ultrastar_audio_input_path, ultrastar_class, ) = parse_ultrastar_txt() + + if not ultrastar_class.mp3: + print( + f"{ULTRASINGER_HEAD} {red_highlighted('Error!')} The provided text file does not have a reference to " + f"an audio file." + ) + exit(1) elif settings.input_file_path.startswith("https:"): # Youtube - print( - f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}" - ) + print(f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}") ( basename_without_ext, settings.song_output, ultrastar_audio_input_path, - (title, artist, year, genre) + (title, artist, year, genre), ) = download_from_youtube() else: # Audio File - print( - f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}" - ) + print(f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}") ( basename_without_ext, settings.song_output, ultrastar_audio_input_path, - (title, artist, year, genre) + (title, artist, year, genre), ) = infos_from_audio_input_file() - settings.cache_path = os.path.join(settings.song_output, "cache") + settings.cache_path = ( + os.path.join(settings.song_output, "cache") + if settings.cache_override_path is None + else settings.cache_override_path + ) settings.processing_audio_path = os.path.join( settings.cache_path, basename_without_ext + ".wav" ) @@ -376,11 +392,12 @@ def run() -> None: # Audio transcription transcribed_data = None language = settings.language - if is_audio: - detected_language, transcribed_data = transcribe_audio() + if not settings.ignore_audio: + transcription_result = transcribe_audio(settings.cache_path) if language is None: - language = detected_language + language = transcription_result.detected_language + transcribed_data = transcription_result.transcribed_data remove_unecessary_punctuations(transcribed_data) if settings.hyphenation: @@ -400,7 +417,6 @@ def run() -> None: if settings.create_audio_chunks: create_audio_chunks( settings.cache_path, - is_audio, transcribed_data, ultrastar_audio_input_path, ultrastar_class, @@ -408,7 +424,7 @@ def run() -> None: # Pitch the audio midi_segments, pitched_data, ultrastar_note_numbers, transcribed_data = pitch_audio( - is_audio, transcribed_data, ultrastar_class + transcribed_data, ultrastar_class, settings.cache_path ) # Create plot @@ -419,7 +435,7 @@ def run() -> None: plot(pitched_data, settings.song_output, midi_segments) # Write Ultrastar txt - if is_audio: + if not settings.ignore_audio: real_bpm, ultrastar_file_output = create_ultrastar_txt_from_automation( basename_without_ext, settings.song_output, @@ -437,15 +453,16 @@ def run() -> None: settings.song_output, ultrastar_class, ultrastar_note_numbers ) - # Calc Points - ultrastar_class, simple_score, accurate_score = calculate_score_points( - is_audio, pitched_data, ultrastar_class, ultrastar_file_output - ) + simple_score = None + accurate_score = None + if settings.calculate_score: + # Calc Points + ultrastar_class, simple_score, accurate_score = calculate_score_points( + pitched_data, ultrastar_class, ultrastar_file_output + ) # Add calculated score to Ultrastar txt #Todo: Missing Karaoke - ultrastar_writer.add_score_to_ultrastar_txt( - ultrastar_file_output, simple_score - ) + ultrastar_writer.add_score_to_ultrastar_txt(ultrastar_file_output, simple_score) # Midi if settings.create_midi: @@ -460,6 +477,7 @@ def run() -> None: # Print Support print_support() + return ultrastar_file_output, simple_score, accurate_score def mute_no_singing_parts(mono_output_path, mute_output_path): @@ -499,58 +517,85 @@ def get_unused_song_output_dir(path: str) -> str: print( f"{ULTRASINGER_HEAD} {red_highlighted('Error: Could not create output folder! (999) is the maximum number of tries.')}" ) - sys.exit(1) + raise ValueError("Could not create output folder! (999) is the maximum number of tries.") return path -def transcribe_audio() -> (str, list[TranscribedData]): +def transcribe_audio(cache_path: str) -> TranscriptionResult: """Transcribe audio with AI""" + transcription_result = None if settings.transcriber == "whisper": - device = "cpu" if settings.force_whisper_cpu else settings.pytorch_device - transcribed_data, detected_language = transcribe_with_whisper( - settings.processing_audio_path, - settings.whisper_model, - device, - settings.whisper_align_model, - settings.whisper_batch_size, - settings.whisper_compute_type, - settings.language, - ) + transcription_config = f"{settings.transcriber}_{settings.whisper_model}_{settings.pytorch_device}_{settings.whisper_align_model}_{settings.whisper_align_model}_{settings.whisper_batch_size}_{settings.whisper_compute_type}_{settings.language}" + transcription_path = os.path.join(cache_path, f"{transcription_config}.json") + cached_transcription_available = check_file_exists(transcription_path) + if settings.skip_cache_transcription or not cached_transcription_available: + transcription_result = transcribe_with_whisper( + settings.processing_audio_path, + settings.whisper_model, + settings.pytorch_device, + settings.whisper_align_model, + settings.whisper_batch_size, + settings.whisper_compute_type, + settings.language, + ) + with open(transcription_path, "w", encoding=FILE_ENCODING) as file: + file.write(transcription_result.to_json()) + else: + print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached transcribed data") + with open(transcription_path) as file: + json = file.read() + transcription_result = TranscriptionResult.from_json(json) else: raise NotImplementedError - return detected_language, transcribed_data + return transcription_result def separate_vocal_from_audio( - basename_without_ext: str, cache_path: str, ultrastar_audio_input_path: str + basename_without_ext: str, cache_path: str, ultrastar_audio_input_path: str ) -> str: """Separate vocal from audio""" + demucs_output_folder = os.path.splitext( + os.path.basename(ultrastar_audio_input_path) + )[0] audio_separation_path = os.path.join( - cache_path, "separated", "htdemucs", basename_without_ext + cache_path, "separated", "htdemucs", demucs_output_folder ) + vocals_path = os.path.join(audio_separation_path, "vocals.wav") + instrumental_path = os.path.join(audio_separation_path, "no_vocals.wav") if settings.use_separated_vocal or settings.create_karaoke: - separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device) + cache_available = check_file_exists(vocals_path) and check_file_exists( + instrumental_path + ) + if settings.skip_cache_vocal_separation or not cache_available: + separate_audio( + ultrastar_audio_input_path, cache_path, settings.pytorch_device + ) + else: + print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached separated vocals") + + if settings.use_separated_vocal: + input_path = vocals_path + else: + input_path = ultrastar_audio_input_path + + convert_audio_to_mono_wav(input_path, settings.processing_audio_path) return audio_separation_path def calculate_score_points( - is_audio: bool, pitched_data: PitchedData, ultrastar_class: UltrastarTxtValue, ultrastar_file_output: str + pitched_data: PitchedData, + ultrastar_class: UltrastarTxtValue, + ultrastar_file_output: str, ): """Calculate score points""" - if is_audio: - ultrastar_class = ultrastar_parser.parse_ultrastar_txt( - ultrastar_file_output - ) + if not settings.ignore_audio: + ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output) ( simple_score, accurate_score, - ) = ultrastar_score_calculator.calculate_score( - pitched_data, ultrastar_class - ) - ultrastar_score_calculator.print_score_calculation( - simple_score, accurate_score - ) + ) = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class) + ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score) else: print( f"{ULTRASINGER_HEAD} {blue_highlighted('Score of original Ultrastar txt')}" @@ -558,32 +603,24 @@ def calculate_score_points( ( simple_score, accurate_score, - ) = ultrastar_score_calculator.calculate_score( - pitched_data, ultrastar_class - ) - ultrastar_score_calculator.print_score_calculation( - simple_score, accurate_score - ) + ) = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class) + ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score) print( f"{ULTRASINGER_HEAD} {blue_highlighted('Score of re-pitched Ultrastar txt')}" ) - ultrastar_class = ultrastar_parser.parse_ultrastar_txt( - ultrastar_file_output - ) + ultrastar_class = ultrastar_parser.parse_ultrastar_txt(ultrastar_file_output) ( simple_score, accurate_score, - ) = ultrastar_score_calculator.calculate_score( - pitched_data, ultrastar_class - ) - ultrastar_score_calculator.print_score_calculation( - simple_score, accurate_score - ) + ) = ultrastar_score_calculator.calculate_score(pitched_data, ultrastar_class) + ultrastar_score_calculator.print_score_calculation(simple_score, accurate_score) return ultrastar_class, simple_score, accurate_score def create_ultrastar_txt_from_ultrastar_data( - song_output: str, ultrastar_class: UltrastarTxtValue, ultrastar_note_numbers: list[int] + song_output: str, + ultrastar_class: UltrastarTxtValue, + ultrastar_note_numbers: list[int], ) -> str: """Create Ultrastar txt from Ultrastar data""" output_repitched_ultrastar = os.path.join( @@ -622,9 +659,7 @@ def create_ultrastar_txt_from_automation( ultrastar_header.language = language cover = basename_without_ext + " [CO].jpg" ultrastar_header.cover = ( - cover - if os_helper.check_file_exists(os.path.join(song_output, cover)) - else None + cover if os_helper.check_file_exists(os.path.join(song_output, cover)) else None ) ultrastar_header.creator = f"{ultrastar_header.creator} {Settings.APP_VERSION}" ultrastar_header.comment = f"{ultrastar_header.comment} {Settings.APP_VERSION}" @@ -640,9 +675,7 @@ def create_ultrastar_txt_from_automation( ultrastar_header.genre = format_separated_string(genre) real_bpm = get_bpm_from_file(ultrastar_audio_input_path) - ultrastar_file_output = os.path.join( - song_output, basename_without_ext + ".txt" - ) + ultrastar_file_output = os.path.join(song_output, basename_without_ext + ".txt") ultrastar_writer.create_ultrastar_txt_from_automation( transcribed_data, ultrastar_note_numbers, @@ -721,9 +754,17 @@ def infos_from_audio_input_file() -> tuple[str, str, str, tuple[str, str, str, s song_output = get_unused_song_output_dir(song_output) os_helper.create_folder(song_output) os_helper.copy(settings.input_file_path, song_output) - os_helper.rename(os.path.join(song_output, os.path.basename(settings.input_file_path)), os.path.join(song_output, basename)) + os_helper.rename( + os.path.join(song_output, os.path.basename(settings.input_file_path)), + os.path.join(song_output, basename), + ) ultrastar_audio_input_path = os.path.join(song_output, basename) - return basename_without_ext, song_output, ultrastar_audio_input_path, (title, artist, year_info, genre_info) + return ( + basename_without_ext, + song_output, + ultrastar_audio_input_path, + (title, artist, year_info, genre_info), + ) FILENAME_REPLACEMENTS = (('?:"', ""), ("<", "("), (">", ")"), ("/\\|*", "-")) @@ -744,7 +785,9 @@ def download_from_youtube() -> tuple[str, str, str, tuple[str, str, str, str]]: (artist, title) = get_youtube_title(settings.input_file_path) # Get additional data for song - (title_info, artist_info, year_info, genre_info) = get_music_infos(f"{artist} - {title}") + (title_info, artist_info, year_info, genre_info) = get_music_infos( + f"{artist} - {title}" + ) if title_info is not None: title = title_info @@ -755,29 +798,29 @@ def download_from_youtube() -> tuple[str, str, str, tuple[str, str, str, str]]: song_output = os.path.join(settings.output_file_path, basename_without_ext) song_output = get_unused_song_output_dir(song_output) os_helper.create_folder(song_output) - download_youtube_audio( - settings.input_file_path, basename_without_ext, song_output - ) - download_youtube_video( - settings.input_file_path, basename_without_ext, song_output - ) + download_youtube_audio(settings.input_file_path, basename_without_ext, song_output) + download_youtube_video(settings.input_file_path, basename_without_ext, song_output) download_youtube_thumbnail( settings.input_file_path, basename_without_ext, song_output ) ultrastar_audio_input_path = os.path.join(song_output, basename) - return basename_without_ext, song_output, ultrastar_audio_input_path, (title, artist, year_info, genre_info) + return ( + basename_without_ext, + song_output, + ultrastar_audio_input_path, + (title, artist, year_info, genre_info), + ) def parse_ultrastar_txt() -> tuple[str, float, str, str, UltrastarTxtValue]: """Parse Ultrastar txt""" - ultrastar_class = ultrastar_parser.parse_ultrastar_txt( - settings.input_file_path - ) + ultrastar_class = ultrastar_parser.parse_ultrastar_txt(settings.input_file_path) real_bpm = ultrastar_converter.ultrastar_bpm_to_real_bpm( float(ultrastar_class.bpm.replace(",", ".")) ) ultrastar_mp3_name = ultrastar_class.mp3 - basename_without_ext = os.path.splitext(ultrastar_mp3_name)[0] + + basename_without_ext = f"{ultrastar_class.artist.strip()} - {ultrastar_class.title.strip()}" dirname = os.path.dirname(settings.input_file_path) ultrastar_audio_input_path = os.path.join(dirname, ultrastar_mp3_name) song_output = os.path.join( @@ -796,37 +839,53 @@ def parse_ultrastar_txt() -> tuple[str, float, str, str, UltrastarTxtValue]: ) -def create_midi_file(real_bpm: float, - song_output: str, - ultrastar_class: UltrastarTxtValue, - basename_without_ext: str) -> None: +def create_midi_file( + real_bpm: float, + song_output: str, + ultrastar_class: UltrastarTxtValue, + basename_without_ext: str, +) -> None: """Create midi file""" - print( - f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}" - ) + print(f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}") voice_instrument = [ convert_ultrastar_to_midi_instrument(ultrastar_class) ] midi_output = os.path.join(song_output, f"{basename_without_ext}.mid") - instruments_to_midi( - voice_instrument, real_bpm, midi_output - ) + instruments_to_midi(voice_instrument, real_bpm, midi_output) -def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrastar_class: UltrastarTxtValue) -> tuple[ - list[MidiSegment], PitchedData, list[int], list[TranscribedData]]: +def pitch_audio( + transcribed_data: list[TranscribedData], + ultrastar_class: UltrastarTxtValue, + cache_path: str +) -> tuple[list[MidiSegment], PitchedData, list[int], list[TranscribedData]]: """Pitch audio""" # todo: chunk pitching as option? # midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name) - device = "cpu" if settings.force_crepe_cpu else settings.tensorflow_device - pitched_data = get_pitch_with_crepe_file( - settings.processing_audio_path, - settings.crepe_model_capacity, - settings.crepe_step_size, - device, - ) - if is_audio: + + pitching_config = f"crepe_{settings.ignore_audio}_{settings.crepe_model_capacity}_{settings.crepe_step_size}_{settings.tensorflow_device}" + pitched_data_path = os.path.join(cache_path, f"{pitching_config}.json") + cache_available = check_file_exists(pitched_data_path) + + if settings.skip_cache_transcription or not cache_available: + pitched_data = get_pitch_with_crepe_file( + settings.processing_audio_path, + settings.crepe_model_capacity, + settings.crepe_step_size, + settings.tensorflow_device, + ) + + pitched_data_json = pitched_data.to_json() + with open(pitched_data_path, "w", encoding=FILE_ENCODING) as file: + file.write(pitched_data_json) + else: + print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached pitch data") + with open(pitched_data_path) as file: + json = file.read() + pitched_data = PitchedData.from_json(json) + + if not settings.ignore_audio: start_times = [] end_times = [] words = [] @@ -834,34 +893,37 @@ def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrast start_times.append(midi_segment.start) end_times.append(midi_segment.end) words.append(midi_segment.word) - midi_segments = create_midi_notes_from_pitched_data(start_times, end_times, words, pitched_data) + midi_segments = create_midi_notes_from_pitched_data( + start_times, end_times, words, pitched_data + ) else: midi_segments = create_midi_notes_from_pitched_data( ultrastar_class.startTimes, ultrastar_class.endTimes, ultrastar_class.words, pitched_data ) + + # fixme: should be in UltraStar ultrastar_note_numbers = convert_midi_notes_to_ultrastar_notes(midi_segments) + # fixme: TrnscribedData in pitch_audio?? new_transcribed_data = [] for i, midi_segment in enumerate(midi_segments): - new_transcribed_data.append(TranscribedData({"word": midi_segment.word, "start": midi_segment.start, "end": midi_segment.end, "is_hyphen": None, "confidence": 1})) + transcribed_midi_data = TranscribedData(word= midi_segment.word, start= midi_segment.start, end= midi_segment.end, is_hyphen= None, confidence= 1) + new_transcribed_data.append(transcribed_midi_data) return midi_segments, pitched_data, ultrastar_note_numbers, new_transcribed_data def create_audio_chunks( cache_path: str, - is_audio: bool, transcribed_data: list[TranscribedData], ultrastar_audio_input_path: str, ultrastar_class: UltrastarTxtValue ) -> None: """Create audio chunks""" - audio_chunks_path = os.path.join( - cache_path, settings.audio_chunk_folder_name - ) + audio_chunks_path = os.path.join(cache_path, settings.audio_chunk_folder_name) os_helper.create_folder(audio_chunks_path) - if is_audio: # and csv + if not settings.ignore_audio: # and csv csv_filename = os.path.join(audio_chunks_path, "_chunks.csv") export_chunks_from_transcribed_data( settings.processing_audio_path, transcribed_data, audio_chunks_path @@ -874,7 +936,11 @@ def create_audio_chunks( def denoise_vocal_audio(input_path: str, output_path: str) -> None: """Denoise vocal audio""" - ffmpeg_reduce_noise(input_path, output_path) + cache_available = check_file_exists(output_path) + if settings.skip_cache_denoise_vocal_audio or not cache_available: + ffmpeg_reduce_noise(input_path, output_path) + else: + print(f"{ULTRASINGER_HEAD} {green_highlighted('cache')} reusing cached denoised audio") def main(argv: list[str]) -> None: @@ -930,6 +996,8 @@ def init_settings(argv: list[str]) -> None: settings.create_karaoke = not arg elif opt in ("--create_audio_chunks"): settings.create_audio_chunks = arg + elif opt in ("--ignore_audio"): + settings.ignore_audio = arg in ["True", "true"] elif opt in ("--force_cpu"): settings.force_cpu = arg if settings.force_cpu: @@ -978,6 +1046,7 @@ def arg_options(): "disable_separation=", "disable_karaoke=", "create_audio_chunks=", + "ignore_audio=", "force_cpu=", "force_whisper_cpu=", "force_crepe_cpu=", diff --git a/src/modules/Audio/separation.py b/src/modules/Audio/separation.py index e6d0485..4f89b44 100644 --- a/src/modules/Audio/separation.py +++ b/src/modules/Audio/separation.py @@ -1,7 +1,10 @@ """Separate vocals from audio""" - +import os +import shlex import subprocess +import demucs.separate + from modules.console_colors import ( ULTRASINGER_HEAD, blue_highlighted, @@ -10,16 +13,11 @@ from modules.console_colors import ( from modules.os_helper import current_executor_path, move, path_join -def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None: +def separate_audio(input_file_path: str, output_folder: str, device="cpu") -> None: """Separate vocals from audio with demucs.""" print( f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker." ) - # Model selection? - # -n htdemucs_ft - subprocess.run( - ["demucs", "-d", device, "--two-stems=vocals", "--float32", input_file_path] - ) - separated_folder = path_join(current_executor_path(), "separated") - move(separated_folder, output_file) \ No newline at end of file + + demucs.separate.main(shlex.split(f'--two-stems vocals -d {device} --float32 --out "{os.path.join(output_folder, "separated")}" "{input_file_path}"')) \ No newline at end of file diff --git a/src/modules/Audio/silence_processing.py b/src/modules/Audio/silence_processing.py index da11172..6f92dee 100644 --- a/src/modules/Audio/silence_processing.py +++ b/src/modules/Audio/silence_processing.py @@ -63,7 +63,7 @@ def remove_silence(silence_parts_list: list[tuple[float, float]], transcribed_da split_word = "~ " is_word_end = True - split_data = TranscribedData({"conf": data.conf, "word": split_word, "end": split_end, "start": silence_end, "is_word_end": is_word_end}) + split_data = TranscribedData(confidence=data.confidence, word=split_word, end=split_end, start=silence_end, is_word_end=is_word_end) if not was_split: data.end = silence_start diff --git a/src/modules/Pitcher/pitched_data.py b/src/modules/Pitcher/pitched_data.py index 13d828c..e60dfd3 100644 --- a/src/modules/Pitcher/pitched_data.py +++ b/src/modules/Pitcher/pitched_data.py @@ -1,7 +1,10 @@ """Pitched data""" from dataclasses import dataclass +from dataclasses_json import dataclass_json + +@dataclass_json @dataclass class PitchedData: """Pitched data from crepe""" diff --git a/src/modules/Pitcher/pitcher.py b/src/modules/Pitcher/pitcher.py index 4312c07..537e21d 100644 --- a/src/modules/Pitcher/pitcher.py +++ b/src/modules/Pitcher/pitcher.py @@ -30,6 +30,10 @@ def get_pitch_with_crepe( times, frequencies, confidence, activation = crepe.predict( audio, sample_rate, model_capacity, step_size=step_size, viterbi=True ) + + # convert to native float for serialization + confidence = [float(x) for x in confidence] + return PitchedData(times, frequencies, confidence) diff --git a/src/modules/Speech_Recognition/TranscribedData.py b/src/modules/Speech_Recognition/TranscribedData.py index 6ff4ee3..465ef75 100644 --- a/src/modules/Speech_Recognition/TranscribedData.py +++ b/src/modules/Speech_Recognition/TranscribedData.py @@ -1,21 +1,29 @@ -"""Docstring""" +from dataclasses import dataclass + +from dataclasses_json import dataclass_json +@dataclass_json +@dataclass class TranscribedData: """Transcribed data from json file""" - def __init__(self, transcribed_json = None): - if transcribed_json is None: - return - # Vosk = conf, Whisper = confidence - self.conf = transcribed_json.get( - "conf", transcribed_json.get("confidence", None) - ) - # Vosk = word, Whisper = text - self.word = transcribed_json.get( - "word", transcribed_json.get("text", None) - ) - self.end = transcribed_json.get("end", None) - self.start = transcribed_json.get("start", None) - self.is_hyphen = transcribed_json.get("is_hyphen", None) - self.is_word_end = transcribed_json.get("is_word_end", True) + confidence: float = 0 + word: str = "" + start: float = 0 + end: float = 0 + is_hyphen: bool = False + is_word_end: bool = True + + +def from_whisper(whisper_dict) -> TranscribedData: + transcribed_data = TranscribedData() + if "score" in whisper_dict: + transcribed_data.confidence = whisper_dict["score"] + if "word" in whisper_dict: + transcribed_data.word = whisper_dict["word"] + if "start" in whisper_dict: + transcribed_data.start = whisper_dict["start"] + if "end" in whisper_dict: + transcribed_data.end = whisper_dict["end"] + return transcribed_data diff --git a/src/modules/Speech_Recognition/TranscriptionResult.py b/src/modules/Speech_Recognition/TranscriptionResult.py new file mode 100644 index 0000000..1fa055f --- /dev/null +++ b/src/modules/Speech_Recognition/TranscriptionResult.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + +from dataclasses_json import dataclass_json + +from modules.Speech_Recognition.TranscribedData import TranscribedData + + +@dataclass_json +@dataclass +class TranscriptionResult: + """Transcription result""" + + transcribed_data: list[TranscribedData] + detected_language: str diff --git a/src/modules/Speech_Recognition/Whisper.py b/src/modules/Speech_Recognition/Whisper.py index abcbb96..accba89 100644 --- a/src/modules/Speech_Recognition/Whisper.py +++ b/src/modules/Speech_Recognition/Whisper.py @@ -2,11 +2,16 @@ import sys +import torch import whisperx from torch.cuda import OutOfMemoryError +from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted -from modules.Speech_Recognition.TranscribedData import TranscribedData +from modules.Speech_Recognition.TranscribedData import TranscribedData, from_whisper + + +MEMORY_ERROR_MESSAGE = f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu" def transcribe_with_whisper( @@ -17,7 +22,7 @@ def transcribe_with_whisper( batch_size: int = 16, compute_type: str = None, language: str = None, -) -> (list[TranscribedData], str): +) -> TranscriptionResult: """Transcribe with whisper""" # Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work. @@ -32,9 +37,50 @@ def transcribe_with_whisper( compute_type = "float16" if device == "cuda" else "int8" try: + torch.cuda.empty_cache() loaded_whisper_model = whisperx.load_model( model, language=language, device=device, compute_type=compute_type ) + + audio = whisperx.load_audio(audio_path) + + print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}") + + result = loaded_whisper_model.transcribe( + audio, batch_size=batch_size, language=language + ) + + detected_language = result["language"] + if language is None: + language = detected_language + + # load alignment model and metadata + try: + model_a, metadata = whisperx.load_align_model( + language_code=language, device=device, model_name=model_name + ) + except ValueError as ve: + print( + f"{red_highlighted(f'{ve}')}" + f"\n" + f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. " + f"Try add it with --align_model [huggingface]." + ) + raise ve + + # align whisper output + result_aligned = whisperx.align( + result["segments"], + model_a, + metadata, + audio, + device, + return_char_alignments=False, + ) + + transcribed_data = convert_to_transcribed_data(result_aligned) + + return TranscriptionResult(transcribed_data, detected_language) except ValueError as value_error: if ( "Requested float16 compute type, but the target device or backend do not support efficient float16 computation." @@ -44,68 +90,27 @@ def transcribe_with_whisper( print( f"{ULTRASINGER_HEAD} Your GPU does not support efficient float16 computation; run UltraSinger with '--whisper_compute_type int8'" ) - sys.exit(1) raise value_error except OutOfMemoryError as oom_exception: print(oom_exception) - print( - f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu" - ) - sys.exit(1) - - audio = whisperx.load_audio(audio_path) - - print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}") - - result = loaded_whisper_model.transcribe( - audio, batch_size=batch_size, language=language - ) - - detected_language = result["language"] - if language is None: - language = detected_language - - # load alignment model and metadata - try: - model_a, metadata = whisperx.load_align_model( - language_code=language, device=device, model_name=model_name - ) - except ValueError as ve: - print( - f"{red_highlighted(f'{ve}')}" - f"\n" - f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. " - f"Try add it with --align_model [huggingface]." - ) - sys.exit(1) - - # align whisper output - result_aligned = whisperx.align( - result["segments"], - model_a, - metadata, - audio, - device, - return_char_alignments=False, - ) - - transcribed_data = convert_to_transcribed_data(result_aligned) - - return transcribed_data, detected_language + print(MEMORY_ERROR_MESSAGE) + raise oom_exception + except Exception as exception: + if "CUDA failed with error out of memory" in str(exception.args[0]): + print(exception) + print(MEMORY_ERROR_MESSAGE) + raise exception def convert_to_transcribed_data(result_aligned): transcribed_data = [] for segment in result_aligned["segments"]: for obj in segment["words"]: - vtd = TranscribedData(obj) # create custom Word object + vtd = from_whisper(obj) # create custom Word object vtd.word = vtd.word + " " # add space to end of word if len(obj) < 4: - previous = transcribed_data[-1] - if not previous: - previous.end = 0 - previous.end = "" + previous = transcribed_data[-1] if len(transcribed_data) != 0 else TranscribedData() vtd.start = previous.end + 0.1 vtd.end = previous.end + 0.2 msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \ diff --git a/src/modules/Ultrastar/ultrastar_converter.py b/src/modules/Ultrastar/ultrastar_converter.py index d9978eb..54a042e 100644 --- a/src/modules/Ultrastar/ultrastar_converter.py +++ b/src/modules/Ultrastar/ultrastar_converter.py @@ -48,30 +48,25 @@ def ultrastar_note_to_midi_note(ultrastar_note: int) -> int: return midi_note -def get_start_time_from_ultrastar(ultrastar_class: UltrastarTxtValue, pos: int) -> float: +def get_start_time_from_ultrastar( + ultrastar_class: UltrastarTxtValue, pos: int +) -> float: """Calculates the start time from the Ultrastar txt""" - gap = int(ultrastar_class.gap) / 1000 - real_bpm = ultrastar_bpm_to_real_bpm( - float(ultrastar_class.bpm.replace(",", ".")) - ) - start_time = ( - beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap - ) + gap = int(float(ultrastar_class.gap.replace(",", ".")) / 1000) + real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(",", "."))) + start_time = beat_to_second(int(ultrastar_class.startBeat[pos]), real_bpm) + gap return start_time def get_end_time_from_ultrastar(ultrastar_class: UltrastarTxtValue, pos: int) -> float: """Calculates the end time from the Ultrastar txt""" - gap = int(ultrastar_class.gap) / 1000 - real_bpm = ultrastar_bpm_to_real_bpm( - float(ultrastar_class.bpm.replace(",", ".")) - ) + gap = int(float(ultrastar_class.gap.replace(",", ".")) / 1000) + real_bpm = ultrastar_bpm_to_real_bpm(float(ultrastar_class.bpm.replace(",", "."))) end_time = ( beat_to_second( - int(ultrastar_class.startBeat[pos]) - + int(ultrastar_class.durations[pos]), + int(ultrastar_class.startBeat[pos]) + int(ultrastar_class.durations[pos]), real_bpm, ) + gap diff --git a/src/modules/Ultrastar/ultrastar_parser.py b/src/modules/Ultrastar/ultrastar_parser.py index f3e6100..cb36ac5 100644 --- a/src/modules/Ultrastar/ultrastar_parser.py +++ b/src/modules/Ultrastar/ultrastar_parser.py @@ -5,7 +5,13 @@ from modules.Ultrastar.ultrastar_converter import ( get_end_time_from_ultrastar, get_start_time_from_ultrastar, ) -from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, UltrastarTxtTag, UltrastarTxtNoteTypeTag, FILE_ENCODING +from modules.Ultrastar.ultrastar_txt import ( + UltrastarTxtValue, + UltrastarTxtTag, + UltrastarTxtNoteTypeTag, + FILE_ENCODING, +) + def parse_ultrastar_txt(input_file: str) -> UltrastarTxtValue: """Parse ultrastar txt file to UltrastarTxt class""" @@ -35,12 +41,23 @@ def parse_ultrastar_txt(input_file: str) -> UltrastarTxtValue: ultrastar_class.gap = line.split(":")[1].replace("\n", "") elif line.startswith(f"#{UltrastarTxtTag.BPM}"): ultrastar_class.bpm = line.split(":")[1].replace("\n", "") - elif line.startswith(( + elif line.startswith(f"#{UltrastarTxtTag.VIDEO}"): + ultrastar_class.video = line.split(":")[1].replace("\n", "") + elif line.startswith(f"#{UltrastarTxtTag.VIDEOGAP}"): + ultrastar_class.videoGap = line.split(":")[1].replace("\n", "") + elif line.startswith(f"#{UltrastarTxtTag.COVER}"): + ultrastar_class.cover = line.split(":")[1].replace("\n", "") + elif line.startswith(f"#{UltrastarTxtTag.BACKGROUND}"): + ultrastar_class.background = line.split(":")[1].replace("\n", "") + elif line.startswith( + ( f"{UltrastarTxtNoteTypeTag.FREESTYLE} ", f"{UltrastarTxtNoteTypeTag.NORMAL} ", f"{UltrastarTxtNoteTypeTag.GOLDEN} ", f"{UltrastarTxtNoteTypeTag.RAP} ", - f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ")): + f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ", + ) + ): parts = line.split() # [0] F : * R G # [1] start beat diff --git a/src/modules/Ultrastar/ultrastar_score_calculator.py b/src/modules/Ultrastar/ultrastar_score_calculator.py index e0deaf5..5e3d653 100644 --- a/src/modules/Ultrastar/ultrastar_score_calculator.py +++ b/src/modules/Ultrastar/ultrastar_score_calculator.py @@ -1,4 +1,7 @@ """Ultrastar score calculator.""" +from dataclasses import dataclass + +from dataclasses_json import dataclass_json import librosa @@ -48,6 +51,8 @@ def add_point(note_type: str, points: Points) -> Points: return points +@dataclass_json +@dataclass class Score: """Docstring""" @@ -121,8 +126,10 @@ def calculate_score(pitched_data: PitchedData, ultrastar_class: UltrastarTxtValu for part in range(parts): start = start_time + step_size * part end = start + step_size + if end_time < end or part == parts - 1: end = end_time + midi_segment = create_midi_note_from_pitched_data( start, end, pitched_data, word ) diff --git a/src/modules/Ultrastar/ultrastar_txt.py b/src/modules/Ultrastar/ultrastar_txt.py index 14f7cda..f477769 100644 --- a/src/modules/Ultrastar/ultrastar_txt.py +++ b/src/modules/Ultrastar/ultrastar_txt.py @@ -19,9 +19,11 @@ class UltrastarTxtTag(str, Enum): GENRE = 'GENRE' # Multi-language support since v1.1.0 YEAR = 'YEAR' # Multi-language support since v1.1.0 COVER = 'COVER' # Path to cover. Should end with `*[CO].jpg` + BACKGROUND = 'BACKGROUND' # Path to background. Is shown when there is no video. Should end with `*[BG].jpg` CREATOR = 'CREATOR' # Multi-language support since v1.1.0 COMMENT = 'COMMENT' VIDEO = 'VIDEO' + VIDEOGAP = 'VIDEOGAP' FILE_END = 'E' LINEBREAK = '-' @@ -32,8 +34,6 @@ class UltrastarTxtTag(str, Enum): TAGS = 'TAGS' # Tags for the song. Can be used for filtering # Unused 0.2.0 - BACKGROUND = 'BACKGROUND' # Path to background. Is shown when there is no video. Should end with `*[BG].jpg` - VIDEOGAP = 'VIDEOGAP' EDITION = 'EDITION' # Multi-language support since v1.1.0 START = 'START' END = 'END' @@ -84,10 +84,12 @@ class UltrastarTxtValue: mp3 = "" audio = "" video = None + videoGap = None gap = "" bpm = "" language = None cover = None + background = None vocals = None instrumental = None tags = None diff --git a/src/modules/Ultrastar/ultrastar_writer.py b/src/modules/Ultrastar/ultrastar_writer.py index e04e5a9..a6edb4b 100644 --- a/src/modules/Ultrastar/ultrastar_writer.py +++ b/src/modules/Ultrastar/ultrastar_writer.py @@ -99,6 +99,8 @@ def create_ultrastar_txt_from_automation( file.write(f"#{UltrastarTxtTag.GENRE}:{ultrastar_class.genre}\n") if ultrastar_class.cover is not None: file.write(f"#{UltrastarTxtTag.COVER}:{ultrastar_class.cover}\n") + if ultrastar_class.background is not None: + file.write(f"#{UltrastarTxtTag.BACKGROUND}:{ultrastar_class.background}\n") file.write(f"#{UltrastarTxtTag.MP3}:{ultrastar_class.mp3}\n") if version.parse(ultrastar_class.version) >= version.parse("1.1.0"): file.write(f"#{UltrastarTxtTag.AUDIO}:{ultrastar_class.audio}\n") @@ -108,7 +110,10 @@ def create_ultrastar_txt_from_automation( file.write(f"#{UltrastarTxtTag.INSTRUMENTAL}:{ultrastar_class.instrumental}\n") if ultrastar_class.tags is not None: file.write(f"#{UltrastarTxtTag.TAGS}:{ultrastar_class.tags}\n") - file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n") + if ultrastar_class.video is not None: + file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n") + if ultrastar_class.videoGap is not None: + file.write(f"#{UltrastarTxtTag.VIDEOGAP}:{ultrastar_class.videoGap}\n") file.write(f"#{UltrastarTxtTag.BPM}:{round(ultrastar_bpm, 2)}\n") # not the real BPM! file.write(f"#{UltrastarTxtTag.GAP}:{int(gap * 1000)}\n") file.write(f"#{UltrastarTxtTag.CREATOR}:{ultrastar_class.creator}\n") diff --git a/src/modules/console_colors.py b/src/modules/console_colors.py index c4cc6f8..59328ff 100644 --- a/src/modules/console_colors.py +++ b/src/modules/console_colors.py @@ -8,6 +8,11 @@ def blue_highlighted(text: str) -> str: return f"{Bcolors.blue}{text}{Bcolors.endc}" +def green_highlighted(text: str) -> str: + """Returns a blue highlighted text""" + return f"{Bcolors.dark_green}{text}{Bcolors.endc}" + + def gold_highlighted(text: str) -> str: """Returns a gold highlighted text""" return f"{Bcolors.gold}{text}{Bcolors.endc}" @@ -37,6 +42,7 @@ class Bcolors: """Colors for the console""" blue = "\033[94m" + dark_green = "\033[32m" red = "\033[91m" light_blue = "\033[96m" cyan = "\033[36m" diff --git a/src/modules/csv_handler.py b/src/modules/csv_handler.py index 58cceeb..3bb01f4 100644 --- a/src/modules/csv_handler.py +++ b/src/modules/csv_handler.py @@ -20,7 +20,7 @@ def export_transcribed_data_to_csv(transcribed_data: list[TranscribedData], file data.word, data.start, data.end, - data.conf, + data.confidence, ] )