diff --git a/README.md b/README.md index 603f665..38881f6 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,7 @@ _Not all options working now!_ --create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default. --keep_cache Keep cache folder after creation. Cache folder is removed by default. --plot Enable creation of plots. Plots are disabled by default. + --quantize_to_key Quantize notes to detected musical key. Removes pitch slides and out-of-key notes. >> ((default) is enabled) --format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0) --musescore_path path to MuseScore executable --keep_numbers Transcribe numbers as digits and not words @@ -236,6 +237,15 @@ you have the option to disable this function; in which case the original audio f -i XYZ --disable_separation ``` +### 🎵 Key Quantization + +By default, UltraSinger quantizes all detected notes to the detected musical key of the song. This helps to: +* Remove pitch slides and vocal transitions between notes +* Correct out-of-key notes from pitch detection errors + +If you want to keep the raw pitch detection without quantization, set `quantize_to_key = False`. +For most songs quantization should produces better results. + ### Sheet Music For Sheet Music generation you need to have `MuseScore` installed on your system. diff --git a/ReleaseNotes.md b/ReleaseNotes.md index e5a6c14..16530c8 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,5 +1,5 @@ -# Version: 0.0.13dev12 -Date: 2026.02.08 +# Version: 0.0.13dev13 +Date: 2026.02.10 - Changes: - Download Cover from MusicBrainz - Fix selection of whisper model @@ -13,6 +13,7 @@ Date: 2026.02.08 - Download yt video/audio only once - Support for video as input - Optimise scale detection + - Added quantization by key # Version: 0.0.12 Date: 2024.12.19 diff --git a/src/Settings.py b/src/Settings.py index 625f59b..66623e1 100644 --- a/src/Settings.py +++ b/src/Settings.py @@ -11,7 +11,7 @@ from modules.Ultrastar.ultrastar_txt import FormatVersion @dataclass class Settings: - APP_VERSION = "0.0.13-dev12" + APP_VERSION = "0.0.13-dev13" CONFIDENCE_THRESHOLD = 0.6 CONFIDENCE_PROMPT_TIMEOUT = 4 @@ -26,6 +26,7 @@ class Settings: keep_cache = False interactive_mode = False user_ffmpeg_path = "" + quantize_to_key = True # Quantize notes to the detected key of the song # Process data Paths input_file_path = "" diff --git a/src/UltraSinger.py b/src/UltraSinger.py index cff3275..7c3a413 100644 --- a/src/UltraSinger.py +++ b/src/UltraSinger.py @@ -17,6 +17,7 @@ from modules.Audio.vocal_chunks import ( create_audio_chunks_from_transcribed_data, create_audio_chunks_from_ultrastar_data, ) +from modules.Audio.key_detector import detect_key_from_audio, get_allowed_notes_for_key from modules.Audio.silence_processing import remove_silence_from_transcription_data, mute_no_singing_parts from modules.Audio.separation import DemucsModel from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3 @@ -156,6 +157,8 @@ def run() -> tuple[str, Score, Score]: print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Vocals will not be separated')}") if not settings.hyphenation: print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Hyphenation will not be applied')}") + if settings.quantize_to_key: + print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Notes will be quantized to the detected musical key')}") process_data = InitProcessData() @@ -168,6 +171,11 @@ def run() -> tuple[str, Score, Score]: # Create process audio process_data.process_data_paths.processing_audio_path = CreateProcessAudio(process_data) + # Detect key + detected_key, detected_mode = detect_key_from_audio(process_data.process_data_paths.processing_audio_path) + if process_data.media_info.music_key is None: + process_data.media_info.music_key = f"{detected_key} {detected_mode}" + # Audio transcription process_data.media_info.language = settings.language if not settings.ignore_audio: @@ -185,10 +193,18 @@ def run() -> tuple[str, Score, Score]: # Pitch audio process_data.pitched_data = pitch_audio(process_data.process_data_paths) + # Allowed keys for quantization + allowed_notes_for_key = None + if settings.quantize_to_key and not settings.ignore_audio: + allowed_notes_for_key = get_allowed_notes_for_key(detected_key, detected_mode) + # Create Midi_Segments if not settings.ignore_audio: - process_data.midi_segments = create_midi_segments_from_transcribed_data(process_data.transcribed_data, - process_data.pitched_data) + process_data.midi_segments = create_midi_segments_from_transcribed_data( + process_data.transcribed_data, + process_data.pitched_data, + allowed_notes_for_key + ) else: process_data.midi_segments = create_repitched_midi_segments_from_ultrastar_txt(process_data.pitched_data, process_data.parsed_file) @@ -301,9 +317,10 @@ def merge_syllable_segments(midi_segments: list[MidiSegment], real_bpm: float) -> tuple[list[MidiSegment], list[TranscribedData]]: """Merge sub-segments of a syllable where the pitch is the same - This function handles two cases: + This function handles three cases: 1. Merge consecutive ~ segments with the SAME pitch (same note held) 2. Detect and merge SLIDES: short ~ segments with ±1-2 semitone jumps between syllables + 3. Merge ANY consecutive segments (including regular syllables) with the SAME pitch into one word """ sixteenth_note = get_sixteenth_note_second(real_bpm) @@ -323,9 +340,13 @@ def merge_syllable_segments(midi_segments: list[MidiSegment], if previous_data is not None: has_breath_pause = (data.start - previous_data.end) > sixteenth_note + # Check if current word is a ~ segment (continuation marker) + current_word_stripped = str(data.word).strip() + is_tilde_segment = current_word_stripped == "~" + # Slide detection: Detect short ~ segments with small pitch jumps is_potential_slide = False - if i > 0 and str(data.word).strip().startswith("~"): + if i > 0 and is_tilde_segment: duration = data.end - data.start # Calculate pitch jump in semitones @@ -339,29 +360,34 @@ def merge_syllable_segments(midi_segments: list[MidiSegment], semitone_diff <= 2 and semitone_diff > 0) except: - # ignore slide detection + # Ignore slide detection on error pass - # Merge only when: - # Case 1: Same note directly consecutive (not a slide, but held note) - # Case 2: Potential slide (short ~ with small pitch jump) - should_merge = (str(data.word).strip().startswith("~") - and previous_data is not None - and (is_same_note or is_potential_slide) - and not has_breath_pause) + # Check if current segment should be merged with previous due to same pitch + should_merge_same_pitch = False + if (i > 0 and + previous_data is not None and + not is_tilde_segment and # Not a ~ segment + is_same_note and + not has_breath_pause and + not previous_data.is_word_end): # Don't merge across word boundaries + should_merge_same_pitch = True - if should_merge: - # Merge with previous segment - # For a slide: Take the note of the previous segment (ignore transition note) + should_merge_tilde = (is_tilde_segment and + previous_data is not None and + (is_same_note or is_potential_slide) and + not has_breath_pause) + + if should_merge_tilde: new_data[-1].end = data.end new_midi_notes[-1].end = data.end # For slides: Keep the original note (not the transition note) if is_potential_slide and not is_same_note: - # The note remains the one from the previous segment new_midi_notes[-1].note = midi_segments[i - 1].note # Take over space and word_end flag from current segment + # "~ " means end of word - add space to previous segment if str(data.word).endswith(" "): if not new_data[-1].word.endswith(" "): new_data[-1].word += " " @@ -369,6 +395,40 @@ def merge_syllable_segments(midi_segments: list[MidiSegment], new_midi_notes[-1].word += " " new_data[-1].is_word_end = True + elif should_merge_same_pitch: + # Merge regular syllable with previous syllable (same pitch) + new_data[-1].end = data.end + new_midi_notes[-1].end = data.end + + # Check if current word has space at end before stripping + has_space = str(data.word).endswith(" ") + + # Concatenate the words (remove trailing space from previous, add current word) + prev_word = new_data[-1].word.rstrip() + curr_word = data.word.rstrip() + + # Remove ~ from BOTH previous and current word if present + if prev_word == "~": + prev_word = "" + if curr_word.startswith("~"): + curr_word = curr_word[1:] + + # Concatenate + new_data[-1].word = prev_word + curr_word + new_midi_notes[-1].word = prev_word + curr_word + + # Preserve space at end if current segment had it + if has_space: + new_data[-1].word += " " + new_midi_notes[-1].word += " " + + if data.is_word_end: + new_data[-1].is_word_end = True + new_data[-1].is_hyphen = False + else: + # Keep hyphen status if either segment was hyphenated + new_data[-1].is_hyphen = new_data[-1].is_hyphen or data.is_hyphen + else: # Add as new segment new_data.append(data) @@ -596,7 +656,7 @@ def infos_from_audio_video_input_file() -> tuple[str, str, str, MediaInfo]: extension = os.path.splitext(basename)[1] if is_video_file(settings.input_file_path): - print(f"{ULTRASINGER_HEAD} {gold_highlighted('Video file detected - separating audio and video')}") + print(f"{ULTRASINGER_HEAD} Video file detected - separating audio and video") video_with_audio_basename = f"{basename_without_ext}{extension}" video_with_audio_path = os.path.join(song_folder_output_path, video_with_audio_basename) @@ -784,6 +844,8 @@ def init_settings(argv: list[str]) -> Settings: settings.cookiefile = arg elif opt in ("--interactive"): settings.interactive_mode = True + elif opt in ("--quantize_to_key"): + settings.quantize_to_key = arg elif opt in ("--ffmpeg"): settings.user_ffmpeg_path = arg if settings.output_folder_path == "": @@ -824,6 +886,7 @@ def arg_options(): "keep_cache", "musescore_path=", "keep_numbers", + "quantize_to_key", "interactive", "cookiefile=", "ffmpeg=" diff --git a/src/modules/Audio/key_detector.py b/src/modules/Audio/key_detector.py new file mode 100644 index 0000000..2632658 --- /dev/null +++ b/src/modules/Audio/key_detector.py @@ -0,0 +1,149 @@ +"""Key detection and pitch quantization to musical scale""" + +import copy +import librosa +import numpy as np + +from modules.Midi.MidiSegment import MidiSegment +from modules.console_colors import ( + ULTRASINGER_HEAD, +blue_highlighted) + +# scales (in semitones relative to root note) +MAJOR_SCALE = [0, 2, 4, 5, 7, 9, 11] +MINOR_SCALE = [0, 2, 3, 5, 7, 8, 10] +NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'] + + +def detect_key_from_audio(audio_path: str) -> tuple[str, str]: + """ + Detect the key and mode (major/minor) of a song from audio file. + + Args: + audio_path: Path to audio file + + Returns: + Tuple of (key_note, mode) e.g., ('C', 'major') or ('A', 'minor') + """ + print(f"{ULTRASINGER_HEAD} Detecting musical key") + + y, sr = librosa.load(audio_path, sr=None, duration=60.0) # Analyze first 60 seconds + chroma = librosa.feature.chroma_cqt(y=y, sr=sr) + chroma_avg = np.mean(chroma, axis=1) + chroma_avg = chroma_avg / np.sum(chroma_avg) + + major_template = np.array([1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1]) # Major scale pattern + minor_template = np.array([1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0]) # Minor scale pattern + major_template = major_template / np.sum(major_template) + minor_template = minor_template / np.sum(minor_template) + + best_correlation = -1 + best_key = None + best_mode = None + + # Try all 12 keys + for shift in range(12): + shifted_major = np.roll(major_template, shift) + shifted_minor = np.roll(minor_template, shift) + + major_corr = np.corrcoef(chroma_avg, shifted_major)[0, 1] + minor_corr = np.corrcoef(chroma_avg, shifted_minor)[0, 1] + + if major_corr > best_correlation: + best_correlation = major_corr + best_key = NOTE_NAMES[shift] + best_mode = 'major' + + if minor_corr > best_correlation: + best_correlation = minor_corr + best_key = NOTE_NAMES[shift] + best_mode = 'minor' + + print(f"{ULTRASINGER_HEAD} Detected key: {blue_highlighted(best_key)} {blue_highlighted(best_mode)}") + return best_key, best_mode + + +def get_allowed_notes_for_key(key_note: str, mode: str) -> set[str]: + """ + Get all allowed note names for a given key across all octaves. + + Args: + key_note: Root note (e.g., 'C', 'D#', 'F') + mode: 'major' or 'minor' + + Returns: + Set of allowed note names (e.g., {'C', 'D', 'E', ...}) + """ + root_idx = NOTE_NAMES.index(key_note) + scale = MAJOR_SCALE if mode == 'major' else MINOR_SCALE + + allowed_notes = set() + for interval in scale: + note_idx = (root_idx + interval) % 12 + allowed_notes.add(NOTE_NAMES[note_idx]) + + return allowed_notes + + +def quantize_note_to_key(note: str, allowed_notes: set[str]) -> str: + """ + Quantize a note to the nearest allowed note in the key. + + Args: + note: MIDI note name (e.g., 'C4', 'D#5') + allowed_notes: Set of allowed note names without octave + + Returns: + Quantized note name + """ + # Parse note and octave + if len(note) == 0: + return note + + # Extract note name and octave + if len(note) >= 2 and note[-2] == '#': + note_name = note[:-1] + octave = note[-1] + else: + note_name = note[:-1] if note[-1].isdigit() else note + octave = note[-1] if note[-1].isdigit() else '' + + # If already in key, return as is + if note_name in allowed_notes: + return note + + # Find nearest allowed note + try: + note_midi = librosa.note_to_midi(note) + except: + return note + + # Try all allowed notes in nearby octaves + min_distance = float('inf') + best_note = note + + for allowed_note_name in allowed_notes: + # Try same octave and adjacent octaves + for oct_offset in [-1, 0, 1]: + if not octave: + test_octave = 4 + oct_offset + else: + test_octave = int(octave) + oct_offset + + if test_octave < 0 or test_octave > 8: + continue + + test_note = f"{allowed_note_name}{test_octave}" + + try: + test_midi = librosa.note_to_midi(test_note) + distance = abs(test_midi - note_midi) + + if distance < min_distance: + min_distance = distance + best_note = test_note + except: + continue + + #print(f"{ULTRASINGER_HEAD} Moved {blue_highlighted(note)} to {blue_highlighted(best_note)}") + return best_note \ No newline at end of file diff --git a/src/modules/Midi/midi_creator.py b/src/modules/Midi/midi_creator.py index 6e9cfb1..a89f1ca 100644 --- a/src/modules/Midi/midi_creator.py +++ b/src/modules/Midi/midi_creator.py @@ -83,9 +83,20 @@ def find_nearest_index(array: list[float], value: float) -> int: return idx -def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str], pitched_data: PitchedData) -> list[ - MidiSegment]: - """Create midi notes from pitched data""" +def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str], + pitched_data: PitchedData, allowed_notes: set[str] = None) -> list[MidiSegment]: + """Create midi notes from pitched data + + Args: + start_times: List of start times + end_times: List of end times + words: List of words/syllables + pitched_data: Pitched data containing frequencies and confidence + allowed_notes: Optional set of allowed note names for key quantization + + Returns: + List of MidiSegments + """ print(f"{ULTRASINGER_HEAD} Creating midi_segments") midi_segments = [] @@ -94,7 +105,7 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis end_time = end_times[index] word = str(words[index]) - midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word) + midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word, allowed_notes) midi_segments.append(midi_segment) # todo: Progress? @@ -102,8 +113,20 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis return midi_segments -def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str) -> MidiSegment: - """Create midi note from pitched data""" +def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str, + allowed_notes: set[str] = None) -> MidiSegment: + """Create midi note from pitched data + + Args: + start_time: Start time of the note + end_time: End time of the note + pitched_data: Pitched data containing frequencies and confidence + word: The word/syllable for this note + allowed_notes: Optional set of allowed note names (without octave) for key quantization + + Returns: + One MidiSegment + """ start = find_nearest_index(pitched_data.times, start_time) end = find_nearest_index(pitched_data.times, end_time) @@ -121,10 +144,25 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch note = most_frequent(notes)[0][0] + if allowed_notes is not None: + from modules.Audio.key_detector import quantize_note_to_key + note = quantize_note_to_key(note, allowed_notes) + return MidiSegment(note, start_time, end_time, word) -def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData) -> list[MidiSegment]: +def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData, + allowed_notes: set[str] = None) -> list[MidiSegment]: + """Create MIDI segments from transcribed data + + Args: + transcribed_data: List of transcribed data segments + pitched_data: Pitched data containing frequencies and confidence + allowed_notes: Optional set of allowed note names for key quantization + + Returns: + List of MidiSegments + """ start_times = [] end_times = [] words = [] @@ -135,7 +173,7 @@ def create_midi_segments_from_transcribed_data(transcribed_data: list[Transcribe end_times.append(midi_segment.end) words.append(midi_segment.word) midi_segments = create_midi_notes_from_pitched_data(start_times, end_times, words, - pitched_data) + pitched_data, allowed_notes) return midi_segments diff --git a/src/modules/ProcessData.py b/src/modules/ProcessData.py index 13a95cf..a3350fd 100644 --- a/src/modules/ProcessData.py +++ b/src/modules/ProcessData.py @@ -25,6 +25,7 @@ class MediaInfo: language: Optional[str] = None cover_url: Optional[str] = None video_url: Optional[str] = None + music_key: Optional[str] = None # (e.g., "C major", "A minor") @dataclass class ProcessData: diff --git a/src/modules/Ultrastar/coverter/ultrastar_txt_converter.py b/src/modules/Ultrastar/coverter/ultrastar_txt_converter.py index f45a1f1..ff94f4b 100644 --- a/src/modules/Ultrastar/coverter/ultrastar_txt_converter.py +++ b/src/modules/Ultrastar/coverter/ultrastar_txt_converter.py @@ -90,6 +90,8 @@ def create_ultrastar_txt_from_automation( ultrastar_txt.videoUrl = media_info.video_url if media_info.cover_url is not None: ultrastar_txt.coverUrl = media_info.cover_url + if media_info.music_key is not None: + ultrastar_txt.tags = media_info.music_key ultrastar_file_output_path = os.path.join(song_folder_output_path, basename + ".txt") create_ultrastar_txt( diff --git a/src/modules/Ultrastar/ultrastar_txt.py b/src/modules/Ultrastar/ultrastar_txt.py index 2e61932..c889b56 100644 --- a/src/modules/Ultrastar/ultrastar_txt.py +++ b/src/modules/Ultrastar/ultrastar_txt.py @@ -33,7 +33,7 @@ class UltrastarTxtTag(str, Enum): AUDIO = 'AUDIO' # Its instead of MP3. Just renamed VOCALS = 'VOCALS' # Vocals only audio INSTRUMENTAL = 'INSTRUMENTAL' # Instrumental only audio - TAGS = 'TAGS' # Tags for the song. Can be used for filtering + TAGS = 'TAGS' # Tags for the song. Can be used for filtering. Separation with "," # 1.2.0 VIDEOURL = 'VIDEOURL' # URL to the video file diff --git a/src/modules/common_print.py b/src/modules/common_print.py index 9e4899f..909ab51 100644 --- a/src/modules/common_print.py +++ b/src/modules/common_print.py @@ -54,6 +54,7 @@ def print_help() -> None: --create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default. --keep_cache Keep cache folder after creation. Cache folder is removed by default. --plot Enable creation of plots. Plots are disabled by default. + --quantize_to_key Quantize notes to the detected musical key. This removes slides and out-of-key notes. --format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0) --musescore_path path to MuseScore executable --keep_numbers Transcribe numbers as digits and not words