Added key detection and quantization

This commit is contained in:
Vadim committed 2026-02-10 01:06:48 +01:00
1 parent d33473ec89
commit 912144afaa
10 files changed
+295 -29

No files matched your search

+10
View File
@@ -131,6 +131,7 @@ _Not all options working now!_
--create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default.
--keep_cache Keep cache folder after creation. Cache folder is removed by default.
--plot Enable creation of plots. Plots are disabled by default.
--quantize_to_key Quantize notes to detected musical key. Removes pitch slides and out-of-key notes. >> ((default) is enabled)
--format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0)
--musescore_path path to MuseScore executable
--keep_numbers Transcribe numbers as digits and not words
@@ -236,6 +237,15 @@ you have the option to disable this function; in which case the original audio f
-i XYZ --disable_separation
```
### 🎵 Key Quantization
By default, UltraSinger quantizes all detected notes to the detected musical key of the song. This helps to:
* Remove pitch slides and vocal transitions between notes
* Correct out-of-key notes from pitch detection errors
If you want to keep the raw pitch detection without quantization, set `quantize_to_key = False`.
For most songs quantization should produces better results.
### Sheet Music
For Sheet Music generation you need to have `MuseScore` installed on your system.
+3 -2
View File
@@ -1,5 +1,5 @@
# Version: 0.0.13dev12
Date: 2026.02.08
# Version: 0.0.13dev13
Date: 2026.02.10
- Changes:
- Download Cover from MusicBrainz
- Fix selection of whisper model
@@ -13,6 +13,7 @@ Date: 2026.02.08
- Download yt video/audio only once
- Support for video as input
- Optimise scale detection
- Added quantization by key
# Version: 0.0.12
Date: 2024.12.19
+2 -1
View File
@@ -11,7 +11,7 @@ from modules.Ultrastar.ultrastar_txt import FormatVersion
@dataclass
class Settings:
APP_VERSION = "0.0.13-dev12"
APP_VERSION = "0.0.13-dev13"
CONFIDENCE_THRESHOLD = 0.6
CONFIDENCE_PROMPT_TIMEOUT = 4
@@ -26,6 +26,7 @@ class Settings:
keep_cache = False
interactive_mode = False
user_ffmpeg_path = ""
quantize_to_key = True # Quantize notes to the detected key of the song
# Process data Paths
input_file_path = ""
+80 -17
View File
@@ -17,6 +17,7 @@ from modules.Audio.vocal_chunks import (
create_audio_chunks_from_transcribed_data,
create_audio_chunks_from_ultrastar_data,
)
from modules.Audio.key_detector import detect_key_from_audio, get_allowed_notes_for_key
from modules.Audio.silence_processing import remove_silence_from_transcription_data, mute_no_singing_parts
from modules.Audio.separation import DemucsModel
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
@@ -156,6 +157,8 @@ def run() -> tuple[str, Score, Score]:
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Vocals will not be separated')}")
if not settings.hyphenation:
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Hyphenation will not be applied')}")
if settings.quantize_to_key:
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Notes will be quantized to the detected musical key')}")
process_data = InitProcessData()
@@ -168,6 +171,11 @@ def run() -> tuple[str, Score, Score]:
# Create process audio
process_data.process_data_paths.processing_audio_path = CreateProcessAudio(process_data)
# Detect key
detected_key, detected_mode = detect_key_from_audio(process_data.process_data_paths.processing_audio_path)
if process_data.media_info.music_key is None:
process_data.media_info.music_key = f"{detected_key} {detected_mode}"
# Audio transcription
process_data.media_info.language = settings.language
if not settings.ignore_audio:
@@ -185,10 +193,18 @@ def run() -> tuple[str, Score, Score]:
# Pitch audio
process_data.pitched_data = pitch_audio(process_data.process_data_paths)
# Allowed keys for quantization
allowed_notes_for_key = None
if settings.quantize_to_key and not settings.ignore_audio:
allowed_notes_for_key = get_allowed_notes_for_key(detected_key, detected_mode)
# Create Midi_Segments
if not settings.ignore_audio:
process_data.midi_segments = create_midi_segments_from_transcribed_data(process_data.transcribed_data,
process_data.pitched_data)
process_data.midi_segments = create_midi_segments_from_transcribed_data(
process_data.transcribed_data,
process_data.pitched_data,
allowed_notes_for_key
)
else:
process_data.midi_segments = create_repitched_midi_segments_from_ultrastar_txt(process_data.pitched_data,
process_data.parsed_file)
@@ -301,9 +317,10 @@ def merge_syllable_segments(midi_segments: list[MidiSegment],
real_bpm: float) -> tuple[list[MidiSegment], list[TranscribedData]]:
"""Merge sub-segments of a syllable where the pitch is the same
This function handles two cases:
This function handles three cases:
1. Merge consecutive ~ segments with the SAME pitch (same note held)
2. Detect and merge SLIDES: short ~ segments with ±1-2 semitone jumps between syllables
3. Merge ANY consecutive segments (including regular syllables) with the SAME pitch into one word
"""
sixteenth_note = get_sixteenth_note_second(real_bpm)
@@ -323,9 +340,13 @@ def merge_syllable_segments(midi_segments: list[MidiSegment],
if previous_data is not None:
has_breath_pause = (data.start - previous_data.end) > sixteenth_note
# Check if current word is a ~ segment (continuation marker)
current_word_stripped = str(data.word).strip()
is_tilde_segment = current_word_stripped == "~"
# Slide detection: Detect short ~ segments with small pitch jumps
is_potential_slide = False
if i > 0 and str(data.word).strip().startswith("~"):
if i > 0 and is_tilde_segment:
duration = data.end - data.start
# Calculate pitch jump in semitones
@@ -339,29 +360,34 @@ def merge_syllable_segments(midi_segments: list[MidiSegment],
semitone_diff <= 2 and
semitone_diff > 0)
except:
# ignore slide detection
# Ignore slide detection on error
pass
# Merge only when:
# Case 1: Same note directly consecutive (not a slide, but held note)
# Case 2: Potential slide (short ~ with small pitch jump)
should_merge = (str(data.word).strip().startswith("~")
and previous_data is not None
and (is_same_note or is_potential_slide)
and not has_breath_pause)
# Check if current segment should be merged with previous due to same pitch
should_merge_same_pitch = False
if (i > 0 and
previous_data is not None and
not is_tilde_segment and # Not a ~ segment
is_same_note and
not has_breath_pause and
not previous_data.is_word_end): # Don't merge across word boundaries
should_merge_same_pitch = True
if should_merge:
# Merge with previous segment
# For a slide: Take the note of the previous segment (ignore transition note)
should_merge_tilde = (is_tilde_segment and
previous_data is not None and
(is_same_note or is_potential_slide) and
not has_breath_pause)
if should_merge_tilde:
new_data[-1].end = data.end
new_midi_notes[-1].end = data.end
# For slides: Keep the original note (not the transition note)
if is_potential_slide and not is_same_note:
# The note remains the one from the previous segment
new_midi_notes[-1].note = midi_segments[i - 1].note
# Take over space and word_end flag from current segment
# "~ " means end of word - add space to previous segment
if str(data.word).endswith(" "):
if not new_data[-1].word.endswith(" "):
new_data[-1].word += " "
@@ -369,6 +395,40 @@ def merge_syllable_segments(midi_segments: list[MidiSegment],
new_midi_notes[-1].word += " "
new_data[-1].is_word_end = True
elif should_merge_same_pitch:
# Merge regular syllable with previous syllable (same pitch)
new_data[-1].end = data.end
new_midi_notes[-1].end = data.end
# Check if current word has space at end before stripping
has_space = str(data.word).endswith(" ")
# Concatenate the words (remove trailing space from previous, add current word)
prev_word = new_data[-1].word.rstrip()
curr_word = data.word.rstrip()
# Remove ~ from BOTH previous and current word if present
if prev_word == "~":
prev_word = ""
if curr_word.startswith("~"):
curr_word = curr_word[1:]
# Concatenate
new_data[-1].word = prev_word + curr_word
new_midi_notes[-1].word = prev_word + curr_word
# Preserve space at end if current segment had it
if has_space:
new_data[-1].word += " "
new_midi_notes[-1].word += " "
if data.is_word_end:
new_data[-1].is_word_end = True
new_data[-1].is_hyphen = False
else:
# Keep hyphen status if either segment was hyphenated
new_data[-1].is_hyphen = new_data[-1].is_hyphen or data.is_hyphen
else:
# Add as new segment
new_data.append(data)
@@ -596,7 +656,7 @@ def infos_from_audio_video_input_file() -> tuple[str, str, str, MediaInfo]:
extension = os.path.splitext(basename)[1]
if is_video_file(settings.input_file_path):
print(f"{ULTRASINGER_HEAD} {gold_highlighted('Video file detected - separating audio and video')}")
print(f"{ULTRASINGER_HEAD} Video file detected - separating audio and video")
video_with_audio_basename = f"{basename_without_ext}{extension}"
video_with_audio_path = os.path.join(song_folder_output_path, video_with_audio_basename)
@@ -784,6 +844,8 @@ def init_settings(argv: list[str]) -> Settings:
settings.cookiefile = arg
elif opt in ("--interactive"):
settings.interactive_mode = True
elif opt in ("--quantize_to_key"):
settings.quantize_to_key = arg
elif opt in ("--ffmpeg"):
settings.user_ffmpeg_path = arg
if settings.output_folder_path == "":
@@ -824,6 +886,7 @@ def arg_options():
"keep_cache",
"musescore_path=",
"keep_numbers",
"quantize_to_key",
"interactive",
"cookiefile=",
"ffmpeg="
+149
View File
@@ -0,0 +1,149 @@
"""Key detection and pitch quantization to musical scale"""
import copy
import librosa
import numpy as np
from modules.Midi.MidiSegment import MidiSegment
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted)
# scales (in semitones relative to root note)
MAJOR_SCALE = [0, 2, 4, 5, 7, 9, 11]
MINOR_SCALE = [0, 2, 3, 5, 7, 8, 10]
NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']
def detect_key_from_audio(audio_path: str) -> tuple[str, str]:
"""
Detect the key and mode (major/minor) of a song from audio file.
Args:
audio_path: Path to audio file
Returns:
Tuple of (key_note, mode) e.g., ('C', 'major') or ('A', 'minor')
"""
print(f"{ULTRASINGER_HEAD} Detecting musical key")
y, sr = librosa.load(audio_path, sr=None, duration=60.0) # Analyze first 60 seconds
chroma = librosa.feature.chroma_cqt(y=y, sr=sr)
chroma_avg = np.mean(chroma, axis=1)
chroma_avg = chroma_avg / np.sum(chroma_avg)
major_template = np.array([1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1]) # Major scale pattern
minor_template = np.array([1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0]) # Minor scale pattern
major_template = major_template / np.sum(major_template)
minor_template = minor_template / np.sum(minor_template)
best_correlation = -1
best_key = None
best_mode = None
# Try all 12 keys
for shift in range(12):
shifted_major = np.roll(major_template, shift)
shifted_minor = np.roll(minor_template, shift)
major_corr = np.corrcoef(chroma_avg, shifted_major)[0, 1]
minor_corr = np.corrcoef(chroma_avg, shifted_minor)[0, 1]
if major_corr > best_correlation:
best_correlation = major_corr
best_key = NOTE_NAMES[shift]
best_mode = 'major'
if minor_corr > best_correlation:
best_correlation = minor_corr
best_key = NOTE_NAMES[shift]
best_mode = 'minor'
print(f"{ULTRASINGER_HEAD} Detected key: {blue_highlighted(best_key)} {blue_highlighted(best_mode)}")
return best_key, best_mode
def get_allowed_notes_for_key(key_note: str, mode: str) -> set[str]:
"""
Get all allowed note names for a given key across all octaves.
Args:
key_note: Root note (e.g., 'C', 'D#', 'F')
mode: 'major' or 'minor'
Returns:
Set of allowed note names (e.g., {'C', 'D', 'E', ...})
"""
root_idx = NOTE_NAMES.index(key_note)
scale = MAJOR_SCALE if mode == 'major' else MINOR_SCALE
allowed_notes = set()
for interval in scale:
note_idx = (root_idx + interval) % 12
allowed_notes.add(NOTE_NAMES[note_idx])
return allowed_notes
def quantize_note_to_key(note: str, allowed_notes: set[str]) -> str:
"""
Quantize a note to the nearest allowed note in the key.
Args:
note: MIDI note name (e.g., 'C4', 'D#5')
allowed_notes: Set of allowed note names without octave
Returns:
Quantized note name
"""
# Parse note and octave
if len(note) == 0:
return note
# Extract note name and octave
if len(note) >= 2 and note[-2] == '#':
note_name = note[:-1]
octave = note[-1]
else:
note_name = note[:-1] if note[-1].isdigit() else note
octave = note[-1] if note[-1].isdigit() else ''
# If already in key, return as is
if note_name in allowed_notes:
return note
# Find nearest allowed note
try:
note_midi = librosa.note_to_midi(note)
except:
return note
# Try all allowed notes in nearby octaves
min_distance = float('inf')
best_note = note
for allowed_note_name in allowed_notes:
# Try same octave and adjacent octaves
for oct_offset in [-1, 0, 1]:
if not octave:
test_octave = 4 + oct_offset
else:
test_octave = int(octave) + oct_offset
if test_octave < 0 or test_octave > 8:
continue
test_note = f"{allowed_note_name}{test_octave}"
try:
test_midi = librosa.note_to_midi(test_note)
distance = abs(test_midi - note_midi)
if distance < min_distance:
min_distance = distance
best_note = test_note
except:
continue
#print(f"{ULTRASINGER_HEAD} Moved {blue_highlighted(note)} to {blue_highlighted(best_note)}")
return best_note
+46 -8
View File
@@ -83,9 +83,20 @@ def find_nearest_index(array: list[float], value: float) -> int:
return idx
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str], pitched_data: PitchedData) -> list[
MidiSegment]:
"""Create midi notes from pitched data"""
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str],
pitched_data: PitchedData, allowed_notes: set[str] = None) -> list[MidiSegment]:
"""Create midi notes from pitched data
Args:
start_times: List of start times
end_times: List of end times
words: List of words/syllables
pitched_data: Pitched data containing frequencies and confidence
allowed_notes: Optional set of allowed note names for key quantization
Returns:
List of MidiSegments
"""
print(f"{ULTRASINGER_HEAD} Creating midi_segments")
midi_segments = []
@@ -94,7 +105,7 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis
end_time = end_times[index]
word = str(words[index])
midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word)
midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word, allowed_notes)
midi_segments.append(midi_segment)
# todo: Progress?
@@ -102,8 +113,20 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis
return midi_segments
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str) -> MidiSegment:
"""Create midi note from pitched data"""
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str,
allowed_notes: set[str] = None) -> MidiSegment:
"""Create midi note from pitched data
Args:
start_time: Start time of the note
end_time: End time of the note
pitched_data: Pitched data containing frequencies and confidence
word: The word/syllable for this note
allowed_notes: Optional set of allowed note names (without octave) for key quantization
Returns:
One MidiSegment
"""
start = find_nearest_index(pitched_data.times, start_time)
end = find_nearest_index(pitched_data.times, end_time)
@@ -121,10 +144,25 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch
note = most_frequent(notes)[0][0]
if allowed_notes is not None:
from modules.Audio.key_detector import quantize_note_to_key
note = quantize_note_to_key(note, allowed_notes)
return MidiSegment(note, start_time, end_time, word)
def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData) -> list[MidiSegment]:
def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData,
allowed_notes: set[str] = None) -> list[MidiSegment]:
"""Create MIDI segments from transcribed data
Args:
transcribed_data: List of transcribed data segments
pitched_data: Pitched data containing frequencies and confidence
allowed_notes: Optional set of allowed note names for key quantization
Returns:
List of MidiSegments
"""
start_times = []
end_times = []
words = []
@@ -135,7 +173,7 @@ def create_midi_segments_from_transcribed_data(transcribed_data: list[Transcribe
end_times.append(midi_segment.end)
words.append(midi_segment.word)
midi_segments = create_midi_notes_from_pitched_data(start_times, end_times, words,
pitched_data)
pitched_data, allowed_notes)
return midi_segments
+1
View File
@@ -25,6 +25,7 @@ class MediaInfo:
language: Optional[str] = None
cover_url: Optional[str] = None
video_url: Optional[str] = None
music_key: Optional[str] = None # (e.g., "C major", "A minor")
@dataclass
class ProcessData:
@@ -90,6 +90,8 @@ def create_ultrastar_txt_from_automation(
ultrastar_txt.videoUrl = media_info.video_url
if media_info.cover_url is not None:
ultrastar_txt.coverUrl = media_info.cover_url
if media_info.music_key is not None:
ultrastar_txt.tags = media_info.music_key
ultrastar_file_output_path = os.path.join(song_folder_output_path, basename + ".txt")
create_ultrastar_txt(
+1 -1
View File
@@ -33,7 +33,7 @@ class UltrastarTxtTag(str, Enum):
AUDIO = 'AUDIO' # Its instead of MP3. Just renamed
VOCALS = 'VOCALS' # Vocals only audio
INSTRUMENTAL = 'INSTRUMENTAL' # Instrumental only audio
TAGS = 'TAGS' # Tags for the song. Can be used for filtering
TAGS = 'TAGS' # Tags for the song. Can be used for filtering. Separation with ","
# 1.2.0
VIDEOURL = 'VIDEOURL' # URL to the video file
+1
View File
@@ -54,6 +54,7 @@ def print_help() -> None:
--create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default.
--keep_cache Keep cache folder after creation. Cache folder is removed by default.
--plot Enable creation of plots. Plots are disabled by default.
--quantize_to_key Quantize notes to the detected musical key. This removes slides and out-of-key notes.
--format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0)
--musescore_path path to MuseScore executable
--keep_numbers Transcribe numbers as digits and not words