mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Merge branch 'main' into Change_youtube_ext
This commit is contained in:
commit
c3de810f82
25 files changed
+397
-327
No files matched your search
@@ -41,6 +41,7 @@ This will help me alot to keep this project alive and improve it.
|
||||
* Type `.venv\Scripts\activate` and press enter.
|
||||
* You should see now a `(.venv)` in front of your console line.
|
||||
* Install the requirements with `pip install -r requirements.txt`.
|
||||
* Install gpu requirements `pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117`
|
||||
* Now you can use the UltraSinger source code with `py UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]`. See [How to use](#how-to-use) for more information.
|
||||
|
||||
For more information about Python environments look [here](https://code.visualstudio.com/docs/python/environments#_global-virtual-or-conda-environments).
|
||||
@@ -51,6 +52,7 @@ As copy:
|
||||
py -m venv .venv
|
||||
.venv\Scripts\activate
|
||||
pip install -r requirements.txt
|
||||
pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
|
||||
|
||||
```
|
||||
|
||||
@@ -192,3 +194,38 @@ for low male voices. Accurate is the real tone specified in the txt. I had txt f
|
||||
singable by humans, but you could still reach the 10k points in the game. The accuracy is important here, because from
|
||||
this MIDI and sheet are created. And you also want to have accurate files
|
||||
|
||||
|
||||
### Use GPU
|
||||
|
||||
With an GPU you can speed up the process and also the quality of the transcription and pitching is better.
|
||||
|
||||
You need a cuda device for this to work.
|
||||
For best use you need to install the cuda driver for your gpu see [here](https://developer.nvidia.com/cuda-downloads).
|
||||
Install torch with cuda separately in your `venv`. See this [link](https://pytorch.org/get-started/locally/).
|
||||
|
||||
Command for `pip`:
|
||||
```
|
||||
pip3 install torch~=2.0.1+cu117 torchvision~=0.15.2+cu117 torchaudio~=2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
|
||||
```
|
||||
|
||||
When you want to use `conda` instead you need a different installation command. See this [link](https://pytorch.org/get-started/locally/).
|
||||
|
||||
#### Info
|
||||
|
||||
If somthing crash because of low VRAM than use a smaller model.
|
||||
Whisper needs more than 8GB VRAM in the `large` model!
|
||||
|
||||
But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`.
|
||||
|
||||
### Use GPU
|
||||
|
||||
With an GPU you can speed up the process and also the quality of the transcription and pitching is better.
|
||||
You need a cuda device for this to work.
|
||||
If you use MAC than sorry, there is no cuda device for MAC machinges.
|
||||
|
||||
#### Info
|
||||
|
||||
If somthing crash because of low VRAM than use a smaller model.
|
||||
Whisper needs more than 8GB VRAM in the `large` model!
|
||||
|
||||
But you can force cpu usage with the extra options `--force_whisper_cpu` and `--force_separation_cpu`.
|
||||
+4
-1
@@ -26,4 +26,7 @@ class Settings:
|
||||
crepe_step_size = 10
|
||||
|
||||
# Device
|
||||
device = "cpu" # cpu|cuda|gpu
|
||||
device = 'cpu' # cpu|cuda
|
||||
force_whisper_cpu = False
|
||||
force_separation_cpu = False
|
||||
|
||||
+60
-53
@@ -15,13 +15,12 @@ from modules import os_helper
|
||||
from modules.Audio.denoise import ffmpeg_reduce_noise
|
||||
from modules.Audio.separation import separate_audio
|
||||
from modules.Audio.vocal_chunks import (
|
||||
convert_audio_to_mono_wav,
|
||||
convert_wav_to_mp3,
|
||||
export_chunks_from_transcribed_data,
|
||||
export_chunks_from_ultrastar_data,
|
||||
export_transcribed_data_to_csv,
|
||||
remove_silence_from_transcribtion_data,
|
||||
)
|
||||
from modules.Audio.silence_processing import remove_silence_from_transcribtion_data
|
||||
from modules.csv_export import export_transcribed_data_to_csv
|
||||
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
|
||||
from modules.Audio.youtube import (
|
||||
download_youtube_audio,
|
||||
download_youtube_thumbnail,
|
||||
@@ -29,12 +28,12 @@ from modules.Audio.youtube import (
|
||||
get_youtube_title,
|
||||
)
|
||||
from modules.DeviceDetection.device_detection import get_available_device
|
||||
from modules.Log import (
|
||||
PRINT_ULTRASTAR,
|
||||
print_blue_highlighted_text,
|
||||
print_gold_highlighted_text,
|
||||
print_light_blue_highlighted_text,
|
||||
print_red_highlighted_text,
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
gold_highlighted,
|
||||
light_blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.Midi import midi_creator
|
||||
from modules.Midi.midi_creator import (
|
||||
@@ -68,20 +67,21 @@ def get_confidence(pitched_data, threshold):
|
||||
conf_f = []
|
||||
conf_c = []
|
||||
for i in enumerate(pitched_data.times):
|
||||
pos = i[0]
|
||||
if pitched_data.confidence[i] > threshold:
|
||||
conf_t.append(pitched_data.times[i])
|
||||
conf_f.append(pitched_data.frequencies[i])
|
||||
conf_c.append(pitched_data.confidence[i])
|
||||
conf_t.append(pitched_data.times[pos])
|
||||
conf_f.append(pitched_data.frequencies[pos])
|
||||
conf_c.append(pitched_data.confidence[pos])
|
||||
return conf_t, conf_f, conf_c
|
||||
|
||||
|
||||
def convert_ultrastar_note_numbers(midi_notes):
|
||||
"""Docstring"""
|
||||
print(f"{PRINT_ULTRASTAR} Creating Ultrastar notes from midi data")
|
||||
print(f"{ULTRASINGER_HEAD} Creating Ultrastar notes from midi data")
|
||||
|
||||
ultrastar_note_numbers = []
|
||||
for i in enumerate(midi_notes):
|
||||
note_number_librosa = librosa.note_to_midi(midi_notes[i])
|
||||
for i, note in enumerate(midi_notes):
|
||||
note_number_librosa = librosa.note_to_midi(note)
|
||||
pitch = ultrastar_converter.midi_note_to_ultrastar_note(
|
||||
note_number_librosa
|
||||
)
|
||||
@@ -96,7 +96,7 @@ def convert_ultrastar_note_numbers(midi_notes):
|
||||
def pitch_each_chunk_with_crepe(directory):
|
||||
"""Docstring"""
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Pitching each chunk with {print_blue_highlighted_text('crepe')}"
|
||||
f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}"
|
||||
)
|
||||
|
||||
midi_notes = []
|
||||
@@ -125,28 +125,28 @@ def pitch_each_chunk_with_crepe(directory):
|
||||
|
||||
def add_hyphen_to_data(transcribed_data, hyphen_words):
|
||||
"""Docstring"""
|
||||
data = []
|
||||
new_data = []
|
||||
|
||||
for i in enumerate(transcribed_data):
|
||||
for i, data in enumerate(transcribed_data):
|
||||
if not hyphen_words[i]:
|
||||
data.append(transcribed_data[i])
|
||||
new_data.append(data)
|
||||
else:
|
||||
chunk_duration = transcribed_data[i].end - transcribed_data[i].start
|
||||
chunk_duration = data.end - data.start
|
||||
chunk_duration = chunk_duration / (len(hyphen_words[i]))
|
||||
|
||||
next_start = transcribed_data[i].start
|
||||
for j in enumerate(hyphen_words[i]):
|
||||
dup = copy.copy(transcribed_data[i])
|
||||
next_start = data.start
|
||||
for j, hyphens in enumerate(hyphen_words[i]):
|
||||
dup = copy.copy(data)
|
||||
dup.start = next_start
|
||||
next_start = transcribed_data[i].end - chunk_duration * (
|
||||
len(hyphen_words[i]) - 1 - j
|
||||
next_start = data.end - chunk_duration * (
|
||||
len(hyphens) - 1 - j
|
||||
)
|
||||
dup.end = next_start
|
||||
dup.word = hyphen_words[i][j]
|
||||
dup.is_hyphen = True
|
||||
data.append(dup)
|
||||
new_data.append(dup)
|
||||
|
||||
return data
|
||||
return new_data
|
||||
|
||||
|
||||
def get_bpm_from_data(data, sampling_rate):
|
||||
@@ -155,7 +155,7 @@ def get_bpm_from_data(data, sampling_rate):
|
||||
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} BPM is {print_blue_highlighted_text(str(round(wav_tempo[0], 2)))}"
|
||||
f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}"
|
||||
)
|
||||
return wav_tempo[0]
|
||||
|
||||
@@ -238,10 +238,10 @@ def plot(input_file, vosk_transcribed_data, midi_notes):
|
||||
plt.plot(conf_t, conf_f, linewidth=0.1)
|
||||
plt.savefig(os.path.join("test", "crepe_0.4.png"))
|
||||
|
||||
for i in enumerate(vosk_transcribed_data):
|
||||
for i, data in enumerate(vosk_transcribed_data):
|
||||
note_frequency = librosa.note_to_hz(midi_notes[i])
|
||||
plt.plot(
|
||||
[vosk_transcribed_data[i].start, vosk_transcribed_data[i].end],
|
||||
[data.start, data.end],
|
||||
[note_frequency, note_frequency],
|
||||
linewidth=1,
|
||||
alpha=0.5,
|
||||
@@ -252,8 +252,8 @@ def plot(input_file, vosk_transcribed_data, midi_notes):
|
||||
def remove_unecessary_punctuations(transcribed_data):
|
||||
"""Docstring"""
|
||||
punctuation = ".,"
|
||||
for i in enumerate(transcribed_data):
|
||||
transcribed_data[i].word = transcribed_data[i].word.translate(
|
||||
for i, data in enumerate(transcribed_data):
|
||||
data.word = data.word.translate(
|
||||
{ord(i): None for i in punctuation}
|
||||
)
|
||||
|
||||
@@ -264,14 +264,14 @@ def hyphenate_each_word(language, transcribed_data):
|
||||
lang_region = language_check(language)
|
||||
if lang_region is None:
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_red_highlighted_text('Error in hyphenation for language ')} {print_blue_highlighted_text(language)} {print_red_highlighted_text(', maybe you want to disable it?')}"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error in hyphenation for language ')} {blue_highlighted(language)} {red_highlighted(', maybe you want to disable it?')}"
|
||||
)
|
||||
return None
|
||||
|
||||
sleep(0.1)
|
||||
for i in tqdm(enumerate(transcribed_data)):
|
||||
for i, data in tqdm(enumerate(transcribed_data)):
|
||||
hyphenated_word.append(
|
||||
hyphenation(transcribed_data[i].word, lang_region)
|
||||
hyphenation(data.word, lang_region)
|
||||
)
|
||||
return hyphenated_word
|
||||
|
||||
@@ -280,13 +280,13 @@ def print_support():
|
||||
"""Docstring"""
|
||||
print()
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('Do you like UltraSinger? And want it to be even better? Then help with your')} {print_light_blue_highlighted_text('support')}{print_gold_highlighted_text('!')}"
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('Do you like UltraSinger? And want it to be even better? Then help with your')} {light_blue_highlighted('support')}{gold_highlighted('!')}"
|
||||
)
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} See project page -> https://github.com/rakuri255/UltraSinger"
|
||||
f"{ULTRASINGER_HEAD} See project page -> https://github.com/rakuri255/UltraSinger"
|
||||
)
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('This will help alot to keep this project alive and improved.')}"
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('This will help alot to keep this project alive and improved.')}"
|
||||
)
|
||||
|
||||
|
||||
@@ -298,7 +298,7 @@ def run():
|
||||
|
||||
if not is_audio: # Parse Ultrastar txt
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('re-pitch mode')}"
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('re-pitch mode')}"
|
||||
)
|
||||
(
|
||||
basename_without_ext,
|
||||
@@ -309,7 +309,7 @@ def run():
|
||||
) = parse_ultrastar_txt()
|
||||
elif settings.input_file_path.startswith("https:"): # Youtube
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('full automatic mode')}"
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
|
||||
)
|
||||
(
|
||||
basename_without_ext,
|
||||
@@ -318,7 +318,7 @@ def run():
|
||||
) = download_from_youtube()
|
||||
else: # Audio File
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_gold_highlighted_text('full automatic mode')}"
|
||||
f"{ULTRASINGER_HEAD} {gold_highlighted('full automatic mode')}"
|
||||
)
|
||||
(
|
||||
basename_without_ext,
|
||||
@@ -428,7 +428,7 @@ def get_unused_song_output_dir(path):
|
||||
i += 1
|
||||
if i > 999:
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_red_highlighted_text('Error: Could not create output folder! (999) is the maximum number of tries.')}"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error: Could not create output folder! (999) is the maximum number of tries.')}"
|
||||
)
|
||||
sys.exit(1)
|
||||
return path
|
||||
@@ -437,9 +437,9 @@ def get_unused_song_output_dir(path):
|
||||
def transcribe_audio(transcribed_data):
|
||||
"""Docstring"""
|
||||
if settings.transcriber == "whisper":
|
||||
device = "cpu" if settings.force_whisper_cpu else settings.device
|
||||
transcribed_data, language = transcribe_with_whisper(
|
||||
settings.mono_audio_path, settings.whisper_model, settings.device
|
||||
)
|
||||
settings.mono_audio_path, settings.whisper_model, device)
|
||||
else: # vosk
|
||||
transcribed_data = transcribe_with_vosk(
|
||||
settings.mono_audio_path, settings.vosk_model_path
|
||||
@@ -450,14 +450,15 @@ def transcribe_audio(transcribed_data):
|
||||
|
||||
|
||||
def separate_vocal_from_audio(
|
||||
basename_without_ext, cache_path, ultrastar_audio_input_path
|
||||
basename_without_ext, cache_path, ultrastar_audio_input_path
|
||||
):
|
||||
"""Docstring"""
|
||||
audio_separation_path = os.path.join(
|
||||
cache_path, "separated", "htdemucs", basename_without_ext
|
||||
)
|
||||
device = "cpu" if settings.force_separation_cpu else settings.device
|
||||
if settings.use_separated_vocal or settings.create_karaoke:
|
||||
separate_audio(ultrastar_audio_input_path, cache_path)
|
||||
separate_audio(ultrastar_audio_input_path, cache_path, device)
|
||||
if settings.use_separated_vocal:
|
||||
vocals_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
convert_audio_to_mono_wav(vocals_path, settings.mono_audio_path)
|
||||
@@ -487,7 +488,7 @@ def calculate_score_points(
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_blue_highlighted_text('Score of original Ultrastar txt')}"
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of original Ultrastar txt')}"
|
||||
)
|
||||
(
|
||||
simple_score,
|
||||
@@ -499,7 +500,7 @@ def calculate_score_points(
|
||||
simple_score, accurate_score
|
||||
)
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_blue_highlighted_text('Score of re-pitched Ultrastar txt')}"
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('Score of re-pitched Ultrastar txt')}"
|
||||
)
|
||||
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
|
||||
ultrastar_file_output
|
||||
@@ -660,7 +661,7 @@ def parse_ultrastar_txt():
|
||||
def create_midi_file(is_audio, real_bpm, song_output, ultrastar_class):
|
||||
"""Docstring"""
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Creating Midi with {print_blue_highlighted_text('pretty_midi')}"
|
||||
f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}"
|
||||
)
|
||||
if is_audio:
|
||||
voice_instrument = [
|
||||
@@ -692,9 +693,9 @@ def pitch_audio(is_audio, transcribed_data, ultrastar_class):
|
||||
if is_audio:
|
||||
start_times = []
|
||||
end_times = []
|
||||
for i in enumerate(transcribed_data):
|
||||
start_times.append(transcribed_data[i].start)
|
||||
end_times.append(transcribed_data[i].end)
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_times.append(data.start)
|
||||
end_times.append(data.end)
|
||||
midi_notes = create_midi_notes_from_pitched_data(
|
||||
start_times, end_times, pitched_data
|
||||
)
|
||||
@@ -761,6 +762,8 @@ def init_settings(argv):
|
||||
"disable_separation=",
|
||||
"disable_karaoke=",
|
||||
"create_audio_chunks=",
|
||||
"force_whisper_cpu=",
|
||||
"force_separation_cpu="
|
||||
]
|
||||
opts, args = getopt.getopt(argv, short, long)
|
||||
if len(opts) == 0:
|
||||
@@ -790,6 +793,10 @@ def init_settings(argv):
|
||||
settings.create_karaoke = not arg
|
||||
elif opt in ("--create_audio_chunks"):
|
||||
settings.create_audio_chunks = arg
|
||||
elif opt in ("--force_whisper_cpu"):
|
||||
settings.force_whisper_cpu = arg
|
||||
elif opt in ("--force_separation_cpu"):
|
||||
settings.force_separation_cpu = arg
|
||||
if settings.output_file_path == "":
|
||||
if settings.input_file_path.startswith("https:"):
|
||||
dirname = os.getcwd()
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Convert audio to other formats"""
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def convert_audio_to_mono_wav(input_file: str, output_file: str) -> None:
|
||||
"""Convert audio to mono wav"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting audio for AI")
|
||||
|
||||
if ".mp3" in input_file:
|
||||
sound = AudioSegment.from_mp3(input_file)
|
||||
elif ".wav" in input_file:
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
else:
|
||||
raise ValueError("data format not supported")
|
||||
|
||||
sound = sound.set_channels(1)
|
||||
sound.export(output_file, format="wav")
|
||||
|
||||
|
||||
def convert_wav_to_mp3(input_file: str, output_file: str) -> None:
|
||||
"""Convert wav to mp3"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting wav to mp3")
|
||||
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
sound.export(output_file, format="mp3")
|
||||
@@ -1,12 +1,13 @@
|
||||
"""Docstring"""
|
||||
"""Reduce noise from audio"""
|
||||
|
||||
import ffmpeg
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR, print_blue_highlighted_text
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
|
||||
|
||||
|
||||
def ffmpeg_reduce_noise(input_file_path, output_file):
|
||||
"""Docstring"""
|
||||
def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None:
|
||||
"""Reduce noise from vocal audio with ffmpeg."""
|
||||
|
||||
# Denoise audio samples with FFT.
|
||||
# A description of the accepted parameters follows.
|
||||
|
||||
@@ -19,7 +20,7 @@ def ffmpeg_reduce_noise(input_file_path, output_file):
|
||||
# With this enabled, noise floor is automatically adjusted.
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Reduce noise from vocal audio with {print_blue_highlighted_text('ffmpeg')}."
|
||||
f"{ULTRASINGER_HEAD} Reduce noise from vocal audio with {blue_highlighted('ffmpeg')}."
|
||||
)
|
||||
(
|
||||
ffmpeg.input(input_file_path)
|
||||
|
||||
+11
-11
@@ -1,27 +1,27 @@
|
||||
"""Docstring"""
|
||||
"""Separate vocals from audio"""
|
||||
|
||||
import subprocess
|
||||
|
||||
from modules.Log import (
|
||||
PRINT_ULTRASTAR,
|
||||
print_blue_highlighted_text,
|
||||
print_red_highlighted_text,
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.os_helper import current_executor_path, move, path_join
|
||||
|
||||
|
||||
def separate_audio(input_file_path, output_file, device="cpu"):
|
||||
"""Docstring"""
|
||||
def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None:
|
||||
"""Separate vocals from audio with demucs."""
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Separating vocals from audio with {print_blue_highlighted_text('demucs')} and {print_red_highlighted_text(device)} as worker."
|
||||
f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker."
|
||||
)
|
||||
# Model selection?
|
||||
# -n mdx_q
|
||||
# -n htdemucs_ft
|
||||
# todo: -d cpu otherwise it automatically uses gpu
|
||||
device = "cpu" if device != "cuda" else "cuda"
|
||||
subprocess.run(
|
||||
["demucs", "--two-stems=vocals", input_file_path], check=True
|
||||
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
|
||||
)
|
||||
separated_folder = path_join(current_executor_path(), "separated")
|
||||
move(separated_folder, output_file)
|
||||
move(separated_folder, output_file)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Silence processing module"""
|
||||
|
||||
import librosa
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: []) -> []:
|
||||
"""Remove silence from given transcribtion data"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data"
|
||||
)
|
||||
|
||||
audio, sample_rate = librosa.load(audio_path, sr=None)
|
||||
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_time = data.start
|
||||
end_time = data.end
|
||||
start_sample = int(start_time * sample_rate)
|
||||
end_sample = int(end_time * sample_rate)
|
||||
chunk = audio[start_sample:end_sample]
|
||||
|
||||
# todo: why 5 works good? It should be 40db ?!?
|
||||
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
|
||||
silence_threshold = 5
|
||||
onsets = librosa.effects.split(
|
||||
chunk, top_db=silence_threshold, frame_length=2048, hop_length=100
|
||||
)
|
||||
|
||||
# Get the duration of the first and last silent intervals
|
||||
if len(onsets) > 0:
|
||||
first_silence = onsets[0][0]
|
||||
last_silence = len(chunk) - onsets[-1][1]
|
||||
|
||||
first_silence_duration = librosa.samples_to_time(
|
||||
first_silence, sr=sample_rate
|
||||
)
|
||||
last_silence_duration = librosa.samples_to_time(
|
||||
last_silence, sr=sample_rate
|
||||
)
|
||||
else:
|
||||
first_silence_duration = 0
|
||||
last_silence_duration = 0
|
||||
|
||||
data.start = (
|
||||
data.start + first_silence_duration
|
||||
)
|
||||
data.end = (
|
||||
data.end - last_silence_duration
|
||||
)
|
||||
|
||||
return transcribed_data
|
||||
+17
-109
@@ -1,41 +1,16 @@
|
||||
"""Docstring"""
|
||||
"""Vocal chunks module."""
|
||||
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import wave
|
||||
|
||||
import librosa
|
||||
from pydub import AudioSegment
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.os_helper import create_folder
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
)
|
||||
|
||||
|
||||
def convert_audio_to_mono_wav(input_file, output_file):
|
||||
"""Convert audio to mono wav"""
|
||||
print(f"{PRINT_ULTRASTAR} Converting audio for AI")
|
||||
|
||||
if ".mp3" in input_file:
|
||||
sound = AudioSegment.from_mp3(input_file)
|
||||
elif ".wav" in input_file:
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
else:
|
||||
raise ValueError("data format not supported")
|
||||
|
||||
sound = sound.set_channels(1)
|
||||
sound.export(output_file, format="wav")
|
||||
|
||||
|
||||
def convert_wav_to_mp3(input_file, output_file):
|
||||
"""Docstring"""
|
||||
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
sound.export(output_file, format="mp3")
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
|
||||
|
||||
class AudioManipulation:
|
||||
@@ -43,88 +18,40 @@ class AudioManipulation:
|
||||
|
||||
|
||||
def export_chunks_from_transcribed_data(
|
||||
audio_filename, transcribed_data, output_folder_name
|
||||
):
|
||||
audio_filename: str, transcribed_data: [], output_folder_name: str
|
||||
) -> None:
|
||||
"""Export transcribed_data as vocal chunks wav files"""
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Export transcribed data as vocal chunks wav files"
|
||||
f"{ULTRASINGER_HEAD} Export transcribed data as vocal chunks wav files"
|
||||
)
|
||||
|
||||
wave_file = wave.open(audio_filename, "rb")
|
||||
sample_rate, n_channels = wave_file.getparams()[2], wave_file.getparams()[0]
|
||||
|
||||
for i in enumerate(transcribed_data):
|
||||
start_byte = int(transcribed_data[i].start * sample_rate * n_channels)
|
||||
end_byte = int(transcribed_data[i].end * sample_rate * n_channels)
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_byte = int(data.start * sample_rate * n_channels)
|
||||
end_byte = int(data.end * sample_rate * n_channels)
|
||||
|
||||
chunk = get_chunk(end_byte, start_byte, wave_file)
|
||||
export_chunk_to_wav_file(
|
||||
chunk, output_folder_name, i, transcribed_data[i].word, wave_file
|
||||
chunk, output_folder_name, i, data.word, wave_file
|
||||
)
|
||||
|
||||
wave_file.close()
|
||||
|
||||
|
||||
def remove_silence_from_transcribtion_data(audio_path, transcribed_data):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Removing silent start and ending, from transcription data"
|
||||
)
|
||||
|
||||
audio, sample_rate = librosa.load(audio_path, sr=None)
|
||||
|
||||
for i in enumerate(transcribed_data):
|
||||
start_time = transcribed_data[i].start
|
||||
end_time = transcribed_data[i].end
|
||||
start_sample = int(start_time * sample_rate)
|
||||
end_sample = int(end_time * sample_rate)
|
||||
chunk = audio[start_sample:end_sample]
|
||||
|
||||
# todo: why 5 works good? It should be 40db ?!?
|
||||
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
|
||||
silence_threshold = 5
|
||||
onsets = librosa.effects.split(
|
||||
chunk, top_db=silence_threshold, frame_length=2048, hop_length=100
|
||||
)
|
||||
|
||||
# Get the duration of the first and last silent intervals
|
||||
if len(onsets) > 0:
|
||||
first_silence = onsets[0][0]
|
||||
last_silence = len(chunk) - onsets[-1][1]
|
||||
|
||||
first_silence_duration = librosa.samples_to_time(
|
||||
first_silence, sr=sample_rate
|
||||
)
|
||||
last_silence_duration = librosa.samples_to_time(
|
||||
last_silence, sr=sample_rate
|
||||
)
|
||||
else:
|
||||
first_silence_duration = 0
|
||||
last_silence_duration = 0
|
||||
|
||||
transcribed_data[i].start = (
|
||||
transcribed_data[i].start + first_silence_duration
|
||||
)
|
||||
transcribed_data[i].end = (
|
||||
transcribed_data[i].end - last_silence_duration
|
||||
)
|
||||
|
||||
return transcribed_data
|
||||
|
||||
|
||||
def export_chunks_from_ultrastar_data(
|
||||
audio_filename, ultrastar_data, folder_name
|
||||
):
|
||||
audio_filename: str, ultrastar_data: UltrastarTxt, folder_name: str
|
||||
) -> None:
|
||||
"""Export ultrastar data as vocal chunks wav files"""
|
||||
print(f"{PRINT_ULTRASTAR} Export Ultrastar data as vocal chunks wav files")
|
||||
print(f"{ULTRASINGER_HEAD} Export Ultrastar data as vocal chunks wav files")
|
||||
|
||||
create_folder(folder_name)
|
||||
|
||||
wave_file = wave.open(audio_filename, "rb")
|
||||
sample_rate, n_channels = wave_file.getparams()[2], wave_file.getparams()[0]
|
||||
|
||||
for i in enumerate(ultrastar_data.words):
|
||||
for i, word in enumerate(ultrastar_data.words):
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_data, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_data, i)
|
||||
|
||||
@@ -133,11 +60,11 @@ def export_chunks_from_ultrastar_data(
|
||||
|
||||
chunk = get_chunk(end_byte, start_byte, wave_file)
|
||||
export_chunk_to_wav_file(
|
||||
chunk, folder_name, i, ultrastar_data.words[i], wave_file
|
||||
chunk, folder_name, i, word, wave_file
|
||||
)
|
||||
|
||||
|
||||
def export_chunk_to_wav_file(chunk, folder_name, i, word, wave_file):
|
||||
def export_chunk_to_wav_file(chunk, folder_name: str, i: int, word: str, wave_file) -> None:
|
||||
"""Export vocal chunks to wav file"""
|
||||
|
||||
clean_word = re.sub("[^A-Za-z0-9]+", "", word)
|
||||
@@ -150,7 +77,7 @@ def export_chunk_to_wav_file(chunk, folder_name, i, word, wave_file):
|
||||
chunk_file.writeframes(chunk)
|
||||
|
||||
|
||||
def get_chunk(end_byte, start_byte, wave_file):
|
||||
def get_chunk(end_byte: int, start_byte: int, wave_file):
|
||||
"""
|
||||
Gets the chunk from wave file.
|
||||
Returns chunk as n frames of audio, as a bytes object.
|
||||
@@ -160,22 +87,3 @@ def get_chunk(end_byte, start_byte, wave_file):
|
||||
wave_file.setpos(start_byte) # ({:.2f})
|
||||
chunk = wave_file.readframes(end_byte - start_byte)
|
||||
return chunk
|
||||
|
||||
|
||||
def export_transcribed_data_to_csv(transcribed_data, filename):
|
||||
"""Export transcribed data to csv"""
|
||||
print(f"{PRINT_ULTRASTAR} Exporting transcribed data to CSV")
|
||||
|
||||
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["word", "start", "end", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i in enumerate(transcribed_data):
|
||||
writer.writerow(
|
||||
[
|
||||
transcribed_data[i].word,
|
||||
transcribed_data[i].start,
|
||||
transcribed_data[i].end,
|
||||
transcribed_data[i].conf,
|
||||
]
|
||||
)
|
||||
+17
-17
@@ -1,4 +1,4 @@
|
||||
"""Docstring"""
|
||||
"""YouTube Downloader"""
|
||||
|
||||
import io
|
||||
import os
|
||||
@@ -6,11 +6,11 @@ import os
|
||||
import yt_dlp
|
||||
from PIL import Image
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def get_youtube_title(url):
|
||||
"""Docstring"""
|
||||
def get_youtube_title(url: str) -> str:
|
||||
"""Get the title of the YouTube video"""
|
||||
|
||||
ydl_opts = {}
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
@@ -21,10 +21,10 @@ def get_youtube_title(url):
|
||||
return result["title"]
|
||||
|
||||
|
||||
def download_youtube_audio(url, clear_filename, output_path):
|
||||
"""Docstring"""
|
||||
def download_youtube_audio(url: str, clear_filename: str, output_path: str):
|
||||
"""Download audio from YouTube"""
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Downloading Audio")
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Audio")
|
||||
ydl_opts = {
|
||||
"format": "bestaudio/best",
|
||||
"outtmpl": output_path + "/" + clear_filename,
|
||||
@@ -36,10 +36,10 @@ def download_youtube_audio(url, clear_filename, output_path):
|
||||
start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def download_youtube_thumbnail(url, clear_filename, output_path):
|
||||
"""Docstring"""
|
||||
def download_youtube_thumbnail(url: str, clear_filename: str, output_path: str):
|
||||
"""Download thumbnail from YouTube"""
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Downloading thumbnail")
|
||||
print(f"{ULTRASINGER_HEAD} Downloading thumbnail")
|
||||
ydl_opts = {
|
||||
"skip_download": True,
|
||||
"writethumbnail": True,
|
||||
@@ -48,8 +48,8 @@ def download_youtube_thumbnail(url, clear_filename, output_path):
|
||||
download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path)
|
||||
|
||||
|
||||
def download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path):
|
||||
"""Docstring"""
|
||||
def download_and_convert_thumbnail(ydl_opts, url: str, clear_filename: str, output_path: str) -> None:
|
||||
"""Download and convert thumbnail from YouTube"""
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
info_dict = ydl.extract_info(url, download=False)
|
||||
@@ -63,10 +63,10 @@ def download_and_convert_thumbnail(ydl_opts, url, clear_filename, output_path):
|
||||
)
|
||||
|
||||
|
||||
def download_youtube_video(url, clear_filename, output_path):
|
||||
"""Docstring"""
|
||||
def download_youtube_video(url: str, clear_filename: str, output_path: str) -> None:
|
||||
"""Download video from YouTube"""
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Downloading Video")
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Video")
|
||||
ydl_opts = {
|
||||
'format': 'bestvideo',
|
||||
'outtmpl': output_path + '/' + clear_filename + '.%(ext)s'
|
||||
@@ -74,8 +74,8 @@ def download_youtube_video(url, clear_filename, output_path):
|
||||
start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def start_download(ydl_opts, url):
|
||||
"""Docstring"""
|
||||
def start_download(ydl_opts, url: str) -> None:
|
||||
"""Start the download the ydl_opts"""
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
errors = ydl.download(url)
|
||||
|
||||
@@ -1,26 +1,21 @@
|
||||
"""Docstring"""
|
||||
|
||||
import tensorflow as tf
|
||||
import torch
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR, print_red_highlighted_text
|
||||
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted
|
||||
|
||||
|
||||
def get_available_device():
|
||||
"""Docstring"""
|
||||
gpus = tf.config.list_physical_devices("GPU")
|
||||
isCuda = torch.cuda.is_available()
|
||||
|
||||
if not gpus:
|
||||
if not isCuda:
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} There are no GPUs available. Using {print_red_highlighted_text('cpu')}."
|
||||
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
|
||||
)
|
||||
|
||||
return "cpu"
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Found available GPUs:")
|
||||
for gpu in gpus:
|
||||
print(f"Name: {gpu.name}, Type: {gpu.device_type}")
|
||||
|
||||
# Todo: Finish this
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} GPU usage are currently in development. Using {print_red_highlighted_text('cpu')}"
|
||||
)
|
||||
return "cpu"
|
||||
return "cuda"
|
||||
@@ -1,45 +0,0 @@
|
||||
"""Docstring"""
|
||||
|
||||
PRINT_ULTRASTAR = "\033[92m[UltraSinger]\033[0m"
|
||||
|
||||
|
||||
def print_blue_highlighted_text(text):
|
||||
"""Docstring"""
|
||||
return Bcolors.BLUE + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_gold_highlighted_text(text):
|
||||
"""Docstring"""
|
||||
return Bcolors.Gold + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_light_blue_highlighted_text(text):
|
||||
"""Docstring"""
|
||||
return Bcolors.LightBlue + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_underlined_text(text):
|
||||
"""Docstring"""
|
||||
return Bcolors.Underline + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_red_highlighted_text(text):
|
||||
"""Docstring"""
|
||||
return Bcolors.RED + text + Bcolors.ENDC
|
||||
|
||||
|
||||
def print_cyan_highlighted_text(text):
|
||||
"""Docstring"""
|
||||
return Bcolors.Cyan + text + Bcolors.ENDC
|
||||
|
||||
|
||||
class Bcolors:
|
||||
"""Docstring"""
|
||||
|
||||
BLUE = "\033[94m"
|
||||
RED = "\033[91m"
|
||||
LightBlue = "\033[96m"
|
||||
Cyan = "\033[36m"
|
||||
Gold = "\033[93m"
|
||||
Underline = "\033[4m"
|
||||
ENDC = "\033[0m"
|
||||
@@ -7,27 +7,31 @@ import librosa
|
||||
import numpy as np
|
||||
import pretty_midi
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.Pitcher.pitcher import get_frequency_with_high_confidence
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
ultrastar_note_to_midi_note,
|
||||
)
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
red_highlighted,
|
||||
)
|
||||
|
||||
|
||||
def convert_ultrastar_to_midi_instrument(ultrastar_class):
|
||||
"""Docstring"""
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Creating midi instrument from Ultrastar txt")
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi instrument from Ultrastar txt")
|
||||
|
||||
instrument = pretty_midi.Instrument(program=0)
|
||||
velocity = 100
|
||||
|
||||
for i in enumerate(ultrastar_class.words):
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, i)
|
||||
pitch = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[i]))
|
||||
pos = i[0]
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, pos)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, pos)
|
||||
pitch = ultrastar_note_to_midi_note(int(ultrastar_class.pitches[pos]))
|
||||
|
||||
note = pretty_midi.Note(velocity, pitch, start_time, end_time)
|
||||
instrument.notes.append(note)
|
||||
@@ -38,7 +42,7 @@ def convert_ultrastar_to_midi_instrument(ultrastar_class):
|
||||
def instruments_to_midi(instruments, bpm, midi_output):
|
||||
"""Docstring"""
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Creating midi file -> {midi_output}")
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi file -> {midi_output}")
|
||||
|
||||
midi_data = pretty_midi.PrettyMIDI(initial_tempo=bpm)
|
||||
for instrument in instruments:
|
||||
@@ -77,13 +81,14 @@ def find_nearest_index(array, value):
|
||||
|
||||
def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
|
||||
"""Docstring"""
|
||||
print(f"{PRINT_ULTRASTAR} Creating midi notes from pitched data")
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi notes from pitched data")
|
||||
|
||||
midi_notes = []
|
||||
|
||||
for i in enumerate(start_times):
|
||||
start_time = start_times[i]
|
||||
end_time = end_times[i]
|
||||
pos = i[0]
|
||||
start_time = start_times[pos]
|
||||
end_time = end_times[pos]
|
||||
|
||||
note = create_midi_note_from_pitched_data(
|
||||
start_time, end_time, pitched_data
|
||||
@@ -101,6 +106,11 @@ def create_midi_note_from_pitched_data(start_time, end_time, pitched_data):
|
||||
start = find_nearest_index(pitched_data.times, start_time)
|
||||
end = find_nearest_index(pitched_data.times, end_time)
|
||||
|
||||
# fixme: why is end smaller as start?
|
||||
if end < start:
|
||||
print(f"{ULTRASINGER_HEAD} {red_highlighted('Error: end time is smaller as start time. The note is C0 now ...')}")
|
||||
return "C0"
|
||||
|
||||
if start == end:
|
||||
freqs = [pitched_data.frequencies[start]]
|
||||
confs = [pitched_data.confidence[start]]
|
||||
|
||||
+11
-10
@@ -5,10 +5,10 @@ import csv
|
||||
import crepe
|
||||
from scipy.io import wavfile
|
||||
|
||||
from modules.Log import (
|
||||
PRINT_ULTRASTAR,
|
||||
print_blue_highlighted_text,
|
||||
print_red_highlighted_text,
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
|
||||
@@ -16,7 +16,7 @@ from modules.Pitcher.pitched_data import PitchedData
|
||||
def get_pitch_with_crepe_file(filename, step_size, model_capacity):
|
||||
"""Docstring"""
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Pitching with {print_blue_highlighted_text('crepe')} and model {print_blue_highlighted_text(model_capacity)} and {print_red_highlighted_text('cpu')} as worker"
|
||||
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
|
||||
)
|
||||
# Todo: add GPU support by using torchcrepe
|
||||
sample_rate, audio = wavfile.read(filename)
|
||||
@@ -47,14 +47,15 @@ def get_pitch_with_crepe(audio, sample_rate, step_size, model_capacity):
|
||||
return pitched_data
|
||||
|
||||
|
||||
def write_lists_to_csv(time, frequency, confidence, filename):
|
||||
def write_lists_to_csv(times, frequencies, confidences, filename):
|
||||
"""Docstring"""
|
||||
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["time", "frequency", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i in enumerate(time):
|
||||
writer.writerow([time[i], frequency[i], confidence[i]])
|
||||
for i in enumerate(times):
|
||||
pos = i[0]
|
||||
writer.writerow([times[pos], frequencies[pos], confidences[pos]])
|
||||
|
||||
|
||||
def read_data_from_csv(filename):
|
||||
@@ -71,8 +72,8 @@ def read_data_from_csv(filename):
|
||||
def get_frequency_with_high_confidence(freqs, confs, threshold=0.4):
|
||||
"""Docstring"""
|
||||
conf_f = []
|
||||
for i in enumerate(confs):
|
||||
if confs[i] > threshold:
|
||||
for i, conf in enumerate(confs):
|
||||
if conf > threshold:
|
||||
conf_f.append(freqs[i])
|
||||
if not conf_f:
|
||||
conf_f = freqs
|
||||
|
||||
@@ -5,7 +5,7 @@ import wave
|
||||
|
||||
from vosk import KaldiRecognizer, Model
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
# todo: Rename to Transcoder?
|
||||
@@ -15,7 +15,7 @@ def transcribe_with_vosk(audio_filename, model_path):
|
||||
"""Docstring"""
|
||||
# Code from here: https://towardsdatascience.com/speech-recognition-with-timestamps-934ede4234b2
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Transcribing {audio_filename} with vosk and model {model_path}"
|
||||
f"{ULTRASINGER_HEAD} Transcribing {audio_filename} with vosk and model {model_path}"
|
||||
)
|
||||
|
||||
model = Model(model_path)
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
|
||||
import whisperx
|
||||
|
||||
from modules.Log import (
|
||||
PRINT_ULTRASTAR,
|
||||
print_blue_highlighted_text,
|
||||
print_red_highlighted_text,
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
red_highlighted,
|
||||
)
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
@@ -14,7 +14,7 @@ def transcribe_with_whisper(audio_path, model, device="cpu"):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Loading {print_blue_highlighted_text('whisper')} with model {print_blue_highlighted_text(model)} and {print_red_highlighted_text(device)} as worker"
|
||||
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
|
||||
)
|
||||
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
@@ -28,7 +28,7 @@ def transcribe_with_whisper(audio_path, model, device="cpu"):
|
||||
)
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Transcribing {audio_path}")
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
|
||||
language = result["language"]
|
||||
@@ -54,7 +54,7 @@ def transcribe_with_whisper(audio_path, model, device="cpu"):
|
||||
for obj in segment["words"]:
|
||||
if len(obj) < 4:
|
||||
print(
|
||||
f"{print_red_highlighted_text('Error: Skipping Word {}, because of missing timings'.format(obj['word']))}"
|
||||
f"{red_highlighted('Error: Skipping Word {}, because of missing timings'.format(obj['word']))}"
|
||||
)
|
||||
continue
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
|
||||
@@ -4,9 +4,9 @@ import string
|
||||
|
||||
from hyphen import Hyphenator, dictools
|
||||
|
||||
from modules.Log import (
|
||||
PRINT_ULTRASTAR,
|
||||
print_blue_highlighted_text,
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def language_check(language="en"):
|
||||
return None
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Hyphenate using language code: {print_blue_highlighted_text(lang_region)}"
|
||||
f"{ULTRASINGER_HEAD} Hyphenate using language code: {blue_highlighted(lang_region)}"
|
||||
)
|
||||
return lang_region
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import speech_recognition as sr
|
||||
from pydub import AudioSegment
|
||||
from pydub.silence import split_on_silence
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
# todo: Code from here: https://www.thepythoncode.com/article/using-speech-recognition-to-convert-speech-to-text-python
|
||||
|
||||
@@ -96,9 +96,9 @@ def transcribe_audio(audio_file):
|
||||
end_time,
|
||||
)
|
||||
except sr.UnknownValueError:
|
||||
print(f"{PRINT_ULTRASTAR} Could not understand audio")
|
||||
print(f"{ULTRASINGER_HEAD} Could not understand audio")
|
||||
except sr.RequestError as error:
|
||||
print(f"{PRINT_ULTRASTAR} Error with recognizing service; {error}")
|
||||
print(f"{ULTRASINGER_HEAD} Error with recognizing service; {error}")
|
||||
|
||||
|
||||
class SpeechToText:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Docstring"""
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
get_end_time_from_ultrastar,
|
||||
get_start_time_from_ultrastar,
|
||||
@@ -10,7 +10,7 @@ from modules.Ultrastar.ultrastar_txt import UltrastarTxt
|
||||
|
||||
def parse_ultrastar_txt(input_file):
|
||||
"""Docstring"""
|
||||
print(f"{PRINT_ULTRASTAR} Parse ultrastar txt -> {input_file}")
|
||||
print(f"{ULTRASINGER_HEAD} Parse ultrastar txt -> {input_file}")
|
||||
|
||||
with open(input_file, "r", encoding="utf-8") as file:
|
||||
txt = file.readlines()
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
import librosa
|
||||
|
||||
from modules.Log import (
|
||||
PRINT_ULTRASTAR,
|
||||
print_blue_highlighted_text,
|
||||
print_cyan_highlighted_text,
|
||||
print_gold_highlighted_text,
|
||||
print_light_blue_highlighted_text,
|
||||
print_underlined_text,
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted,
|
||||
cyan_highlighted,
|
||||
gold_highlighted,
|
||||
light_blue_highlighted,
|
||||
underlined,
|
||||
)
|
||||
from modules.Midi.midi_creator import create_midi_note_from_pitched_data
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
@@ -78,14 +78,14 @@ def print_score(score):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Total: {print_cyan_highlighted_text(str(score.score))}, notes: {print_blue_highlighted_text(str(score.notes))}, line bonus: {print_light_blue_highlighted_text(str(score.line_bonus))}, golden notes: {print_gold_highlighted_text(str(score.golden))}"
|
||||
f"{ULTRASINGER_HEAD} Total: {cyan_highlighted(str(score.score))}, notes: {blue_highlighted(str(score.notes))}, line bonus: {light_blue_highlighted(str(score.line_bonus))}, golden notes: {gold_highlighted(str(score.golden))}"
|
||||
)
|
||||
|
||||
|
||||
def calculate_score(pitched_data, ultrastar_class):
|
||||
"""Docstring"""
|
||||
|
||||
print(PRINT_ULTRASTAR + " Calculating Ultrastar Points")
|
||||
print(ULTRASINGER_HEAD + " Calculating Ultrastar Points")
|
||||
|
||||
simple_points = Points()
|
||||
accurate_points = Points()
|
||||
@@ -95,14 +95,15 @@ def calculate_score(pitched_data, ultrastar_class):
|
||||
)
|
||||
|
||||
for i in enumerate(ultrastar_class.words):
|
||||
pos = i[0]
|
||||
if ultrastar_class.words == "":
|
||||
continue
|
||||
|
||||
if ultrastar_class.noteType[i] == "F":
|
||||
if ultrastar_class.noteType[pos] == "F":
|
||||
continue
|
||||
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, i)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, i)
|
||||
start_time = get_start_time_from_ultrastar(ultrastar_class, pos)
|
||||
end_time = get_end_time_from_ultrastar(ultrastar_class, pos)
|
||||
duration = end_time - start_time
|
||||
step_size = 0.01 # todo: should be beat length ?
|
||||
parts = int(duration / step_size)
|
||||
@@ -112,7 +113,7 @@ def calculate_score(pitched_data, ultrastar_class):
|
||||
simple_part_line_bonus_points = 0
|
||||
|
||||
ultrastar_midi_note = ultrastar_note_to_midi_note(
|
||||
int(ultrastar_class.pitches[i])
|
||||
int(ultrastar_class.pitches[pos])
|
||||
)
|
||||
ultrastar_note = librosa.midi_to_note(ultrastar_midi_note)
|
||||
|
||||
@@ -126,14 +127,14 @@ def calculate_score(pitched_data, ultrastar_class):
|
||||
if pitch_note[:-1] == ultrastar_note[:-1]:
|
||||
# Ignore octave high
|
||||
simple_points = add_point(
|
||||
ultrastar_class.noteType[i], simple_points
|
||||
ultrastar_class.noteType[pos], simple_points
|
||||
)
|
||||
simple_part_line_bonus_points += 1
|
||||
|
||||
if pitch_note == ultrastar_note:
|
||||
# Octave high must be the same
|
||||
accurate_points = add_point(
|
||||
ultrastar_class.noteType[i], accurate_points
|
||||
ultrastar_class.noteType[pos], accurate_points
|
||||
)
|
||||
accurate_part_line_bonus_points += 1
|
||||
|
||||
@@ -153,11 +154,11 @@ def print_score_calculation(simple_points, accurate_points):
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_underlined_text('Simple (octave high ignored)')} points"
|
||||
f"{ULTRASINGER_HEAD} {underlined('Simple (octave high ignored)')} points"
|
||||
)
|
||||
print_score(simple_points)
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} {print_underlined_text('Accurate (octave high matches)')} points:"
|
||||
f"{ULTRASINGER_HEAD} {underlined('Accurate (octave high matches)')} points:"
|
||||
)
|
||||
print_score(accurate_points)
|
||||
@@ -11,7 +11,7 @@ class UltrastarTxt:
|
||||
bpm = ""
|
||||
cover = None
|
||||
creator = "UltraSinger [GitHub]"
|
||||
fixer = "YOUR NAME"
|
||||
fixer = "YOUR NAME" # Only for UltraStar World Party
|
||||
comment = "UltraSinger [GitHub]"
|
||||
startBeat = []
|
||||
startTimes = []
|
||||
|
||||
@@ -4,7 +4,7 @@ import re
|
||||
|
||||
import langcodes
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Ultrastar.ultrastar_converter import (
|
||||
real_bpm_to_ultrastar_bpm,
|
||||
second_to_beat,
|
||||
@@ -41,7 +41,7 @@ def create_ultrastar_txt_from_automation(
|
||||
"""Docstring"""
|
||||
|
||||
print(
|
||||
f"{PRINT_ULTRASTAR} Creating {ultrastar_file_output} from transcription."
|
||||
f"{ULTRASINGER_HEAD} Creating {ultrastar_file_output} from transcription."
|
||||
)
|
||||
|
||||
real_bpm = real_bpm_to_ultrastar_bpm(bpm)
|
||||
@@ -69,10 +69,10 @@ def create_ultrastar_txt_from_automation(
|
||||
|
||||
# Write the singing part
|
||||
previous_end_beat = 0
|
||||
for i in enumerate(transcribed_data):
|
||||
start_time = (transcribed_data[i].start - gap) * multiplication
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_time = (data.start - gap) * multiplication
|
||||
end_time = (
|
||||
transcribed_data[i].end - transcribed_data[i].start
|
||||
data.end - data.start
|
||||
) * multiplication
|
||||
start_beat = round(second_to_beat(start_time, bpm))
|
||||
duration = round(second_to_beat(end_time, bpm))
|
||||
@@ -91,13 +91,13 @@ def create_ultrastar_txt_from_automation(
|
||||
file.write(str(start_beat) + " ")
|
||||
file.write(str(duration) + " ")
|
||||
file.write(str(note_numbers[i]) + " ")
|
||||
file.write(transcribed_data[i].word)
|
||||
file.write(data.word)
|
||||
file.write("\n")
|
||||
|
||||
# detect silence between words
|
||||
if i < len(transcribed_data) - 1:
|
||||
silence = (
|
||||
transcribed_data[i + 1].start - transcribed_data[i].end
|
||||
transcribed_data[i + 1].start - data.end
|
||||
)
|
||||
else:
|
||||
silence = 0
|
||||
@@ -108,7 +108,7 @@ def create_ultrastar_txt_from_automation(
|
||||
# 'n1' show next at time in real beat
|
||||
file.write("- ")
|
||||
show_next = (
|
||||
second_to_beat(transcribed_data[i].end - gap, bpm)
|
||||
second_to_beat(data.end - gap, bpm)
|
||||
* multiplication
|
||||
)
|
||||
file.write(str(round(show_next)))
|
||||
@@ -155,14 +155,14 @@ def add_score_to_ultrastar_txt(ultrastar_file_output, score):
|
||||
text = file.read()
|
||||
text = text.split("\n")
|
||||
|
||||
for i in enumerate(text):
|
||||
if text[i].startswith("#COMMENT:"):
|
||||
for i, line in enumerate(text):
|
||||
if line.startswith("#COMMENT:"):
|
||||
text[
|
||||
i
|
||||
] = f"{text[i]} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
|
||||
] = f"{line} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
|
||||
break
|
||||
|
||||
if text[i].startswith(("F ", ": ", "* ", "R ", "G ")):
|
||||
if line.startswith(("F ", ": ", "* ", "R ", "G ")):
|
||||
text.insert(
|
||||
i,
|
||||
f"#COMMENT: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Colors for the console"""
|
||||
|
||||
ULTRASINGER_HEAD = "\033[92m[UltraSinger]\033[0m"
|
||||
|
||||
|
||||
def blue_highlighted(text: str) -> str:
|
||||
"""Returns a blue highlighted text"""
|
||||
return f"{Bcolors.blue}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def gold_highlighted(text: str) -> str:
|
||||
"""Returns a gold highlighted text"""
|
||||
return f"{Bcolors.gold}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def light_blue_highlighted(text: str) -> str:
|
||||
"""Returns a light blue highlighted text"""
|
||||
return f"{Bcolors.light_blue}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def underlined(text: str) -> str:
|
||||
"""Returns an underlined text"""
|
||||
return f"{Bcolors.underline}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def red_highlighted(text: str) -> str:
|
||||
"""Returns a red highlighted text"""
|
||||
return f"{Bcolors.red}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
def cyan_highlighted(text: str) -> str:
|
||||
"""Returns a cyan highlighted text"""
|
||||
return f"{Bcolors.cyan}{text}{Bcolors.endc}"
|
||||
|
||||
|
||||
class Bcolors:
|
||||
"""Colors for the console"""
|
||||
|
||||
blue = "\033[94m"
|
||||
red = "\033[91m"
|
||||
light_blue = "\033[96m"
|
||||
cyan = "\033[36m"
|
||||
gold = "\033[93m"
|
||||
underline = "\033[4m"
|
||||
endc = "\033[0m"
|
||||
@@ -0,0 +1,24 @@
|
||||
"""CSV export module"""
|
||||
|
||||
import csv
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def export_transcribed_data_to_csv(transcribed_data: [], filename: str) -> None:
|
||||
"""Export transcribed data to csv"""
|
||||
print(f"{ULTRASINGER_HEAD} Exporting transcribed data to CSV")
|
||||
|
||||
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header = ["word", "start", "end", "confidence"]
|
||||
writer.writerow(header)
|
||||
for i, data in enumerate(transcribed_data):
|
||||
writer.writerow(
|
||||
[
|
||||
data.word,
|
||||
data.start,
|
||||
data.end,
|
||||
data.conf,
|
||||
]
|
||||
)
|
||||
@@ -3,12 +3,12 @@
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from modules.Log import PRINT_ULTRASTAR
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def create_folder(folder_name):
|
||||
"""Docstring"""
|
||||
print(f"{PRINT_ULTRASTAR} Creating output folder. -> {folder_name}")
|
||||
print(f"{ULTRASINGER_HEAD} Creating output folder. -> {folder_name}")
|
||||
is_exist = os.path.exists(folder_name)
|
||||
if not is_exist:
|
||||
os.makedirs(folder_name)
|
||||
|
||||
+5
-3
@@ -1,11 +1,13 @@
|
||||
crepe==0.0.13
|
||||
librosa==0.9.2
|
||||
pretty_midi==0.2.9
|
||||
pretty_midi~=0.2.10
|
||||
pydub==0.25.1
|
||||
Levenshtein==0.20.9
|
||||
scipy==1.10.0
|
||||
Levenshtein~=0.21.0
|
||||
scipy~=1.10.1
|
||||
vosk==0.3.45
|
||||
tensorflow~=2.12.0
|
||||
#tensorflow-directml-plugin~=0.4.0.dev230202
|
||||
#cudatoolkit==11.8.0
|
||||
matplotlib~=3.7.1
|
||||
yt_dlp~=2023.3.4
|
||||
numpy~=1.23.5
|
||||
|
||||
Reference in new issue
Block a user