Merge branch 'main' into additional_song_info_musicbrainz

# Conflicts:
#	requirements.txt
#	src/UltraSinger.py
#	src/modules/musicbrainz_client.py
This commit is contained in:
Vadim committed 2023-06-18 16:06:38 +02:00
commit e5d4ce6e60
41 files changed
+916 -568

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
name: Check Requirements
on:
push:
paths:
- "requirements.txt"
pull_request:
paths:
- "requirements.txt"
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
matrix:
include:
- os: ubuntu-latest
TARGET: Linux
- os: macos-latest
TARGET: macOS
- os: windows-latest
TARGET: Windows
python-version: [ "3.10" ]
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Install gpu dependencies
if: matrix.TARGET != 'macOS'
run: pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
+31
View File
@@ -0,0 +1,31 @@
name: Pylint
#on: [push]
on:
push:
paths:
- "none.txt"
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pylint
- name: Analysing the code with pylint
run: |
pylint $(git ls-files '*.py')
+28
View File
@@ -0,0 +1,28 @@
name: Pytest
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.10" ]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
#pip install -e .
- name: Run pytest
run: python -m pytest --import-mode=append pytest/
+49 -10
View File
@@ -1,8 +1,12 @@
[![Discord](https://img.shields.io/discord/1048892118732656731?logo=discord)](https://discord.gg/7EqhhjFd)
![Status](https://img.shields.io/badge/status-development-yellow)
![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/rakuri255/UltraSinger/main.yml)
[![GitHub](https://img.shields.io/github/license/rakuri255/UltraSinger)](https://github.com/rakuri255/UltraSinger/blob/main/LICENSE)
[![CodeFactor](https://www.codefactor.io/repository/github/rakuri255/ultrasinger/badge)](https://www.codefactor.io/repository/github/rakuri255/ultrasinger)
[![Check Requirements](https://github.com/rakuri255/UltraSinger/actions/workflows/main.yml/badge.svg)](https://github.com/rakuri255/UltraSinger/actions/workflows/main.yml)
[![Pytest](https://github.com/rakuri255/UltraSinger/actions/workflows/pytest.yml/badge.svg)](https://github.com/rakuri255/UltraSinger/actions/workflows/pytest.yml)
# UltraSinger
> ⚠️ _This project is still under development!_
@@ -46,7 +50,7 @@ This will help me alot to keep this project alive and improve it.
For more information about Python environments look [here](https://code.visualstudio.com/docs/python/environments#_global-virtual-or-conda-environments).
As copy:
Installation As copy:
```commandline
py -m venv .venv
@@ -56,6 +60,27 @@ pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu11
```
Run UltraSinger:
* Activate the environment with `.venv\Scripts\activate`. (You dont need this if you already activated it, or just installed with the step above)
* Navigate to src folder `cd src`
* Start UltraSinger `py UltraSinger.py`
Start environment only once:
```commandline
.venv\Scripts\activate
cd src
```
Start UltraSinger:
```commandline
py UltraSinger.py
```
## How to use the App
_Not all options working now!_
@@ -87,19 +112,22 @@ _Not all options working now!_
[transcription]
# Default is whisper
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--vosk Needs model
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--align_model Use other languages model for Whisper provided from huggingface.co
--vosk Needs model
[pitcher]
# Default is crepe
--crepe tiny|small|medium|large|full >> ((default) is full)
[extra]
--hyphenation True|False >> ((default) is True)
--disable_separation True|False >> ((default) is False)
--disable_karaoke True|False >> ((default) is False)
--create_audio_chunks True|False >> ((default) is False)
[pitcher]
# Default is crepe
--crepe tiny|small|medium|large|full >> ((default) is full)
--plot True|False >> ((default) is False)
```
For standard use, you only need to use [opt]. All other options are optional.
@@ -137,12 +165,23 @@ Also keep in mind that while a larger model is more accurate, it also takes long
#### Whisper
For the first test run, use the `tiny`, to be accurate use the `large-v2` model
For the first test run, use the `tiny`, to be accurate use the `large-v2` model.
```commandline
-i XYZ --whisper large
```
##### Whisper languages
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
If the language is not in this list, you need to find a phoneme-based ASR model from
[huggingface model hub](https://huggingface.co). It will download automatically.
Example for romanian:
```commandline
-i XYZ --align_model "gigant/romanian-wav2vec2"
```
#### Vosk
If you want to use `Vosk` than you need the model. It is not included. You can download it here [Link](https://alphacephei.com/vosk/models).
@@ -221,7 +260,7 @@ But you can force cpu usage with the extra options `--force_whisper_cpu` and `--
With an GPU you can speed up the process and also the quality of the transcription and pitching is better.
You need a cuda device for this to work.
If you use MAC than sorry, there is no cuda device for MAC machinges.
If you use MAC than sorry, there is no cuda device for MAC machines.
#### Info
@@ -1,21 +0,0 @@
"""Docstring"""
import torch
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted
def get_available_device():
"""Docstring"""
isCuda = torch.cuda.is_available()
if not isCuda:
print(
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
)
return "cpu"
else:
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
return "cuda"
-9
View File
@@ -1,9 +0,0 @@
"""Docstring"""
class PitchedData:
"""Docstring"""
times = []
frequencies = []
confidence = []
-84
View File
@@ -1,84 +0,0 @@
"""Docstring"""
import csv
import crepe
from scipy.io import wavfile
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.Pitcher.pitched_data import PitchedData
def get_pitch_with_crepe_file(filename, step_size, model_capacity):
"""Docstring"""
print(
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
)
# Todo: add GPU support by using torchcrepe
sample_rate, audio = wavfile.read(filename)
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
def get_pitch_with_crepe(audio, sample_rate, step_size, model_capacity):
"""Docstring"""
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
def write_lists_to_csv(times, frequencies, confidences, filename):
"""Docstring"""
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile)
header = ["time", "frequency", "confidence"]
writer.writerow(header)
for i in enumerate(times):
pos = i[0]
writer.writerow([times[pos], frequencies[pos], confidences[pos]])
def read_data_from_csv(filename):
"""Docstring"""
csv_data = []
with open(filename, "r", encoding="utf-8") as csv_file:
csv_reader = csv.reader(csv_file)
for line in csv_reader:
csv_data.append(line)
headless_data = csv_data[1:]
return headless_data
def get_frequency_with_high_confidence(freqs, confs, threshold=0.4):
"""Docstring"""
conf_f = []
for i, conf in enumerate(confs):
if conf > threshold:
conf_f.append(freqs[i])
if not conf_f:
conf_f = freqs
return conf_f
class Pitcher:
"""Docstring"""
-56
View File
@@ -1,56 +0,0 @@
"""Docstring"""
import string
from hyphen import Hyphenator, dictools
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
)
def language_check(language="en"):
"""Docstring"""
dict_langs = dictools.LANGUAGES
lang_region = None
for dict_lang in dict_langs:
if dict_lang.startswith(language):
lang_region = dict_lang
break
if lang_region is None:
return None
print(
f"{ULTRASINGER_HEAD} Hyphenate using language code: {blue_highlighted(lang_region)}"
)
return lang_region
def contains_punctuation(word):
"""Docstring"""
return any(elem in word for elem in string.punctuation)
def hyphenation(word, lang_region):
"""Docstring"""
if contains_punctuation(word):
return None
hyphenator = Hyphenator(lang_region)
syllabus = hyphenator.syllables(word)
length = len(syllabus)
if length > 1:
hyphen = []
for i in range(length):
hyphen.append(syllabus[i])
else:
hyphen = None
return hyphen
-22
View File
@@ -1,22 +0,0 @@
"""Docstring"""
class UltrastarTxt:
"""Docstring"""
artist = ""
title = ""
mp3 = ""
gap = ""
bpm = ""
cover = None
creator = "UltraSinger [GitHub]"
fixer = "YOUR NAME" # Only for UltraStar World Party
comment = "UltraSinger [GitHub]"
startBeat = []
startTimes = []
endTimes = []
durations = []
pitches = []
words = []
noteType = []
-24
View File
@@ -1,24 +0,0 @@
"""CSV export module"""
import csv
from modules.console_colors import ULTRASINGER_HEAD
def export_transcribed_data_to_csv(transcribed_data: [], filename: str) -> None:
"""Export transcribed data to csv"""
print(f"{ULTRASINGER_HEAD} Exporting transcribed data to CSV")
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile)
header = ["word", "start", "end", "confidence"]
writer.writerow(header)
for i, data in enumerate(transcribed_data):
writer.writerow(
[
data.word,
data.start,
data.end,
data.conf,
]
)
-44
View File
@@ -1,44 +0,0 @@
"""Docstring"""
import os
import shutil
from modules.console_colors import ULTRASINGER_HEAD
def create_folder(folder_name):
"""Docstring"""
print(f"{ULTRASINGER_HEAD} Creating output folder. -> {folder_name}")
is_exist = os.path.exists(folder_name)
if not is_exist:
os.makedirs(folder_name)
def move(original, target):
"""Docstring"""
shutil.move(original, target)
def copy(original, target):
"""Docstring"""
shutil.copy(original, target)
def current_executor_path():
"""Docstring"""
return os.getcwd()
def path_join(path1, path2):
"""Docstring"""
return os.path.join(path1, path2)
def check_file_exists(file_path):
"""Docstring"""
return os.path.isfile(file_path)
def check_if_folder_exists(song_output):
"""Docstring"""
return os.path.isdir(song_output)
@@ -0,0 +1,16 @@
"""Tests for hyphenation.py"""
from src.modules.Speech_Recognition.hyphenation import hyphenation
from hyphen import Hyphenator
def test_hypenation():
"""Test case for hyphenation function."""
assert hyphenation("begegnen", Hyphenator("de_DE")) == ["be", "geg", "nen"]
assert hyphenation(".b,e~g'eg*nen, ", Hyphenator("de_DE")) == [".b,e", "~g'eg", "*nen, "]
assert hyphenation("Abend, ", Hyphenator("de_AT")) == None
assert hyphenation("Abend.", Hyphenator("de_AT")) == None
@@ -0,0 +1,24 @@
"""Tests for ultrastar_converter.py"""
from src.modules.Ultrastar.ultrastar_converter import real_bpm_to_ultrastar_bpm
def test_real_bpm_to_ultrastar_bpm():
"""Test case for real_bpm_to_ultrastar_bpm function."""
# Real BPM is a multiple of 4
assert real_bpm_to_ultrastar_bpm(120) == 30
# Real BPM is not a multiple of 4
assert real_bpm_to_ultrastar_bpm(130) == 32.5
# Real BPM is zero
assert real_bpm_to_ultrastar_bpm(0) == 0
# Real BPM is a decimal number
assert real_bpm_to_ultrastar_bpm(123.45) == 30.8625
# Real BPM is a large number
assert real_bpm_to_ultrastar_bpm(1000000) == 250000
# Real BPM is a small number
assert real_bpm_to_ultrastar_bpm(0.001) == 0.00025
+2 -2
View File
@@ -4,7 +4,7 @@ pretty_midi~=0.2.10
pydub==0.25.1
Levenshtein~=0.21.0
scipy~=1.10.1
vosk==0.3.45
vosk==0.3.44
tensorflow~=2.12.0
#tensorflow-directml-plugin~=0.4.0.dev230202
#cudatoolkit==11.8.0
@@ -21,5 +21,5 @@ Pillow~=9.5.0
isort==5.12
black==23.3
pylint==2.17
pytest~=7.3.1
musicbrainzngs~=0.7.1
+1 -1
View File
@@ -17,6 +17,7 @@ class Settings:
transcriber = "whisper" # whisper|vosk
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
# Vosk
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
@@ -29,4 +30,3 @@ class Settings:
device = 'cpu' # cpu|cuda
force_whisper_cpu = False
force_separation_cpu = False
+128 -153
View File
@@ -8,81 +8,64 @@ from time import sleep
import Levenshtein
import librosa
from matplotlib import pyplot as plt
from tqdm import tqdm
from modules import os_helper
from modules.Audio.denoise import ffmpeg_reduce_noise
from modules.Audio.separation import separate_audio
from modules.Audio.vocal_chunks import (
from src.modules import os_helper
from src.modules.Audio.denoise import ffmpeg_reduce_noise
from src.modules.Audio.separation import separate_audio
from src.modules.Audio.vocal_chunks import (
export_chunks_from_transcribed_data,
export_chunks_from_ultrastar_data,
)
from modules.Audio.silence_processing import remove_silence_from_transcribtion_data
from modules.csv_export import export_transcribed_data_to_csv
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
from modules.Audio.youtube import (
from src.modules.Audio.silence_processing import remove_silence_from_transcribtion_data
from src.modules.csv_handler import export_transcribed_data_to_csv
from src.modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
from src.modules.Audio.youtube import (
download_youtube_audio,
download_youtube_thumbnail,
download_youtube_video,
get_youtube_title,
)
from modules.DeviceDetection.device_detection import get_available_device
from modules.console_colors import (
from src.modules.DeviceDetection.device_detection import get_available_device
from src.modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
gold_highlighted,
light_blue_highlighted,
red_highlighted,
)
from modules.Midi import midi_creator
from modules.Midi.midi_creator import (
from src.modules.Midi import midi_creator
from src.modules.Midi.midi_creator import (
convert_frequencies_to_notes,
create_midi_notes_from_pitched_data,
most_frequent,
)
from modules.Pitcher.pitcher import (
from src.modules.Pitcher.pitcher import (
get_frequency_with_high_confidence,
get_pitch_with_crepe_file,
)
from modules.Speech_Recognition.hyphenation import hyphenation, language_check
from modules.Speech_Recognition.Vosk import transcribe_with_vosk
from modules.Speech_Recognition.Whisper import transcribe_with_whisper
from modules.Ultrastar import (
ultrastar_converter,
ultrastar_parser,
ultrastar_score_calculator,
ultrastar_writer,
)
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
from src.modules.Pitcher.pitched_data import PitchedData
from src.modules.Speech_Recognition.hyphenation import hyphenation, language_check, create_hyphenator
from src.modules.Speech_Recognition.Vosk import transcribe_with_vosk
from src.modules.Speech_Recognition.Whisper import transcribe_with_whisper
from src.modules.Ultrastar import ultrastar_score_calculator, ultrastar_writer, ultrastar_converter, ultrastar_parser
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
from Settings import Settings
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
from src.modules.plot import plot
from modules.musicbrainz_client import get_music_infos
settings = Settings()
def get_confidence(pitched_data, threshold):
"""Docstring"""
# todo: replace get_frequency_with_high_conf from pitcher
conf_t = []
conf_f = []
conf_c = []
for i in enumerate(pitched_data.times):
pos = i[0]
if pitched_data.confidence[i] > threshold:
conf_t.append(pitched_data.times[pos])
conf_f.append(pitched_data.frequencies[pos])
conf_c.append(pitched_data.confidence[pos])
return conf_t, conf_f, conf_c
def convert_ultrastar_note_numbers(midi_notes):
"""Docstring"""
def convert_midi_notes_to_ultrastar_notes(midi_notes: list[str]) -> list[int]:
"""Convert midi notes to ultrastar notes"""
print(f"{ULTRASINGER_HEAD} Creating Ultrastar notes from midi data")
ultrastar_note_numbers = []
for i, note in enumerate(midi_notes):
note_number_librosa = librosa.note_to_midi(note)
for i in enumerate(midi_notes):
pos = i[0]
note_number_librosa = librosa.note_to_midi(midi_notes[pos])
pitch = ultrastar_converter.midi_note_to_ultrastar_note(
note_number_librosa
)
@@ -94,8 +77,8 @@ def convert_ultrastar_note_numbers(midi_notes):
return ultrastar_note_numbers
def pitch_each_chunk_with_crepe(directory):
"""Docstring"""
def pitch_each_chunk_with_crepe(directory: str) -> list[str]:
"""Pitch each chunk with crepe and return midi notes"""
print(
f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}"
)
@@ -124,8 +107,8 @@ def pitch_each_chunk_with_crepe(directory):
return midi_notes
def add_hyphen_to_data(transcribed_data, hyphen_words):
"""Docstring"""
def add_hyphen_to_data(transcribed_data: list[TranscribedData], hyphen_words: list[list[str]]):
"""Add hyphen to transcribed data return new data list"""
new_data = []
for i, data in enumerate(transcribed_data):
@@ -136,14 +119,15 @@ def add_hyphen_to_data(transcribed_data, hyphen_words):
chunk_duration = chunk_duration / (len(hyphen_words[i]))
next_start = data.start
for j, hyphens in enumerate(hyphen_words[i]):
for j in enumerate(hyphen_words[i]):
hyphenated_word_index = j[0]
dup = copy.copy(data)
dup.start = next_start
next_start = data.end - chunk_duration * (
len(hyphens) - 1 - j
len(hyphen_words[i]) - 1 - hyphenated_word_index
)
dup.end = next_start
dup.word = hyphen_words[i][j]
dup.word = hyphen_words[i][hyphenated_word_index]
dup.is_hyphen = True
new_data.append(dup)
@@ -151,7 +135,7 @@ def add_hyphen_to_data(transcribed_data, hyphen_words):
def get_bpm_from_data(data, sampling_rate):
"""Docstring"""
"""Get real bpm from audio data"""
onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate)
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
@@ -161,8 +145,8 @@ def get_bpm_from_data(data, sampling_rate):
return wav_tempo[0]
def get_bpm_from_file(wav_file):
"""Docstring"""
def get_bpm_from_file(wav_file: str):
"""Get real bpm from audio file"""
data, sampling_rate = librosa.load(wav_file, sr=None)
return get_bpm_from_data(data, sampling_rate)
@@ -185,8 +169,8 @@ def correct_words(recognized_words, word_list_file):
return recognized_words
def print_help():
"""Docstring"""
def print_help() -> None:
"""Print help text"""
help_string = """
UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]
@@ -227,31 +211,8 @@ def print_help():
print(help_string)
def plot(input_file, vosk_transcribed_data, midi_notes):
"""Docstring"""
pitched_data = get_pitch_with_crepe_file(
input_file, settings.crepe_step_size, settings.crepe_model_capacity
)
conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4)
plt.ylim(0, 600)
plt.xlim(0, 50)
plt.plot(conf_t, conf_f, linewidth=0.1)
plt.savefig(os.path.join("test", "crepe_0.4.png"))
for i, data in enumerate(vosk_transcribed_data):
note_frequency = librosa.note_to_hz(midi_notes[i])
plt.plot(
[data.start, data.end],
[note_frequency, note_frequency],
linewidth=1,
alpha=0.5,
)
plt.savefig(os.path.join("test", "pit.png"), dpi=2000)
def remove_unecessary_punctuations(transcribed_data):
"""Docstring"""
def remove_unecessary_punctuations(transcribed_data: list[TranscribedData]) -> None:
"""Remove unecessary punctuations from transcribed data"""
punctuation = ".,"
for i, data in enumerate(transcribed_data):
data.word = data.word.translate(
@@ -259,8 +220,8 @@ def remove_unecessary_punctuations(transcribed_data):
)
def hyphenate_each_word(language, transcribed_data):
"""Docstring"""
def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData]) -> list[list[str]] | None:
"""Hyphenate each word in the transcribed data."""
hyphenated_word = []
lang_region = language_check(language)
if lang_region is None:
@@ -270,15 +231,17 @@ def hyphenate_each_word(language, transcribed_data):
return None
sleep(0.1)
for i, data in tqdm(enumerate(transcribed_data)):
hyphenator = create_hyphenator(lang_region)
for i in tqdm(enumerate(transcribed_data)):
pos = i[0]
hyphenated_word.append(
hyphenation(data.word, lang_region)
hyphenation(transcribed_data[pos].word, hyphenator)
)
return hyphenated_word
def print_support():
"""Docstring"""
def print_support() -> None:
"""Print support text"""
print()
print(
f"{ULTRASINGER_HEAD} {gold_highlighted('Do you like UltraSinger? And want it to be even better? Then help with your')} {light_blue_highlighted('support')}{gold_highlighted('!')}"
@@ -291,8 +254,8 @@ def print_support():
)
def run():
"""Docstring"""
def run() -> None:
"""The processing function of this program"""
is_audio = ".txt" not in settings.input_file_path
ultrastar_class = None
real_bpm = None
@@ -325,7 +288,7 @@ def run():
basename_without_ext,
song_output,
ultrastar_audio_input_path,
) = setup_audio_input_file()
) = infos_from_audio_input_file()
cache_path = os.path.join(song_output, "cache")
settings.mono_audio_path = os.path.join(
@@ -348,7 +311,7 @@ def run():
transcribed_data = None
language = None
if is_audio:
language, transcribed_data = transcribe_audio(transcribed_data)
language, transcribed_data = transcribe_audio()
remove_unecessary_punctuations(transcribed_data)
transcribed_data = remove_silence_from_transcribtion_data(
settings.mono_audio_path, transcribed_data
@@ -381,8 +344,8 @@ def run():
)
# Create plot
if is_audio and settings.create_plot:
plot(ultrastar_audio_input_path, transcribed_data, midi_notes)
if settings.create_plot:
plot(pitched_data, transcribed_data, midi_notes, song_output)
# Write Ultrastar txt
if is_audio:
@@ -418,8 +381,8 @@ def run():
print_support()
def get_unused_song_output_dir(path):
"""Docstring"""
def get_unused_song_output_dir(path: str) -> str:
"""Get an unused song output dir"""
# check if dir exists and add (i) if it does
i = 1
if os_helper.check_if_folder_exists(path):
@@ -438,12 +401,12 @@ def get_unused_song_output_dir(path):
return path
def transcribe_audio(transcribed_data):
"""Docstring"""
def transcribe_audio() -> (str, list[TranscribedData]):
"""Transcribe audio with AI"""
if settings.transcriber == "whisper":
device = "cpu" if settings.force_whisper_cpu else settings.device
transcribed_data, language = transcribe_with_whisper(
settings.mono_audio_path, settings.whisper_model, device)
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
else: # vosk
transcribed_data = transcribe_with_vosk(
settings.mono_audio_path, settings.vosk_model_path
@@ -454,9 +417,9 @@ def transcribe_audio(transcribed_data):
def separate_vocal_from_audio(
basename_without_ext, cache_path, ultrastar_audio_input_path
):
"""Docstring"""
basename_without_ext: str, cache_path: str, ultrastar_audio_input_path: str
) -> str:
"""Separate vocal from audio"""
audio_separation_path = os.path.join(
cache_path, "separated", "htdemucs", basename_without_ext
)
@@ -474,9 +437,9 @@ def separate_vocal_from_audio(
def calculate_score_points(
is_audio, pitched_data, ultrastar_class, ultrastar_file_output
is_audio: bool, pitched_data: PitchedData, ultrastar_class: UltrastarTxtValue, ultrastar_file_output: str
):
"""Docstring"""
"""Calculate score points"""
if is_audio:
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
ultrastar_file_output
@@ -522,9 +485,9 @@ def calculate_score_points(
def create_ultrastar_txt_from_ultrastar_data(
song_output, ultrastar_class, ultrastar_note_numbers
):
"""Docstring"""
song_output: str, ultrastar_class: UltrastarTxtValue, ultrastar_note_numbers: list[int]
) -> str:
"""Create Ultrastar txt from Ultrastar data"""
output_repitched_ultrastar = os.path.join(
song_output, ultrastar_class.title + ".txt"
)
@@ -537,16 +500,16 @@ def create_ultrastar_txt_from_ultrastar_data(
def create_ultrastar_txt_from_automation(
audio_separation_path,
basename_without_ext,
song_output,
transcribed_data,
ultrastar_audio_input_path,
ultrastar_note_numbers,
language,
audio_separation_path: str,
basename_without_ext: str,
song_output: str,
transcribed_data: list[TranscribedData],
ultrastar_audio_input_path: str,
ultrastar_note_numbers: list[int],
language: str,
):
"""Docstring"""
ultrastar_header = UltrastarTxt()
"""Create Ultrastar txt from automation"""
ultrastar_header = UltrastarTxtValue()
ultrastar_header.title = basename_without_ext
ultrastar_header.artist = basename_without_ext
ultrastar_header.mp3 = basename_without_ext + ".mp3"
@@ -589,8 +552,8 @@ def create_ultrastar_txt_from_automation(
return real_bpm, ultrastar_file_output
def setup_audio_input_file():
"""Docstring"""
def infos_from_audio_input_file() -> tuple[str, str, str]:
"""Infos from audio input file"""
basename = os.path.basename(settings.input_file_path)
basename_without_ext = os.path.splitext(basename)[0]
song_output = os.path.join(settings.output_file_path, basename_without_ext)
@@ -605,7 +568,7 @@ FILENAME_REPLACEMENTS = (('?:"', ""), ("<", "("), (">", ")"), ("/\\|*", "-"))
def sanitize_filename(fname: str) -> str:
"""Docstring"""
"""Sanitize filename"""
for old, new in FILENAME_REPLACEMENTS:
for char in old:
fname = fname.replace(char, new)
@@ -614,8 +577,8 @@ def sanitize_filename(fname: str) -> str:
return fname
def download_from_youtube():
"""Docstring"""
def download_from_youtube() -> tuple[str, str, str]:
"""Download from YouTube"""
title = get_youtube_title(settings.input_file_path)
basename_without_ext = sanitize_filename(title)
basename = basename_without_ext + ".mp3"
@@ -635,8 +598,8 @@ def download_from_youtube():
return basename_without_ext, song_output, ultrastar_audio_input_path
def parse_ultrastar_txt():
"""Docstring"""
def parse_ultrastar_txt() -> tuple[str, float, str, str, UltrastarTxtValue]:
"""Parse Ultrastar txt"""
ultrastar_class = ultrastar_parser.parse_ultrastar_txt(
settings.input_file_path
)
@@ -662,8 +625,8 @@ def parse_ultrastar_txt():
)
def create_midi_file(is_audio, real_bpm, song_output, ultrastar_class):
"""Docstring"""
def create_midi_file(is_audio: bool, real_bpm: float, song_output: str, ultrastar_class: UltrastarTxtValue) -> None:
"""Create midi file"""
print(
f"{ULTRASINGER_HEAD} Creating Midi with {blue_highlighted('pretty_midi')}"
)
@@ -685,8 +648,9 @@ def create_midi_file(is_audio, real_bpm, song_output, ultrastar_class):
)
def pitch_audio(is_audio, transcribed_data, ultrastar_class):
"""Docstring"""
def pitch_audio(is_audio: bool, transcribed_data: list[TranscribedData], ultrastar_class: UltrastarTxtValue) -> tuple[
list[str], PitchedData, list[int]]:
"""Pitch audio"""
# todo: chunk pitching as option?
# midi_notes = pitch_each_chunk_with_crepe(chunk_folder_name)
pitched_data = get_pitch_with_crepe_file(
@@ -708,18 +672,18 @@ def pitch_audio(is_audio, transcribed_data, ultrastar_class):
midi_notes = create_midi_notes_from_pitched_data(
ultrastar_class.startTimes, ultrastar_class.endTimes, pitched_data
)
ultrastar_note_numbers = convert_ultrastar_note_numbers(midi_notes)
ultrastar_note_numbers = convert_midi_notes_to_ultrastar_notes(midi_notes)
return midi_notes, pitched_data, ultrastar_note_numbers
def create_audio_chunks(
cache_path,
is_audio,
transcribed_data,
ultrastar_audio_input_path,
ultrastar_class,
):
"""Docstring"""
cache_path: str,
is_audio: bool,
transcribed_data: list[TranscribedData],
ultrastar_audio_input_path: str,
ultrastar_class: UltrastarTxtValue
) -> None:
"""Create audio chunks"""
audio_chunks_path = os.path.join(
cache_path, settings.audio_chunk_folder_name
)
@@ -736,8 +700,8 @@ def create_audio_chunks(
)
def denoise_vocal_audio(basename_without_ext, cache_path):
"""Docstring"""
def denoise_vocal_audio(basename_without_ext: str, cache_path: str) -> None:
"""Denoise vocal audio"""
denoised_path = os.path.join(
cache_path, basename_without_ext + "_denoised.wav"
)
@@ -745,30 +709,17 @@ def denoise_vocal_audio(basename_without_ext, cache_path):
settings.mono_audio_path = denoised_path
def main(argv):
"""Docstring"""
def main(argv: list[str]) -> None:
"""Main function"""
init_settings(argv)
run()
# todo: cleanup
sys.exit()
def init_settings(argv):
"""Docstring"""
short = "hi:o:amv:"
long = [
"ifile=",
"ofile=",
"crepe=",
"vosk=",
"whisper=",
"hyphenation=",
"disable_separation=",
"disable_karaoke=",
"create_audio_chunks=",
"force_whisper_cpu=",
"force_separation_cpu="
]
def init_settings(argv: list[str]) -> None:
"""Init settings"""
long, short = arg_options()
opts, args = getopt.getopt(argv, short, long)
if len(opts) == 0:
print_help()
@@ -784,11 +735,15 @@ def init_settings(argv):
elif opt in ("--whisper"):
settings.transcriber = "whisper"
settings.whisper_model = arg
elif opt in ("--align_model"):
settings.whisper_align_model = arg
elif opt in ("--vosk"):
settings.transcriber = "vosk"
settings.vosk_model_path = arg
elif opt in ("--crepe"):
settings.crepe_model_capacity = arg
elif opt in ("--plot"):
settings.create_plot = arg
elif opt in ("--hyphenation"):
settings.hyphenation = arg
elif opt in ("--disable_separation"):
@@ -810,5 +765,25 @@ def init_settings(argv):
settings.device = get_available_device()
def arg_options():
short = "hi:o:amv:"
long = [
"ifile=",
"ofile=",
"crepe=",
"vosk=",
"whisper=",
"align_model=",
"plot=",
"hyphenation=",
"disable_separation=",
"disable_karaoke=",
"create_audio_chunks=",
"force_whisper_cpu=",
"force_separation_cpu="
]
return long, short
if __name__ == "__main__":
main(sys.argv[1:])
@@ -2,7 +2,7 @@
from pydub import AudioSegment
from modules.console_colors import ULTRASINGER_HEAD
from src.modules.console_colors import ULTRASINGER_HEAD
def convert_audio_to_mono_wav(input_file: str, output_file: str) -> None:
@@ -2,7 +2,7 @@
import ffmpeg
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
from src.modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
def ffmpeg_reduce_noise(input_file_path: str, output_file: str) -> None:
@@ -2,12 +2,12 @@
import subprocess
from modules.console_colors import (
from src.modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.os_helper import current_executor_path, move, path_join
from src.modules.os_helper import current_executor_path, move, path_join
def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None:
@@ -2,10 +2,11 @@
import librosa
from modules.console_colors import ULTRASINGER_HEAD
from src.modules.console_colors import ULTRASINGER_HEAD
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: []) -> []:
def remove_silence_from_transcribtion_data(audio_path: str, transcribed_data: list[TranscribedData]) -> list[TranscribedData]:
"""Remove silence from given transcribtion data"""
print(
@@ -4,13 +4,13 @@ import os
import re
import wave
from modules.console_colors import ULTRASINGER_HEAD
from modules.os_helper import create_folder
from modules.Ultrastar.ultrastar_converter import (
from src.modules.console_colors import ULTRASINGER_HEAD
from src.modules.os_helper import create_folder
from src.modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
)
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
class AudioManipulation:
@@ -41,7 +41,7 @@ def export_chunks_from_transcribed_data(
def export_chunks_from_ultrastar_data(
audio_filename: str, ultrastar_data: UltrastarTxt, folder_name: str
audio_filename: str, ultrastar_data: UltrastarTxtValue, folder_name: str
) -> None:
"""Export ultrastar data as vocal chunks wav files"""
print(f"{ULTRASINGER_HEAD} Export Ultrastar data as vocal chunks wav files")
@@ -6,7 +6,7 @@ import os
import yt_dlp
from PIL import Image
from modules.console_colors import ULTRASINGER_HEAD
from src.modules.console_colors import ULTRASINGER_HEAD
def get_youtube_title(url: str) -> str:
@@ -0,0 +1,19 @@
"""Device detection module."""
import torch
from src.modules.console_colors import ULTRASINGER_HEAD, red_highlighted
def get_available_device() -> str:
"""Get available worker device e.g cuda or cpu"""
is_cuda = torch.cuda.is_available()
if not is_cuda:
print(
f"{ULTRASINGER_HEAD} There are no {red_highlighted('cuda')} devices available. -> Using {red_highlighted('cpu')}."
)
return "cpu"
print(f"{ULTRASINGER_HEAD} Using {red_highlighted('cuda')} GPU.")
return "cuda"
@@ -1,4 +1,4 @@
"""Docstring"""
"""Midi creator module"""
import math
from collections import Counter
@@ -7,20 +7,22 @@ import librosa
import numpy as np
import pretty_midi
from modules.Pitcher.pitcher import get_frequency_with_high_confidence
from modules.Ultrastar.ultrastar_converter import (
from src.modules.Pitcher.pitcher import get_frequency_with_high_confidence
from src.modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
ultrastar_note_to_midi_note,
)
from modules.console_colors import (
from src.modules.console_colors import (
ULTRASINGER_HEAD,
red_highlighted,
)
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
from src.modules.Pitcher.pitched_data import PitchedData
def convert_ultrastar_to_midi_instrument(ultrastar_class):
"""Docstring"""
def convert_ultrastar_to_midi_instrument(ultrastar_class: UltrastarTxtValue) -> object:
"""Converts an Ultrastar data to a midi instrument"""
print(f"{ULTRASINGER_HEAD} Creating midi instrument from Ultrastar txt")
@@ -39,8 +41,8 @@ def convert_ultrastar_to_midi_instrument(ultrastar_class):
return instrument
def instruments_to_midi(instruments, bpm, midi_output):
"""Docstring"""
def instruments_to_midi(instruments: list[object], bpm: float, midi_output: str) -> None:
"""Write instruments to midi file"""
print(f"{ULTRASINGER_HEAD} Creating midi file -> {midi_output}")
@@ -54,21 +56,21 @@ class MidiCreator:
"""Docstring"""
def convert_frequencies_to_notes(frequency):
"""Docstring"""
def convert_frequencies_to_notes(frequency: [str]) -> list[list[str]]:
"""Converts frequencies to notes"""
notes = []
for freq in frequency:
notes.append(librosa.hz_to_note(float(freq)))
return notes
def most_frequent(array):
"""Docstring"""
def most_frequent(array: [str]) -> list[tuple[str, int]]:
"""Get most frequent item in array"""
return Counter(array).most_common(1)
def find_nearest_index(array, value):
"""Docstring"""
def find_nearest_index(array: list[float], value: float) -> int:
"""Nearest index in array"""
idx = np.searchsorted(array, value, side="left")
if idx > 0 and (
idx == len(array)
@@ -79,8 +81,8 @@ def find_nearest_index(array, value):
return idx
def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
"""Docstring"""
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], pitched_data: PitchedData) -> list[str]:
"""Create midi notes from pitched data"""
print(f"{ULTRASINGER_HEAD} Creating midi notes from pitched data")
midi_notes = []
@@ -100,17 +102,12 @@ def create_midi_notes_from_pitched_data(start_times, end_times, pitched_data):
return midi_notes
def create_midi_note_from_pitched_data(start_time, end_time, pitched_data):
"""Docstring"""
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData) -> str:
"""Create midi note from pitched data"""
start = find_nearest_index(pitched_data.times, start_time)
end = find_nearest_index(pitched_data.times, end_time)
# fixme: why is end smaller as start?
if end < start:
print(f"{ULTRASINGER_HEAD} {red_highlighted('Error: end time is smaller as start time. The note is C0 now ...')}")
return "C0"
if start == end:
freqs = [pitched_data.frequencies[start]]
confs = [pitched_data.confidence[start]]
+9
View File
@@ -0,0 +1,9 @@
"""Pitched data"""
class PitchedData:
"""Pitched data from crepe"""
times = list[float]
frequencies = list[float]
confidence = list[float]
+60
View File
@@ -0,0 +1,60 @@
"""Pitcher module"""
import crepe
from scipy.io import wavfile
from src.modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from src.modules.Pitcher.pitched_data import PitchedData
def get_pitch_with_crepe_file(filename: str, step_size: int, model_capacity: str) -> PitchedData:
"""Pitch with crepe"""
print(
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted('cpu')} as worker"
)
# Todo: add GPU support by using torchcrepe
sample_rate, audio = wavfile.read(filename)
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
def get_pitch_with_crepe(audio, sample_rate: int, step_size: int, model_capacity: str) -> PitchedData:
"""Pitch with crepe"""
pitched_data = PitchedData()
(
pitched_data.times,
pitched_data.frequencies,
pitched_data.confidence,
activation,
) = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
return pitched_data
def get_frequency_with_high_confidence(frequencies: list[float], confidences: list[float], threshold=0.4) -> list[float]:
"""Get frequency with high confidence"""
conf_f = []
for i, conf in enumerate(confidences):
if conf > threshold:
conf_f.append(frequencies[i])
if not conf_f:
conf_f = frequencies
return conf_f
class Pitcher:
"""Docstring"""
@@ -1,18 +1,18 @@
"""Docstring"""
"""Vosk speech recognition module."""
import json
import wave
from vosk import KaldiRecognizer, Model
from modules.console_colors import ULTRASINGER_HEAD
from modules.Speech_Recognition.TranscribedData import TranscribedData
from src.modules.console_colors import ULTRASINGER_HEAD
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
# todo: Rename to Transcoder?
def transcribe_with_vosk(audio_filename, model_path):
"""Docstring"""
def transcribe_with_vosk(audio_filename: str, model_path: str) -> list[TranscribedData]:
"""Transcribe with vosk"""
# Code from here: https://towardsdatascience.com/speech-recognition-with-timestamps-934ede4234b2
print(
f"{ULTRASINGER_HEAD} Transcribing {audio_filename} with vosk and model {model_path}"
@@ -1,21 +1,24 @@
"""Docstring"""
"""Whisper Speech Recognition Module"""
import whisperx
import sys
from modules.console_colors import (
from src.modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
red_highlighted,
)
from modules.Speech_Recognition.TranscribedData import TranscribedData
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
def transcribe_with_whisper(audio_path, model, device="cpu"):
"""Docstring"""
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (list[TranscribedData], str):
"""Transcribe with whisper"""
print(
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
)
if model_name is not None:
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
batch_size = 16 # reduce if low on GPU mem
compute_type = (
@@ -26,6 +29,7 @@ def transcribe_with_whisper(audio_path, model, device="cpu"):
loaded_whisper_model = whisperx.load_model(
model, device=device, compute_type=compute_type
)
audio = whisperx.load_audio(audio_path)
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
@@ -34,9 +38,14 @@ def transcribe_with_whisper(audio_path, model, device="cpu"):
language = result["language"]
# load alignment model and metadata
model_a, metadata = whisperx.load_align_model(
language_code=language, device=device
)
try:
model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name)
except ValueError as ve:
print(f"{red_highlighted(f'{ve}')}"
f"\n"
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
f"Try add it with --align_model [hugingface].")
sys.exit(0)
# align whisper output
result_aligned = whisperx.align(
@@ -0,0 +1,102 @@
"""Hyphenation module"""
import string
from hyphen import Hyphenator, dictools
from src.modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
)
def language_check(language="en") -> str | None:
"""Check if language is supported"""
dict_langs = dictools.LANGUAGES
lang_region = None
for dict_lang in dict_langs:
if dict_lang.startswith(language):
lang_region = dict_lang
break
if lang_region is None:
return None
print(
f"{ULTRASINGER_HEAD} Hyphenate using language code: {blue_highlighted(lang_region)}"
)
return lang_region
def contains_punctuation(word: str) -> bool:
"""Check if word contains punctuation"""
return any(elem in word for elem in string.punctuation)
def clean_word(word: str):
"""Remove punctuation from word"""
cleaned_string = ""
removed_indices = []
removed_symbols = []
for i, char in enumerate(word):
if char not in string.punctuation and char not in " ":
cleaned_string += char
else:
removed_indices.append(i)
removed_symbols.append(char)
return cleaned_string, removed_indices, removed_symbols
def insert_removed_symbols(separated_array, removed_indices, symbols):
"""Insert symbols into the syllables"""
result = []
symbol_index = 0
i = 0
# Add removed symbols to the syllables
for syllable in separated_array:
tmp = ""
for char in syllable:
if i in removed_indices:
tmp += symbols[symbol_index]
symbol_index += 1
i += 1
tmp += char
i += 1
result.append(tmp)
# Add remaining symbols to the last syllable
if symbol_index < len(symbols):
tmp = result[-1]
for i in range(symbol_index, len(symbols)):
tmp += symbols[i]
result[-1] = tmp
return result
def create_hyphenator(lang_region: str) -> Hyphenator:
"""Create hyphenator"""
hyphenator = Hyphenator(lang_region)
return hyphenator
def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None:
"""Hyphenate word"""
cleaned_string, removed_indices, removed_symbols = clean_word(word)
syllabus = hyphenator.syllables(cleaned_string)
length = len(syllabus)
if length > 1:
hyphen = []
for i in range(length):
hyphen.append(syllabus[i])
hyphen = insert_removed_symbols(hyphen, removed_indices, removed_symbols)
else:
hyphen = None
return hyphen
@@ -6,7 +6,7 @@ import speech_recognition as sr
from pydub import AudioSegment
from pydub.silence import split_on_silence
from modules.console_colors import ULTRASINGER_HEAD
from src.modules.console_colors import ULTRASINGER_HEAD
# todo: Code from here: https://www.thepythoncode.com/article/using-speech-recognition-to-convert-speech-to-text-python
@@ -1,22 +1,23 @@
"""Docstring"""
"""Ultrastar Converter"""
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
def real_bpm_to_ultrastar_bpm(real_bpm):
"""Docstring"""
def real_bpm_to_ultrastar_bpm(real_bpm: float) -> float:
"""Converts real BPM to UltraStar BPM"""
# The UltraStar BPM info is a fifth beat of the real BPM
ultrastar_bpm = real_bpm / 4
return ultrastar_bpm
def ultrastar_bpm_to_real_bpm(ultrastar_bpm):
"""Docstring"""
def ultrastar_bpm_to_real_bpm(ultrastar_bpm: float) -> float:
"""Converts UltraStar BPM to real BPM"""
# The UltraStar BPM info is a fifth beat of the real BPM
bpm = ultrastar_bpm * 4
return bpm
def second_to_beat(seconds, real_bpm):
"""Docstring"""
def second_to_beat(seconds: float, real_bpm: float) -> float:
"""Converts seconds to beat"""
# BPM = 60 * beat / T
# T * BPM = 60 * beat
# beat = T * BPM / 60
@@ -24,31 +25,31 @@ def second_to_beat(seconds, real_bpm):
return beat
def beat_to_second(beat, real_bpm):
"""Docstring"""
def beat_to_second(beat: float, real_bpm: float) -> float:
"""Converts beat to seconds"""
seconds = beat * 60 / real_bpm
return seconds
def midi_note_to_ultrastar_note(midi_note):
"""Docstring"""
def midi_note_to_ultrastar_note(midi_note: int) -> int:
"""Converts Midi note to UltraStar note"""
# C4 == 48 in Midi
ultrastar_note = midi_note - 48
return ultrastar_note
def ultrastar_note_to_midi_note(ultrastar_note):
"""Docstring"""
def ultrastar_note_to_midi_note(ultrastar_note: int) -> int:
"""Converts UltraStar note to Midi note"""
# C4 == 48 in Midi
midi_note = ultrastar_note + 48
return midi_note
def get_start_time_from_ultrastar(ultrastar_class, pos):
"""Docstring"""
def get_start_time_from_ultrastar(ultrastar_class: UltrastarTxtValue, pos: int) -> float:
"""Calculates the start time from the Ultrastar txt"""
gap = int(ultrastar_class.gap) / 1000
real_bpm = ultrastar_bpm_to_real_bpm(
@@ -60,8 +61,8 @@ def get_start_time_from_ultrastar(ultrastar_class, pos):
return start_time
def get_end_time_from_ultrastar(ultrastar_class, pos):
"""Docstring"""
def get_end_time_from_ultrastar(ultrastar_class: UltrastarTxtValue, pos: int) -> float:
"""Calculates the end time from the Ultrastar txt"""
gap = int(ultrastar_class.gap) / 1000
real_bpm = ultrastar_bpm_to_real_bpm(
@@ -1,38 +1,42 @@
"""Docstring"""
"""Ultrastar txt parser"""
from modules.console_colors import ULTRASINGER_HEAD
from modules.Ultrastar.ultrastar_converter import (
from src.modules.console_colors import ULTRASINGER_HEAD
from src.modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
)
from modules.Ultrastar.ultrastar_txt import UltrastarTxt
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, UltrastarTxtTag, UltrastarTxtNoteTypeTag, FILE_ENCODING
def parse_ultrastar_txt(input_file):
"""Docstring"""
def parse_ultrastar_txt(input_file: str) -> UltrastarTxtValue:
"""Parse ultrastar txt file to UltrastarTxt class"""
print(f"{ULTRASINGER_HEAD} Parse ultrastar txt -> {input_file}")
with open(input_file, "r", encoding="utf-8") as file:
with open(input_file, "r", encoding=FILE_ENCODING) as file:
txt = file.readlines()
ultrastar_class = UltrastarTxt()
ultrastar_class = UltrastarTxtValue()
count = 0
# Strips the newline character
for line in txt:
count += 1
if line.startswith("#"):
if line.startswith("#ARTIST"):
if line.startswith(f"#{UltrastarTxtTag.ARTIST}"):
ultrastar_class.artist = line.split(":")[1].replace("\n", "")
elif line.startswith("#TITLE"):
elif line.startswith(f"#{UltrastarTxtTag.TITLE}"):
ultrastar_class.title = line.split(":")[1].replace("\n", "")
elif line.startswith("#MP3"):
elif line.startswith(f"#{UltrastarTxtTag.MP3}"):
ultrastar_class.mp3 = line.split(":")[1].replace("\n", "")
elif line.startswith("#GAP"):
elif line.startswith(f"#{UltrastarTxtTag.GAP}"):
ultrastar_class.gap = line.split(":")[1].replace("\n", "")
elif line.startswith("#BPM"):
elif line.startswith(f"#{UltrastarTxtTag.BPM}"):
ultrastar_class.bpm = line.split(":")[1].replace("\n", "")
elif line.startswith(("F ", ": ", "* ", "R ", "G ")):
elif line.startswith((
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
f"{UltrastarTxtNoteTypeTag.RAP} ",
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ")):
parts = line.split()
# [0] F : * R G
# [1] start beat
@@ -1,8 +1,8 @@
"""Docstring"""
"""Ultrastar score calculator."""
import librosa
from modules.console_colors import (
from src.modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted,
cyan_highlighted,
@@ -10,12 +10,14 @@ from modules.console_colors import (
light_blue_highlighted,
underlined,
)
from modules.Midi.midi_creator import create_midi_note_from_pitched_data
from modules.Ultrastar.ultrastar_converter import (
from src.modules.Midi.midi_creator import create_midi_note_from_pitched_data
from src.modules.Ultrastar.ultrastar_converter import (
get_end_time_from_ultrastar,
get_start_time_from_ultrastar,
ultrastar_note_to_midi_note,
)
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
from src.modules.Pitcher.pitched_data import PitchedData
MAX_SONG_SCORE = 10000
MAX_SONG_LINE_BONUS = 1000
@@ -32,8 +34,8 @@ class Points:
parts = 0
def add_point(note_type, points):
"""Docstring"""
def add_point(note_type: str, points: Points) -> Points:
"""Add calculated points to the points object."""
if note_type == ":":
points.notes += 1
@@ -56,8 +58,8 @@ class Score:
score = 0
def get_score(points):
"""Docstring"""
def get_score(points: Points) -> Score:
"""Score calculation."""
score = Score()
score.max_score = (
@@ -74,16 +76,16 @@ def get_score(points):
return score
def print_score(score):
"""Docstring"""
def print_score(score: Score) -> None:
"""Print score."""
print(
f"{ULTRASINGER_HEAD} Total: {cyan_highlighted(str(score.score))}, notes: {blue_highlighted(str(score.notes))}, line bonus: {light_blue_highlighted(str(score.line_bonus))}, golden notes: {gold_highlighted(str(score.golden))}"
)
def calculate_score(pitched_data, ultrastar_class):
"""Docstring"""
def calculate_score(pitched_data: PitchedData, ultrastar_class: UltrastarTxtValue) -> (Score, Score):
"""Calculate score."""
print(ULTRASINGER_HEAD + " Calculating Ultrastar Points")
@@ -150,8 +152,8 @@ def calculate_score(pitched_data, ultrastar_class):
return get_score(simple_points), get_score(accurate_points)
def print_score_calculation(simple_points, accurate_points):
"""Docstring"""
def print_score_calculation(simple_points: Score, accurate_points: Score) -> None:
"""Print score calculation."""
print(
f"{ULTRASINGER_HEAD} {underlined('Simple (octave high ignored)')} points"
+79
View File
@@ -0,0 +1,79 @@
"""Ultrastar TXT"""
from enum import Enum
FILE_ENCODING = "utf-8"
class UltrastarTxtTag(str, Enum):
"""Tags for Ultrastar TXT files."""
ARTIST = 'ARTIST'
TITLE = 'TITLE'
MP3 = 'MP3'
GAP = 'GAP'
BPM = 'BPM'
LANGUAGE = 'LANGUAGE'
COVER = 'COVER' # Path to cover. Should end with `*[CO].jpg`
CREATOR = 'CREATOR'
COMMENT = 'COMMENT'
VIDEO = 'VIDEO'
FILE_END = 'E'
LINEBREAK = '-'
# Only used in UltraStar World Party
FIXER = 'FIXER'
# Unused
BACKGROUND = 'BACKGROUND' # Path to background. Is shown when there is no video. Should end with `*[BG].jpg`
VIDEOGAP = 'VIDEOGAP'
GENRE = 'GENRE'
EDITION = 'EDITION'
YEAR = 'YEAR'
START = 'START'
END = 'END'
RESOLUTION = 'RESOLUTION' # Changes the grid resolution of the editor. Only for the editor and nothing for singing.
NOTESGAP = 'NOTESGAP'
RELATIVE = 'RELATIVE'
ENCODING = 'ENCODING'
PREVIEWSTART = 'PREVIEWSTART'
MEDLEYSTARTBEAT = 'MEDLEYSTARTBEAT'
MEDLEYENDBEAT = 'MEDLEYENDBEAT'
CALCMEDLEY = 'CALCMEDLEY'
DUETSINGERP1 = 'DUETSINGERP1'
DUETSINGERP2 = 'DUETSINGERP2'
P1 = 'P1' # Only for UltraStar Deluxe
P2 = 'P2' # Only for UltraStar Deluxe
class UltrastarTxtNoteTypeTag(str, Enum):
"""Note types for Ultrastar TXT files."""
NORMAL = ':'
RAP = 'R'
RAP_GOLDEN = 'G'
FREESTYLE = 'F'
GOLDEN = '*'
class UltrastarTxtValue:
"""Vaules for Ultrastar TXT files."""
artist = ""
title = ""
mp3 = ""
video = None
gap = ""
bpm = ""
language = None
cover = None
creator = "UltraSinger [GitHub]"
comment = "UltraSinger [GitHub]"
startBeat = []
startTimes = []
endTimes = []
durations = []
pitches = []
words = []
noteType = [] # F, R, G, *, :
# Only used in UltraStar World Party
fixer = "YOUR NAME"
@@ -1,18 +1,21 @@
"""Docstring"""
"""Ultrastar writer module"""
import re
import langcodes
from modules.console_colors import ULTRASINGER_HEAD
from modules.Ultrastar.ultrastar_converter import (
from src.modules.console_colors import ULTRASINGER_HEAD
from src.modules.Ultrastar.ultrastar_converter import (
real_bpm_to_ultrastar_bpm,
second_to_beat,
)
from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, UltrastarTxtTag, UltrastarTxtNoteTypeTag, FILE_ENCODING
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
from src.modules.Ultrastar.ultrastar_score_calculator import Score
def get_multiplier(real_bpm):
"""Docstring"""
def get_multiplier(real_bpm: float) -> int:
"""Calculates the multiplier for the BPM"""
if real_bpm == 0:
raise Exception("BPM is 0")
@@ -25,20 +28,20 @@ def get_multiplier(real_bpm):
return multiplier - 2
def get_language_name(language):
"""Docstring"""
def get_language_name(language: str) -> str:
"""Creates the language name from the language code"""
return langcodes.Language.make(language=language).display_name()
def create_ultrastar_txt_from_automation(
transcribed_data,
note_numbers,
ultrastar_file_output,
ultrastar_class,
bpm=120,
):
"""Docstring"""
transcribed_data: list[TranscribedData],
note_numbers: list[int],
ultrastar_file_output: str,
ultrastar_class: UltrastarTxtValue,
bpm=120,
) -> None:
"""Creates an Ultrastar txt file from the automation data"""
print(
f"{ULTRASINGER_HEAD} Creating {ultrastar_file_output} from transcription."
@@ -48,32 +51,32 @@ def create_ultrastar_txt_from_automation(
multiplication = get_multiplier(real_bpm)
ultrastar_bpm = real_bpm * get_multiplier(real_bpm)
with open(ultrastar_file_output, "w", encoding="utf-8") as file:
with open(ultrastar_file_output, "w", encoding=FILE_ENCODING) as file:
gap = transcribed_data[0].start
file.write(f"#ARTIST:{ultrastar_class.artist}\n")
file.write(f"#TITLE:{ultrastar_class.title}\n")
file.write(f"#CREATOR:{ultrastar_class.creator}\n")
file.write(f"#FIXER:{ultrastar_class.fixer}\n")
file.write(f"#{UltrastarTxtTag.ARTIST}:{ultrastar_class.artist}\n")
file.write(f"#{UltrastarTxtTag.TITLE}:{ultrastar_class.title}\n")
file.write(f"#{UltrastarTxtTag.CREATOR}:{ultrastar_class.creator}\n")
file.write(f"#{UltrastarTxtTag.FIXER}:{ultrastar_class.fixer}\n")
if ultrastar_class.language is not None:
file.write(
f"#LANGUAGE:{get_language_name(ultrastar_class.language)}\n"
f"#{UltrastarTxtTag.LANGUAGE}:{get_language_name(ultrastar_class.language)}\n"
)
if ultrastar_class.cover is not None:
file.write(f"#COVER:{ultrastar_class.cover}\n")
file.write(f"#MP3:{ultrastar_class.mp3}\n")
file.write(f"#VIDEO:{ultrastar_class.video}\n")
file.write(f"#BPM:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
file.write(f"#GAP:{int(gap * 1000)}\n")
file.write(f"#COMMENT:{ultrastar_class.comment}\n")
file.write(f"#{UltrastarTxtTag.COVER}:{ultrastar_class.cover}\n")
file.write(f"#{UltrastarTxtTag.MP3}:{ultrastar_class.mp3}\n")
file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n")
file.write(f"#{UltrastarTxtTag.BPM}:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
file.write(f"#{UltrastarTxtTag.GAP}:{int(gap * 1000)}\n")
file.write(f"#{UltrastarTxtTag.COMMENT}:{ultrastar_class.comment}\n")
# Write the singing part
previous_end_beat = 0
for i, data in enumerate(transcribed_data):
start_time = (data.start - gap) * multiplication
end_time = (
data.end - data.start
) * multiplication
data.end - data.start
) * multiplication
start_beat = round(second_to_beat(start_time, bpm))
duration = round(second_to_beat(end_time, bpm))
@@ -87,7 +90,7 @@ def create_ultrastar_txt_from_automation(
# 'n2' duration at real beat
# 'n3' pitch where 0 == C4
# 'w' lyric
file.write(": ")
file.write(f"{UltrastarTxtNoteTypeTag.NORMAL} ")
file.write(str(start_beat) + " ")
file.write(str(duration) + " ")
file.write(str(note_numbers[i]) + " ")
@@ -97,7 +100,7 @@ def create_ultrastar_txt_from_automation(
# detect silence between words
if i < len(transcribed_data) - 1:
silence = (
transcribed_data[i + 1].start - data.end
transcribed_data[i + 1].start - data.end
)
else:
silence = 0
@@ -106,34 +109,34 @@ def create_ultrastar_txt_from_automation(
# - 10
# '-' end of current sing part
# 'n1' show next at time in real beat
file.write("- ")
file.write(f"{UltrastarTxtTag.LINEBREAK} ")
show_next = (
second_to_beat(data.end - gap, bpm)
* multiplication
second_to_beat(data.end - gap, bpm)
* multiplication
)
file.write(str(round(show_next)))
file.write("\n")
file.write("E")
file.write(f"{UltrastarTxtTag.FILE_END}")
def create_repitched_txt_from_ultrastar_data(
input_file, note_numbers, output_repitched_ultrastar
):
"""Docstring"""
input_file: str, note_numbers: list[int], output_repitched_ultrastar: str
) -> None:
"""Creates a repitched ultrastar txt file from the original one"""
# todo: just add '_repitched' to input_file
print(
"{PRINT_ULTRASTAR} Creating repitched ultrastar txt -> {input_file}_repitch.txt"
)
# todo: to reader
with open(input_file, "r", encoding="utf-8") as file:
with open(input_file, "r", encoding=FILE_ENCODING) as file:
txt = file.readlines()
i = 0
# todo: just add '_repitched' to input_file
with open(output_repitched_ultrastar, "w", encoding="utf-8") as file:
with open(output_repitched_ultrastar, "w", encoding=FILE_ENCODING) as file:
for line in txt:
if line.startswith(":"):
if line.startswith(f"#{UltrastarTxtNoteTypeTag.NORMAL} "):
parts = re.findall(r"\S+|\s+", line)
# between are whitespaces
# [0] :
@@ -149,29 +152,34 @@ def create_repitched_txt_from_ultrastar_data(
file.write(line)
def add_score_to_ultrastar_txt(ultrastar_file_output, score):
"""Docstring"""
with open(ultrastar_file_output, "r", encoding="utf-8") as file:
def add_score_to_ultrastar_txt(ultrastar_file_output: str, score: Score) -> None:
"""Adds the score to the ultrastar txt file"""
with open(ultrastar_file_output, "r", encoding=FILE_ENCODING) as file:
text = file.read()
text = text.split("\n")
for i, line in enumerate(text):
if line.startswith("#COMMENT:"):
if line.startswith(f"#{UltrastarTxtTag.COMMENT}:"):
text[
i
] = f"{line} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
break
if line.startswith(("F ", ": ", "* ", "R ", "G ")):
if line.startswith((
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
f"{UltrastarTxtNoteTypeTag.RAP} ",
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ")):
text.insert(
i,
f"#COMMENT: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
f"#{UltrastarTxtTag.COMMENT}: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
)
break
text = "\n".join(text)
with open(ultrastar_file_output, "w", encoding="utf-8") as file:
with open(ultrastar_file_output, "w", encoding=FILE_ENCODING) as file:
file.write(text)
File renamed without changes.
+47
View File
@@ -0,0 +1,47 @@
"""CSV export module"""
import csv
from src.modules.console_colors import ULTRASINGER_HEAD
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
def export_transcribed_data_to_csv(transcribed_data: list[TranscribedData], filename: str) -> None:
"""Export transcribed data to csv"""
print(f"{ULTRASINGER_HEAD} Exporting transcribed data to CSV")
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile)
header = ["word", "start", "end", "confidence"]
writer.writerow(header)
for i, data in enumerate(transcribed_data):
writer.writerow(
[
data.word,
data.start,
data.end,
data.conf,
]
)
def write_lists_to_csv(times, frequencies, confidences, filename: str):
"""Write lists to csv"""
with open(filename, "w", encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile)
header = ["time", "frequency", "confidence"]
writer.writerow(header)
for i in enumerate(times):
pos = i[0]
writer.writerow([times[pos], frequencies[pos], confidences[pos]])
def read_data_from_csv(filename: str):
"""Read data from csv"""
csv_data = []
with open(filename, "r", encoding="utf-8") as csv_file:
csv_reader = csv.reader(csv_file)
for line in csv_reader:
csv_data.append(line)
headless_data = csv_data[1:]
return headless_data
File renamed without changes.
+44
View File
@@ -0,0 +1,44 @@
"""OS helper functions."""
import os
import shutil
from src.modules.console_colors import ULTRASINGER_HEAD
def create_folder(folder_name: str) -> None:
"""Creates a folder if it doesn't exist."""
print(f"{ULTRASINGER_HEAD} Creating output folder. -> {folder_name}")
is_exist = os.path.exists(folder_name)
if not is_exist:
os.makedirs(folder_name)
def move(original: str, target: str) -> None:
"""Moves a file from one location to another."""
shutil.move(original, target)
def copy(original:str, target: str) -> None:
"""Copies a file from one location to another."""
shutil.copy(original, target)
def current_executor_path() -> str:
"""Current executor path"""
return os.getcwd()
def path_join(path1: str, path2: str) -> str:
"""Joins two paths together"""
return os.path.join(path1, path2)
def check_file_exists(file_path: str) -> bool:
"""Checks if a file exists."""
return os.path.isfile(file_path)
def check_if_folder_exists(song_output: str) -> bool:
"""Checks if a folder exists."""
return os.path.isdir(song_output)
+71
View File
@@ -0,0 +1,71 @@
"""Plot transcribed data"""
import os
import librosa
from matplotlib import pyplot as plt
from matplotlib.patches import Rectangle
from src.modules.Pitcher.pitched_data import PitchedData
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
from src.modules.console_colors import ULTRASINGER_HEAD
def get_frequency_range(midi_note: str) -> float:
midi = librosa.note_to_midi(midi_note)
frequency_range = librosa.midi_to_hz(midi + 1) - librosa.midi_to_hz(midi)
return frequency_range
def plot(pitched_data: PitchedData,
transcribed_data: list[TranscribedData],
midi_notes: list[str],
output_path: str
) -> None:
"""Plot transcribed data"""
print(f"{ULTRASINGER_HEAD} Creating plot")
conf_t, conf_f, conf_c = get_confidence(pitched_data, 0.4)
plt.ylim(0, max(conf_f))
plt.xlim(0, pitched_data.times[-1])
plt.plot(conf_t, conf_f, linewidth=0.1)
for i, data in enumerate(transcribed_data):
note_frequency = librosa.note_to_hz(midi_notes[i])
frequency_range = get_frequency_range(midi_notes[i])
xy_start_pos = (data.start, note_frequency - frequency_range / 2)
width = data.end - data.start
height = frequency_range
rect = Rectangle(
xy_start_pos,
width,
height,
edgecolor='none',
facecolor='red',
alpha=0.5)
plt.gca().add_patch(rect)
plt.xlabel("Time (s)")
plt.ylabel("Frequency (Hz)")
plt.ylim(0, 800)
dpi = 4000
plt.savefig(os.path.join(output_path, "plot.png"), dpi=dpi)
def get_confidence(
pitched_data: PitchedData,
threshold: float
) -> tuple[list[float], list[float], list[float]]:
"""Get high confidence data"""
# todo: replace get_frequency_with_high_conf from pitcher
conf_t = []
conf_f = []
conf_c = []
for i in enumerate(pitched_data.times):
pos = i[0]
if pitched_data.confidence[pos] > threshold:
conf_t.append(pitched_data.times[pos])
conf_f.append(pitched_data.frequencies[pos])
conf_c.append(pitched_data.confidence[pos])
return conf_t, conf_f, conf_c