mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
7 files changed
+42
-34
No files matched your search
@@ -0,0 +1,6 @@
|
||||
Version: 0.0.4
|
||||
Date: 2023.12.16
|
||||
Changes:
|
||||
- Optimized the conversion to Mono
|
||||
- Removed limitation to mp3 and wav audio formats
|
||||
- Added option float32 to demucs
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
class Settings:
|
||||
APP_VERSION = "0.0.3"
|
||||
APP_VERSION = "0.0.4"
|
||||
|
||||
create_midi = True
|
||||
create_plot = False
|
||||
|
||||
+19
-15
@@ -9,7 +9,7 @@ import Levenshtein
|
||||
import librosa
|
||||
from tqdm import tqdm
|
||||
|
||||
from modules import os_helper, timer
|
||||
from modules import os_helper
|
||||
from modules.Audio.denoise import ffmpeg_reduce_noise
|
||||
from modules.Audio.separation import separate_audio
|
||||
from modules.Audio.vocal_chunks import (
|
||||
@@ -336,8 +336,23 @@ def run() -> None:
|
||||
basename_without_ext, cache_path, ultrastar_audio_input_path
|
||||
)
|
||||
|
||||
if settings.use_separated_vocal:
|
||||
input_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
else:
|
||||
input_path = ultrastar_audio_input_path
|
||||
|
||||
# Denoise vocal audio
|
||||
denoise_vocal_audio(basename_without_ext, cache_path)
|
||||
denoised_output_path = os.path.join(
|
||||
cache_path, basename_without_ext + "_denoised.wav"
|
||||
)
|
||||
denoise_vocal_audio(input_path, denoised_output_path)
|
||||
|
||||
# Convert to mono audio
|
||||
mono_output_path = os.path.join(
|
||||
cache_path, basename_without_ext + "_mono.wav"
|
||||
)
|
||||
convert_audio_to_mono_wav(denoised_output_path, mono_output_path)
|
||||
settings.mono_audio_path = mono_output_path
|
||||
|
||||
# Audio transcription
|
||||
transcribed_data = None
|
||||
@@ -469,15 +484,8 @@ def separate_vocal_from_audio(
|
||||
if settings.use_separated_vocal or settings.create_karaoke:
|
||||
separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device)
|
||||
|
||||
if settings.use_separated_vocal:
|
||||
input_path = os.path.join(audio_separation_path, "vocals.wav")
|
||||
else:
|
||||
input_path = ultrastar_audio_input_path
|
||||
|
||||
convert_audio_to_mono_wav(input_path, settings.mono_audio_path)
|
||||
return audio_separation_path
|
||||
|
||||
|
||||
def calculate_score_points(
|
||||
is_audio: bool, pitched_data: PitchedData, ultrastar_class: UltrastarTxtValue, ultrastar_file_output: str
|
||||
):
|
||||
@@ -784,13 +792,9 @@ def create_audio_chunks(
|
||||
)
|
||||
|
||||
|
||||
def denoise_vocal_audio(basename_without_ext: str, cache_path: str) -> None:
|
||||
def denoise_vocal_audio(input_path: str, output_path: str) -> None:
|
||||
"""Denoise vocal audio"""
|
||||
denoised_path = os.path.join(
|
||||
cache_path, basename_without_ext + "_denoised.wav"
|
||||
)
|
||||
ffmpeg_reduce_noise(settings.mono_audio_path, denoised_path)
|
||||
settings.mono_audio_path = denoised_path
|
||||
ffmpeg_reduce_noise(input_path, output_path)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> None:
|
||||
|
||||
@@ -1,28 +1,22 @@
|
||||
"""Convert audio to other formats"""
|
||||
|
||||
from pydub import AudioSegment
|
||||
import librosa
|
||||
import soundfile as sf
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
|
||||
def convert_audio_to_mono_wav(input_file: str, output_file: str) -> None:
|
||||
def convert_audio_to_mono_wav(input_file_path: str, output_file_path: str) -> None:
|
||||
"""Convert audio to mono wav"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting audio for AI")
|
||||
|
||||
if ".mp3" in input_file:
|
||||
sound = AudioSegment.from_mp3(input_file)
|
||||
elif ".wav" in input_file:
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
else:
|
||||
raise ValueError("data format not supported")
|
||||
|
||||
sound = sound.set_channels(1)
|
||||
sound.export(output_file, format="wav")
|
||||
y, sr = librosa.load(input_file_path, mono=True, sr=None)
|
||||
sf.write(output_file_path, y, sr)
|
||||
|
||||
|
||||
def convert_wav_to_mp3(input_file: str, output_file: str) -> None:
|
||||
def convert_wav_to_mp3(input_file_path: str, output_file_path: str) -> None:
|
||||
"""Convert wav to mp3"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting wav to mp3")
|
||||
|
||||
sound = AudioSegment.from_wav(input_file)
|
||||
sound.export(output_file, format="mp3")
|
||||
sound = AudioSegment.from_wav(input_file_path)
|
||||
sound.export(output_file_path, format="mp3")
|
||||
@@ -17,10 +17,9 @@ def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None
|
||||
f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker."
|
||||
)
|
||||
# Model selection?
|
||||
# -n mdx_q
|
||||
# -n htdemucs_ft
|
||||
subprocess.run(
|
||||
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
|
||||
["demucs", "-d", device, "--two-stems=vocals", "--float32", input_file_path]
|
||||
)
|
||||
separated_folder = path_join(current_executor_path(), "separated")
|
||||
move(separated_folder, output_file)
|
||||
@@ -24,6 +24,9 @@ def get_pitch_with_crepe(
|
||||
audio, sample_rate: int, model_capacity: str, step_size: int = 10
|
||||
) -> PitchedData:
|
||||
"""Pitch with crepe"""
|
||||
|
||||
# Info: The model is trained on 16 kHz audio, so if the input audio has a different sample rate, it will be first resampled to 16 kHz using resampy inside crepe.
|
||||
|
||||
times, frequencies, confidence, activation = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
|
||||
@@ -20,6 +20,8 @@ def transcribe_with_whisper(
|
||||
) -> (list[TranscribedData], str):
|
||||
"""Transcribe with whisper"""
|
||||
|
||||
# Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work.
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
|
||||
)
|
||||
@@ -106,8 +108,8 @@ def convert_to_transcribed_data(result_aligned):
|
||||
previous.end = ""
|
||||
vtd.start = previous.end + 0.1
|
||||
vtd.end = previous.end + 0.2
|
||||
msg = f'Error: There is no timestamp for word: {obj["word"]}. ' \
|
||||
f'Fixing it by placing it after the previous word: {previous.word}. At start: {vtd.start} end: {vtd.end}. Fix it manually!'
|
||||
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
|
||||
f'Fixing it by placing it after the previous word: "{previous.word}". At start: {vtd.start} end: {vtd.end}. Fix it manually!'
|
||||
print(f"{red_highlighted(msg)}")
|
||||
transcribed_data.append(vtd) # and add it to list
|
||||
return transcribed_data
|
||||
Reference in new issue
Block a user