Merge pull request #107 from rakuri255/Optimize-Mono

Optimize mono
This commit is contained in:
Vadim Rangnau authored and GitHub committed 2023-12-16 13:27:43 +01:00
commit e39cfa0c9b
7 files changed
+42 -34

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
Version: 0.0.4
Date: 2023.12.16
Changes:
- Optimized the conversion to Mono
- Removed limitation to mp3 and wav audio formats
- Added option float32 to demucs
+1 -1
View File
@@ -1,5 +1,5 @@
class Settings:
APP_VERSION = "0.0.3"
APP_VERSION = "0.0.4"
create_midi = True
create_plot = False
+19 -15
View File
@@ -9,7 +9,7 @@ import Levenshtein
import librosa
from tqdm import tqdm
from modules import os_helper, timer
from modules import os_helper
from modules.Audio.denoise import ffmpeg_reduce_noise
from modules.Audio.separation import separate_audio
from modules.Audio.vocal_chunks import (
@@ -336,8 +336,23 @@ def run() -> None:
basename_without_ext, cache_path, ultrastar_audio_input_path
)
if settings.use_separated_vocal:
input_path = os.path.join(audio_separation_path, "vocals.wav")
else:
input_path = ultrastar_audio_input_path
# Denoise vocal audio
denoise_vocal_audio(basename_without_ext, cache_path)
denoised_output_path = os.path.join(
cache_path, basename_without_ext + "_denoised.wav"
)
denoise_vocal_audio(input_path, denoised_output_path)
# Convert to mono audio
mono_output_path = os.path.join(
cache_path, basename_without_ext + "_mono.wav"
)
convert_audio_to_mono_wav(denoised_output_path, mono_output_path)
settings.mono_audio_path = mono_output_path
# Audio transcription
transcribed_data = None
@@ -469,15 +484,8 @@ def separate_vocal_from_audio(
if settings.use_separated_vocal or settings.create_karaoke:
separate_audio(ultrastar_audio_input_path, cache_path, settings.pytorch_device)
if settings.use_separated_vocal:
input_path = os.path.join(audio_separation_path, "vocals.wav")
else:
input_path = ultrastar_audio_input_path
convert_audio_to_mono_wav(input_path, settings.mono_audio_path)
return audio_separation_path
def calculate_score_points(
is_audio: bool, pitched_data: PitchedData, ultrastar_class: UltrastarTxtValue, ultrastar_file_output: str
):
@@ -784,13 +792,9 @@ def create_audio_chunks(
)
def denoise_vocal_audio(basename_without_ext: str, cache_path: str) -> None:
def denoise_vocal_audio(input_path: str, output_path: str) -> None:
"""Denoise vocal audio"""
denoised_path = os.path.join(
cache_path, basename_without_ext + "_denoised.wav"
)
ffmpeg_reduce_noise(settings.mono_audio_path, denoised_path)
settings.mono_audio_path = denoised_path
ffmpeg_reduce_noise(input_path, output_path)
def main(argv: list[str]) -> None:
+8 -14
View File
@@ -1,28 +1,22 @@
"""Convert audio to other formats"""
from pydub import AudioSegment
import librosa
import soundfile as sf
from modules.console_colors import ULTRASINGER_HEAD
def convert_audio_to_mono_wav(input_file: str, output_file: str) -> None:
def convert_audio_to_mono_wav(input_file_path: str, output_file_path: str) -> None:
"""Convert audio to mono wav"""
print(f"{ULTRASINGER_HEAD} Converting audio for AI")
if ".mp3" in input_file:
sound = AudioSegment.from_mp3(input_file)
elif ".wav" in input_file:
sound = AudioSegment.from_wav(input_file)
else:
raise ValueError("data format not supported")
sound = sound.set_channels(1)
sound.export(output_file, format="wav")
y, sr = librosa.load(input_file_path, mono=True, sr=None)
sf.write(output_file_path, y, sr)
def convert_wav_to_mp3(input_file: str, output_file: str) -> None:
def convert_wav_to_mp3(input_file_path: str, output_file_path: str) -> None:
"""Convert wav to mp3"""
print(f"{ULTRASINGER_HEAD} Converting wav to mp3")
sound = AudioSegment.from_wav(input_file)
sound.export(output_file, format="mp3")
sound = AudioSegment.from_wav(input_file_path)
sound.export(output_file_path, format="mp3")
+1 -2
View File
@@ -17,10 +17,9 @@ def separate_audio(input_file_path: str, output_file: str, device="cpu") -> None
f"{ULTRASINGER_HEAD} Separating vocals from audio with {blue_highlighted('demucs')} and {red_highlighted(device)} as worker."
)
# Model selection?
# -n mdx_q
# -n htdemucs_ft
subprocess.run(
["demucs", "-d", device, "--two-stems=vocals", input_file_path]
["demucs", "-d", device, "--two-stems=vocals", "--float32", input_file_path]
)
separated_folder = path_join(current_executor_path(), "separated")
move(separated_folder, output_file)
+3
View File
@@ -24,6 +24,9 @@ def get_pitch_with_crepe(
audio, sample_rate: int, model_capacity: str, step_size: int = 10
) -> PitchedData:
"""Pitch with crepe"""
# Info: The model is trained on 16 kHz audio, so if the input audio has a different sample rate, it will be first resampled to 16 kHz using resampy inside crepe.
times, frequencies, confidence, activation = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
+4 -2
View File
@@ -20,6 +20,8 @@ def transcribe_with_whisper(
) -> (list[TranscribedData], str):
"""Transcribe with whisper"""
# Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work.
print(
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
)
@@ -106,8 +108,8 @@ def convert_to_transcribed_data(result_aligned):
previous.end = ""
vtd.start = previous.end + 0.1
vtd.end = previous.end + 0.2
msg = f'Error: There is no timestamp for word: {obj["word"]}. ' \
f'Fixing it by placing it after the previous word: {previous.word}. At start: {vtd.start} end: {vtd.end}. Fix it manually!'
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
f'Fixing it by placing it after the previous word: "{previous.word}". At start: {vtd.start} end: {vtd.end}. Fix it manually!'
print(f"{red_highlighted(msg)}")
transcribed_data.append(vtd) # and add it to list
return transcribed_data