diff --git a/.gitignore b/.gitignore index 9c544e3..141ad7c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ venv # default output folder output +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md index 9f8f6b4..de92eb6 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,10 @@ _Not all options working now!_ # Default is whisper --whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2 English-only model > tiny.en|base.en|small.en|medium.en - --align_model Use other languages model for Whisper provided from huggingface.co - + --align_model Use other languages model for Whisper provided from huggingface.co + --language Override the language detected by whisper, does not affect transcription but steps after transcription + --batch_size Reduce if low on GPU mem >> ((default) is 16) + --compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu) --vosk Needs model [pitcher] diff --git a/src/Settings.py b/src/Settings.py index 93d5b25..00ca5cd 100644 --- a/src/Settings.py +++ b/src/Settings.py @@ -10,6 +10,8 @@ class Settings: output_file_path = "" mono_audio_path = "" + language = None + # Transcribe audio_chunk_folder_name = "audio-chunks" @@ -18,6 +20,8 @@ class Settings: whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2 # English-only model tiny.en|base.en|small.en|medium.en whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2" + whisper_batch_size = 16 # reduce if low on GPU mem + whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy) # Vosk vosk_model_path = "" # "models/vosk-model-small-en-us-0.15" diff --git a/src/UltraSinger.py b/src/UltraSinger.py index 4a21e7f..75e017f 100644 --- a/src/UltraSinger.py +++ b/src/UltraSinger.py @@ -4,7 +4,6 @@ import copy import getopt import os import sys -from time import sleep import Levenshtein import librosa @@ -230,7 +229,6 @@ def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData]) ) return None - sleep(0.1) hyphenator = create_hyphenator(lang_region) for i in tqdm(enumerate(transcribed_data)): pos = i[0] @@ -309,9 +307,12 @@ def run() -> None: # Audio transcription transcribed_data = None - language = None + language = settings.language if is_audio: - language, transcribed_data = transcribe_audio() + language_from_transcription, transcribed_data = transcribe_audio() + if language is None: + language = language_from_transcription + remove_unecessary_punctuations(transcribed_data) transcribed_data = remove_silence_from_transcribtion_data( settings.mono_audio_path, transcribed_data @@ -410,13 +411,13 @@ def transcribe_audio() -> (str, list[TranscribedData]): if settings.transcriber == "whisper": device = "cpu" if settings.force_whisper_cpu else settings.device transcribed_data, language = transcribe_with_whisper( - settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model) + settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model, settings.whisper_batch_size, settings.whisper_compute_type, settings.language) else: # vosk transcribed_data = transcribe_with_vosk( settings.mono_audio_path, settings.vosk_model_path ) - # todo: make language selectable - language = "en" + if settings.language is None: + language = "en" return language, transcribed_data @@ -779,6 +780,12 @@ def init_settings(argv: list[str]) -> None: settings.whisper_model = arg elif opt in ("--align_model"): settings.whisper_align_model = arg + elif opt in ("--batch_size"): + settings.whisper_batch_size = int(arg) + elif opt in ("--compute_type"): + settings.whisper_compute_type = arg + elif opt in ("--language"): + settings.language = arg elif opt in ("--vosk"): settings.transcriber = "vosk" settings.vosk_model_path = arg @@ -816,6 +823,9 @@ def arg_options(): "vosk=", "whisper=", "align_model=", + "batch_size=", + "compute_type=", + "language=", "plot=", "hyphenation=", "disable_separation=", diff --git a/src/modules/Speech_Recognition/Whisper.py b/src/modules/Speech_Recognition/Whisper.py index 12bb7ee..1329d8f 100644 --- a/src/modules/Speech_Recognition/Whisper.py +++ b/src/modules/Speech_Recognition/Whisper.py @@ -11,7 +11,7 @@ from modules.console_colors import ( from modules.Speech_Recognition.TranscribedData import TranscribedData -def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> ( +def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None, batch_size: int = 16, compute_type: str = None, language: str = None) -> ( list[TranscribedData], str): """Transcribe with whisper""" @@ -21,10 +21,10 @@ list[TranscribedData], str): if model_name is not None: print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}") - batch_size = 16 # reduce if low on GPU mem - compute_type = ( - "float16" if device == "cuda" else "int8" - ) # change to "int8" if low on GPU mem (may reduce accuracy) + if compute_type is None: + compute_type = ( + "float16" if device == "cuda" else "int8" + ) loaded_whisper_model = whisperx.load_model( model, device=device, compute_type=compute_type @@ -35,7 +35,9 @@ list[TranscribedData], str): print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}") result = loaded_whisper_model.transcribe(audio, batch_size=batch_size) - language = result["language"] + + if language is None: + language = result["language"] # load alignment model and metadata try: diff --git a/src/modules/Speech_Recognition/hyphenation.py b/src/modules/Speech_Recognition/hyphenation.py index a166c56..c117059 100644 --- a/src/modules/Speech_Recognition/hyphenation.py +++ b/src/modules/Speech_Recognition/hyphenation.py @@ -88,6 +88,11 @@ def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None: """Hyphenate word""" cleaned_string, removed_indices, removed_symbols = clean_word(word) + + # Hyphenation of word longer than 100 characters throws exception + if len(cleaned_string) > 100: + return None + syllabus = hyphenator.syllables(cleaned_string) length = len(syllabus)