add options for overriding language, whisper batch size and compute type

This commit is contained in:
Benedikt Wagener committed 2023-09-02 20:18:09 +02:00
1 parent 32fb5dd999
commit 6696b8be15
6 files changed
+39 -15

No files matched your search

+1
View File
@@ -7,3 +7,4 @@ venv
# default output folder
output
.vscode/
+4 -2
View File
@@ -114,8 +114,10 @@ _Not all options working now!_
# Default is whisper
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--align_model Use other languages model for Whisper provided from huggingface.co
--align_model Use other languages model for Whisper provided from huggingface.co
--language Override the language detected by whisper, does not affect transcription but steps after transcription
--batch_size Reduce if low on GPU mem >> ((default) is 16)
--compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
--vosk Needs model
[pitcher]
+4
View File
@@ -10,6 +10,8 @@ class Settings:
output_file_path = ""
mono_audio_path = ""
language = None
# Transcribe
audio_chunk_folder_name = "audio-chunks"
@@ -18,6 +20,8 @@ class Settings:
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
whisper_batch_size = 16 # reduce if low on GPU mem
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
# Vosk
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
+17 -7
View File
@@ -4,7 +4,6 @@ import copy
import getopt
import os
import sys
from time import sleep
import Levenshtein
import librosa
@@ -230,7 +229,6 @@ def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData])
)
return None
sleep(0.1)
hyphenator = create_hyphenator(lang_region)
for i in tqdm(enumerate(transcribed_data)):
pos = i[0]
@@ -309,9 +307,12 @@ def run() -> None:
# Audio transcription
transcribed_data = None
language = None
language = settings.language
if is_audio:
language, transcribed_data = transcribe_audio()
language_from_transcription, transcribed_data = transcribe_audio()
if language is None:
language = language_from_transcription
remove_unecessary_punctuations(transcribed_data)
transcribed_data = remove_silence_from_transcribtion_data(
settings.mono_audio_path, transcribed_data
@@ -410,13 +411,13 @@ def transcribe_audio() -> (str, list[TranscribedData]):
if settings.transcriber == "whisper":
device = "cpu" if settings.force_whisper_cpu else settings.device
transcribed_data, language = transcribe_with_whisper(
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model, settings.whisper_batch_size, settings.whisper_compute_type, settings.language)
else: # vosk
transcribed_data = transcribe_with_vosk(
settings.mono_audio_path, settings.vosk_model_path
)
# todo: make language selectable
language = "en"
if settings.language is None:
language = "en"
return language, transcribed_data
@@ -779,6 +780,12 @@ def init_settings(argv: list[str]) -> None:
settings.whisper_model = arg
elif opt in ("--align_model"):
settings.whisper_align_model = arg
elif opt in ("--batch_size"):
settings.whisper_batch_size = int(arg)
elif opt in ("--compute_type"):
settings.whisper_compute_type = arg
elif opt in ("--language"):
settings.language = arg
elif opt in ("--vosk"):
settings.transcriber = "vosk"
settings.vosk_model_path = arg
@@ -816,6 +823,9 @@ def arg_options():
"vosk=",
"whisper=",
"align_model=",
"batch_size=",
"compute_type=",
"language=",
"plot=",
"hyphenation=",
"disable_separation=",
+8 -6
View File
@@ -11,7 +11,7 @@ from modules.console_colors import (
from modules.Speech_Recognition.TranscribedData import TranscribedData
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None, batch_size: int = 16, compute_type: str = None, language: str = None) -> (
list[TranscribedData], str):
"""Transcribe with whisper"""
@@ -21,10 +21,10 @@ list[TranscribedData], str):
if model_name is not None:
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
batch_size = 16 # reduce if low on GPU mem
compute_type = (
"float16" if device == "cuda" else "int8"
) # change to "int8" if low on GPU mem (may reduce accuracy)
if compute_type is None:
compute_type = (
"float16" if device == "cuda" else "int8"
)
loaded_whisper_model = whisperx.load_model(
model, device=device, compute_type=compute_type
@@ -35,7 +35,9 @@ list[TranscribedData], str):
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
language = result["language"]
if language is None:
language = result["language"]
# load alignment model and metadata
try:
@@ -88,6 +88,11 @@ def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None:
"""Hyphenate word"""
cleaned_string, removed_indices, removed_symbols = clean_word(word)
# Hyphenation of word longer than 100 characters throws exception
if len(cleaned_string) > 100:
return None
syllabus = hyphenator.syllables(cleaned_string)
length = len(syllabus)