mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
add options for overriding language, whisper batch size and compute type
This commit is contained in:
1 parent
32fb5dd999
commit
6696b8be15
6 files changed
+39
-15
No files matched your search
@@ -7,3 +7,4 @@ venv
|
||||
|
||||
# default output folder
|
||||
output
|
||||
.vscode/
|
||||
@@ -114,8 +114,10 @@ _Not all options working now!_
|
||||
# Default is whisper
|
||||
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
|
||||
English-only model > tiny.en|base.en|small.en|medium.en
|
||||
--align_model Use other languages model for Whisper provided from huggingface.co
|
||||
|
||||
--align_model Use other languages model for Whisper provided from huggingface.co
|
||||
--language Override the language detected by whisper, does not affect transcription but steps after transcription
|
||||
--batch_size Reduce if low on GPU mem >> ((default) is 16)
|
||||
--compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
|
||||
--vosk Needs model
|
||||
|
||||
[pitcher]
|
||||
|
||||
@@ -10,6 +10,8 @@ class Settings:
|
||||
output_file_path = ""
|
||||
mono_audio_path = ""
|
||||
|
||||
language = None
|
||||
|
||||
# Transcribe
|
||||
audio_chunk_folder_name = "audio-chunks"
|
||||
|
||||
@@ -18,6 +20,8 @@ class Settings:
|
||||
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
|
||||
# English-only model tiny.en|base.en|small.en|medium.en
|
||||
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
|
||||
whisper_batch_size = 16 # reduce if low on GPU mem
|
||||
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
|
||||
# Vosk
|
||||
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
|
||||
|
||||
+17
-7
@@ -4,7 +4,6 @@ import copy
|
||||
import getopt
|
||||
import os
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
import Levenshtein
|
||||
import librosa
|
||||
@@ -230,7 +229,6 @@ def hyphenate_each_word(language: str, transcribed_data: list[TranscribedData])
|
||||
)
|
||||
return None
|
||||
|
||||
sleep(0.1)
|
||||
hyphenator = create_hyphenator(lang_region)
|
||||
for i in tqdm(enumerate(transcribed_data)):
|
||||
pos = i[0]
|
||||
@@ -309,9 +307,12 @@ def run() -> None:
|
||||
|
||||
# Audio transcription
|
||||
transcribed_data = None
|
||||
language = None
|
||||
language = settings.language
|
||||
if is_audio:
|
||||
language, transcribed_data = transcribe_audio()
|
||||
language_from_transcription, transcribed_data = transcribe_audio()
|
||||
if language is None:
|
||||
language = language_from_transcription
|
||||
|
||||
remove_unecessary_punctuations(transcribed_data)
|
||||
transcribed_data = remove_silence_from_transcribtion_data(
|
||||
settings.mono_audio_path, transcribed_data
|
||||
@@ -410,13 +411,13 @@ def transcribe_audio() -> (str, list[TranscribedData]):
|
||||
if settings.transcriber == "whisper":
|
||||
device = "cpu" if settings.force_whisper_cpu else settings.device
|
||||
transcribed_data, language = transcribe_with_whisper(
|
||||
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
|
||||
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model, settings.whisper_batch_size, settings.whisper_compute_type, settings.language)
|
||||
else: # vosk
|
||||
transcribed_data = transcribe_with_vosk(
|
||||
settings.mono_audio_path, settings.vosk_model_path
|
||||
)
|
||||
# todo: make language selectable
|
||||
language = "en"
|
||||
if settings.language is None:
|
||||
language = "en"
|
||||
return language, transcribed_data
|
||||
|
||||
|
||||
@@ -779,6 +780,12 @@ def init_settings(argv: list[str]) -> None:
|
||||
settings.whisper_model = arg
|
||||
elif opt in ("--align_model"):
|
||||
settings.whisper_align_model = arg
|
||||
elif opt in ("--batch_size"):
|
||||
settings.whisper_batch_size = int(arg)
|
||||
elif opt in ("--compute_type"):
|
||||
settings.whisper_compute_type = arg
|
||||
elif opt in ("--language"):
|
||||
settings.language = arg
|
||||
elif opt in ("--vosk"):
|
||||
settings.transcriber = "vosk"
|
||||
settings.vosk_model_path = arg
|
||||
@@ -816,6 +823,9 @@ def arg_options():
|
||||
"vosk=",
|
||||
"whisper=",
|
||||
"align_model=",
|
||||
"batch_size=",
|
||||
"compute_type=",
|
||||
"language=",
|
||||
"plot=",
|
||||
"hyphenation=",
|
||||
"disable_separation=",
|
||||
|
||||
@@ -11,7 +11,7 @@ from modules.console_colors import (
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (
|
||||
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None, batch_size: int = 16, compute_type: str = None, language: str = None) -> (
|
||||
list[TranscribedData], str):
|
||||
"""Transcribe with whisper"""
|
||||
|
||||
@@ -21,10 +21,10 @@ list[TranscribedData], str):
|
||||
if model_name is not None:
|
||||
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
|
||||
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
compute_type = (
|
||||
"float16" if device == "cuda" else "int8"
|
||||
) # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
if compute_type is None:
|
||||
compute_type = (
|
||||
"float16" if device == "cuda" else "int8"
|
||||
)
|
||||
|
||||
loaded_whisper_model = whisperx.load_model(
|
||||
model, device=device, compute_type=compute_type
|
||||
@@ -35,7 +35,9 @@ list[TranscribedData], str):
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
|
||||
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
|
||||
language = result["language"]
|
||||
|
||||
if language is None:
|
||||
language = result["language"]
|
||||
|
||||
# load alignment model and metadata
|
||||
try:
|
||||
|
||||
@@ -88,6 +88,11 @@ def hyphenation(word: str, hyphenator: Hyphenator) -> list[str] | None:
|
||||
"""Hyphenate word"""
|
||||
|
||||
cleaned_string, removed_indices, removed_symbols = clean_word(word)
|
||||
|
||||
# Hyphenation of word longer than 100 characters throws exception
|
||||
if len(cleaned_string) > 100:
|
||||
return None
|
||||
|
||||
syllabus = hyphenator.syllables(cleaned_string)
|
||||
|
||||
length = len(syllabus)
|
||||
|
||||
Reference in new issue
Block a user