- Other language support

This commit is contained in:
Vadim committed 2023-06-17 13:41:01 +02:00
1 parent 9ee351e742
commit 1d1ec8da08
4 files changed
+35 -9

No files matched your search

+17 -4
View File
@@ -112,9 +112,11 @@ _Not all options working now!_
[transcription]
# Default is whisper
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--vosk Needs model
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--align_model Use other languages model for Whisper provided from huggingface.co
--vosk Needs model
[pitcher]
# Default is crepe
@@ -163,12 +165,23 @@ Also keep in mind that while a larger model is more accurate, it also takes long
#### Whisper
For the first test run, use the `tiny`, to be accurate use the `large-v2` model
For the first test run, use the `tiny`, to be accurate use the `large-v2` model.
```commandline
-i XYZ --whisper large
```
##### Whisper languages
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
If the language is not in this list, you need to find a phoneme-based ASR model from
[huggingface model hub](https://huggingface.co). It will download automatically.
Example for romanian:
```commandline
-i XYZ --align_model "gigant/romanian-wav2vec2"
```
#### Vosk
If you want to use `Vosk` than you need the model. It is not included. You can download it here [Link](https://alphacephei.com/vosk/models).
+1
View File
@@ -17,6 +17,7 @@ class Settings:
transcriber = "whisper" # whisper|vosk
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
# Vosk
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
+4 -1
View File
@@ -400,7 +400,7 @@ def transcribe_audio() -> (str, list[TranscribedData]):
if settings.transcriber == "whisper":
device = "cpu" if settings.force_whisper_cpu else settings.device
transcribed_data, language = transcribe_with_whisper(
settings.mono_audio_path, settings.whisper_model, device)
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
else: # vosk
transcribed_data = transcribe_with_vosk(
settings.mono_audio_path, settings.vosk_model_path
@@ -729,6 +729,8 @@ def init_settings(argv: list[str]) -> None:
elif opt in ("--whisper"):
settings.transcriber = "whisper"
settings.whisper_model = arg
elif opt in ("--align_model"):
settings.whisper_align_model = arg
elif opt in ("--vosk"):
settings.transcriber = "vosk"
settings.vosk_model_path = arg
@@ -765,6 +767,7 @@ def arg_options():
"crepe=",
"vosk=",
"whisper=",
"align_model=",
"plot=",
"hyphenation=",
"disable_separation=",
+13 -4
View File
@@ -1,6 +1,7 @@
"""Whisper Speech Recognition Module"""
import whisperx
import sys
from src.modules.console_colors import (
ULTRASINGER_HEAD,
@@ -10,12 +11,14 @@ from src.modules.console_colors import (
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[TranscribedData], str):
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (list[TranscribedData], str):
"""Transcribe with whisper"""
print(
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
)
if model_name is not None:
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
batch_size = 16 # reduce if low on GPU mem
compute_type = (
@@ -26,6 +29,7 @@ def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[
loaded_whisper_model = whisperx.load_model(
model, device=device, compute_type=compute_type
)
audio = whisperx.load_audio(audio_path)
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
@@ -34,9 +38,14 @@ def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[
language = result["language"]
# load alignment model and metadata
model_a, metadata = whisperx.load_align_model(
language_code=language, device=device
)
try:
model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name)
except ValueError as ve:
print(f"{red_highlighted(f'{ve}')}"
f"\n"
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
f"Try add it with --align_model [hugingface].")
sys.exit(0)
# align whisper output
result_aligned = whisperx.align(