Merge pull request #62 from rakuri255/other_language_support_for_whisper

Other language support for whisper
This commit is contained in:
Vadim Rangnau authored and GitHub committed 2023-06-17 13:42:31 +02:00
commit d00b063e4b
4 files changed
+36 -10

No files matched your search

+18 -5
View File
@@ -112,9 +112,11 @@ _Not all options working now!_
[transcription]
# Default is whisper
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--vosk Needs model
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--align_model Use other languages model for Whisper provided from huggingface.co
--vosk Needs model
[pitcher]
# Default is crepe
@@ -163,12 +165,23 @@ Also keep in mind that while a larger model is more accurate, it also takes long
#### Whisper
For the first test run, use the `tiny`, to be accurate use the `large-v2` model
For the first test run, use the `tiny`, to be accurate use the `large-v2` model.
```commandline
-i XYZ --whisper large
```
##### Whisper languages
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
If the language is not in this list, you need to find a phoneme-based ASR model from
[huggingface model hub](https://huggingface.co). It will download automatically.
Example for romanian:
```commandline
-i XYZ --align_model "gigant/romanian-wav2vec2"
```
#### Vosk
If you want to use `Vosk` than you need the model. It is not included. You can download it here [Link](https://alphacephei.com/vosk/models).
@@ -247,7 +260,7 @@ But you can force cpu usage with the extra options `--force_whisper_cpu` and `--
With an GPU you can speed up the process and also the quality of the transcription and pitching is better.
You need a cuda device for this to work.
If you use MAC than sorry, there is no cuda device for MAC machinges.
If you use MAC than sorry, there is no cuda device for MAC machines.
#### Info
+1
View File
@@ -17,6 +17,7 @@ class Settings:
transcriber = "whisper" # whisper|vosk
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
# Vosk
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
+4 -1
View File
@@ -400,7 +400,7 @@ def transcribe_audio() -> (str, list[TranscribedData]):
if settings.transcriber == "whisper":
device = "cpu" if settings.force_whisper_cpu else settings.device
transcribed_data, language = transcribe_with_whisper(
settings.mono_audio_path, settings.whisper_model, device)
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
else: # vosk
transcribed_data = transcribe_with_vosk(
settings.mono_audio_path, settings.vosk_model_path
@@ -729,6 +729,8 @@ def init_settings(argv: list[str]) -> None:
elif opt in ("--whisper"):
settings.transcriber = "whisper"
settings.whisper_model = arg
elif opt in ("--align_model"):
settings.whisper_align_model = arg
elif opt in ("--vosk"):
settings.transcriber = "vosk"
settings.vosk_model_path = arg
@@ -765,6 +767,7 @@ def arg_options():
"crepe=",
"vosk=",
"whisper=",
"align_model=",
"plot=",
"hyphenation=",
"disable_separation=",
+13 -4
View File
@@ -1,6 +1,7 @@
"""Whisper Speech Recognition Module"""
import whisperx
import sys
from src.modules.console_colors import (
ULTRASINGER_HEAD,
@@ -10,12 +11,14 @@ from src.modules.console_colors import (
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[TranscribedData], str):
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (list[TranscribedData], str):
"""Transcribe with whisper"""
print(
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
)
if model_name is not None:
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
batch_size = 16 # reduce if low on GPU mem
compute_type = (
@@ -26,6 +29,7 @@ def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[
loaded_whisper_model = whisperx.load_model(
model, device=device, compute_type=compute_type
)
audio = whisperx.load_audio(audio_path)
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
@@ -34,9 +38,14 @@ def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[
language = result["language"]
# load alignment model and metadata
model_a, metadata = whisperx.load_align_model(
language_code=language, device=device
)
try:
model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name)
except ValueError as ve:
print(f"{red_highlighted(f'{ve}')}"
f"\n"
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
f"Try add it with --align_model [hugingface].")
sys.exit(0)
# align whisper output
result_aligned = whisperx.align(