mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Merge pull request #62 from rakuri255/other_language_support_for_whisper
Other language support for whisper
This commit is contained in:
4 files changed
+36
-10
No files matched your search
@@ -112,9 +112,11 @@ _Not all options working now!_
|
||||
|
||||
[transcription]
|
||||
# Default is whisper
|
||||
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
|
||||
English-only model > tiny.en|base.en|small.en|medium.en
|
||||
--vosk Needs model
|
||||
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
|
||||
English-only model > tiny.en|base.en|small.en|medium.en
|
||||
--align_model Use other languages model for Whisper provided from huggingface.co
|
||||
|
||||
--vosk Needs model
|
||||
|
||||
[pitcher]
|
||||
# Default is crepe
|
||||
@@ -163,12 +165,23 @@ Also keep in mind that while a larger model is more accurate, it also takes long
|
||||
|
||||
#### Whisper
|
||||
|
||||
For the first test run, use the `tiny`, to be accurate use the `large-v2` model
|
||||
For the first test run, use the `tiny`, to be accurate use the `large-v2` model.
|
||||
|
||||
```commandline
|
||||
-i XYZ --whisper large
|
||||
```
|
||||
|
||||
##### Whisper languages
|
||||
|
||||
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
|
||||
If the language is not in this list, you need to find a phoneme-based ASR model from
|
||||
[huggingface model hub](https://huggingface.co). It will download automatically.
|
||||
|
||||
Example for romanian:
|
||||
```commandline
|
||||
-i XYZ --align_model "gigant/romanian-wav2vec2"
|
||||
```
|
||||
|
||||
#### Vosk
|
||||
|
||||
If you want to use `Vosk` than you need the model. It is not included. You can download it here [Link](https://alphacephei.com/vosk/models).
|
||||
@@ -247,7 +260,7 @@ But you can force cpu usage with the extra options `--force_whisper_cpu` and `--
|
||||
|
||||
With an GPU you can speed up the process and also the quality of the transcription and pitching is better.
|
||||
You need a cuda device for this to work.
|
||||
If you use MAC than sorry, there is no cuda device for MAC machinges.
|
||||
If you use MAC than sorry, there is no cuda device for MAC machines.
|
||||
|
||||
#### Info
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ class Settings:
|
||||
transcriber = "whisper" # whisper|vosk
|
||||
whisper_model = "large-v2" # Multilingual model tiny|base|small|medium|large-v1|large-v2
|
||||
# English-only model tiny.en|base.en|small.en|medium.en
|
||||
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
|
||||
|
||||
# Vosk
|
||||
vosk_model_path = "" # "models/vosk-model-small-en-us-0.15"
|
||||
|
||||
+4
-1
@@ -400,7 +400,7 @@ def transcribe_audio() -> (str, list[TranscribedData]):
|
||||
if settings.transcriber == "whisper":
|
||||
device = "cpu" if settings.force_whisper_cpu else settings.device
|
||||
transcribed_data, language = transcribe_with_whisper(
|
||||
settings.mono_audio_path, settings.whisper_model, device)
|
||||
settings.mono_audio_path, settings.whisper_model, device, settings.whisper_align_model)
|
||||
else: # vosk
|
||||
transcribed_data = transcribe_with_vosk(
|
||||
settings.mono_audio_path, settings.vosk_model_path
|
||||
@@ -729,6 +729,8 @@ def init_settings(argv: list[str]) -> None:
|
||||
elif opt in ("--whisper"):
|
||||
settings.transcriber = "whisper"
|
||||
settings.whisper_model = arg
|
||||
elif opt in ("--align_model"):
|
||||
settings.whisper_align_model = arg
|
||||
elif opt in ("--vosk"):
|
||||
settings.transcriber = "vosk"
|
||||
settings.vosk_model_path = arg
|
||||
@@ -765,6 +767,7 @@ def arg_options():
|
||||
"crepe=",
|
||||
"vosk=",
|
||||
"whisper=",
|
||||
"align_model=",
|
||||
"plot=",
|
||||
"hyphenation=",
|
||||
"disable_separation=",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Whisper Speech Recognition Module"""
|
||||
|
||||
import whisperx
|
||||
import sys
|
||||
|
||||
from src.modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
@@ -10,12 +11,14 @@ from src.modules.console_colors import (
|
||||
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[TranscribedData], str):
|
||||
def transcribe_with_whisper(audio_path: str, model: str, device="cpu", model_name: str = None) -> (list[TranscribedData], str):
|
||||
"""Transcribe with whisper"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model)} and {red_highlighted(device)} as worker"
|
||||
)
|
||||
if model_name is not None:
|
||||
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(model_name)}")
|
||||
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
compute_type = (
|
||||
@@ -26,6 +29,7 @@ def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[
|
||||
loaded_whisper_model = whisperx.load_model(
|
||||
model, device=device, compute_type=compute_type
|
||||
)
|
||||
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Transcribing {audio_path}")
|
||||
@@ -34,9 +38,14 @@ def transcribe_with_whisper(audio_path: str, model: str, device="cpu") -> (list[
|
||||
language = result["language"]
|
||||
|
||||
# load alignment model and metadata
|
||||
model_a, metadata = whisperx.load_align_model(
|
||||
language_code=language, device=device
|
||||
)
|
||||
try:
|
||||
model_a, metadata = whisperx.load_align_model(language_code=language, device=device, model_name=model_name)
|
||||
except ValueError as ve:
|
||||
print(f"{red_highlighted(f'{ve}')}"
|
||||
f"\n"
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language. "
|
||||
f"Try add it with --align_model [hugingface].")
|
||||
sys.exit(0)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(
|
||||
|
||||
Reference in new issue
Block a user