- Replaced Whisper-timestamp with whisperX

This commit is contained in:
Vadim committed 2023-05-29 13:53:23 +02:00
1 parent a15a1e6d5c
commit 21b136ccae
4 files changed
+15 -70

No files matched your search

+3 -2
View File
@@ -83,7 +83,8 @@ _Not all options working now!_
[transcription]
# Default is whisper
--whisper tiny|base|small|medium|large >> ((default) is large)
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
English-only model > tiny.en|base.en|small.en|medium.en
--vosk Needs model
[extra]
@@ -132,7 +133,7 @@ Also keep in mind that while a larger model is more accurate, it also takes long
#### Whisper
For the first test run, use the `tiny`, to be accurate use the `large` model
For the first test run, use the `tiny`, to be accurate use the `large-v2` model
```commandline
-i XYZ --whisper large
+1 -1
View File
@@ -15,7 +15,7 @@ class Settings:
# Whisper
transcriber = 'whisper' # whisper|vosk
whisper_model = 'large' # Multilingual model tiny|base|small|medium|large
whisper_model = 'large-v2' # Multilingual model tiny|base|small|medium|large-v1|large-v2
# English-only model tiny.en|base.en|small.en|medium.en
# Vosk
+11 -66
View File
@@ -1,88 +1,33 @@
import whisper_timestamped as whisper
import whisperx
from moduls.Speech_Recognition.TranscribedData import TranscribedData
from moduls.Log import PRINT_ULTRASTAR
from moduls.Log import print_blue_highlighted_text, print_red_highlighted_text
def transcribe_with_whisper_old(audioPath, model, device="cpu"):
def transcribe_with_whisper(audio_path, model, device="cpu"):
print(f"{PRINT_ULTRASTAR} Loading {print_blue_highlighted_text('whisper')} with model {print_blue_highlighted_text(model)} and {print_red_highlighted_text(device)} as worker")
model = whisper.load_model(model, device=device)
batch_size = 16 # reduce if low on GPU mem
compute_type = "float16" if device == "cuda" else "int8" # change to "int8" if low on GPU mem (may reduce accuracy)
# load audio and pad/trim it to fit 30 seconds
audio = whisper.load_audio(audioPath)
audio_30 = whisper.pad_or_trim(audio)
print(f"{PRINT_ULTRASTAR} Start detecting language")
# make log-Mel spectrogram and move to the same device as the model
mel = whisper.log_mel_spectrogram(audio_30).to(model.device)
# detect the spoken language
_, probs = model.detect_language(mel)
language = max(probs, key=probs.get)
print(f"{PRINT_ULTRASTAR} Detected language: {print_blue_highlighted_text(language)}")
print(f"{PRINT_ULTRASTAR} Transcribing {audioPath}")
results = whisper.transcribe(model, audio, language=language)
transcribed_data = []
for segment in results["segments"]:
# todo:
# if sentence != 'segments':
# continue
# to class
for obj in segment["words"]:
vtd = TranscribedData(obj) # create custom Word object
vtd.word = vtd.word + ' '
transcribed_data.append(vtd) # and add it to list
return transcribed_data, language
def transcribe_with_whisper(audioPath, model, device="cpu"):
print("{} Loading {} with model {} and {} as worker".format(PRINT_ULTRASTAR, print_blue_highlighted_text("whisper"),
print_blue_highlighted_text(model),
print_red_highlighted_text(device)))
# transcribe with original whisper
# todo compute_type int8 for cpu?
model = whisperx.load_model("tiny", device=device, compute_type="int8")
loaded_whisper_model = whisperx.load_model(model, device=device, compute_type=compute_type)
audio = whisperx.load_audio(audio_path)
# load audio and pad/trim it to fit 30 seconds
audio = whisperx.load_audio(audioPath)
audio_30 = whisperx.pad_or_trim(audio)
print(f"{PRINT_ULTRASTAR} Transcribing {audio_path}")
# make log-Mel spectrogram and move to the same device as the model
mel = whisperx.log_mel_spectrogram(audio_30).to(model.device)
# detect the spoken language
_, probs = model.detect_language(mel)
language = max(probs, key=probs.get)
print(f"{PRINT_ULTRASTAR} Detected language: {print_blue_highlighted_text(language)}")
print("{} Transcribing {}".format(PRINT_ULTRASTAR, audioPath))
results = whisperx.transcribe(model, audio, language=language)
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
language = result["language"]
# load alignment model and metadata
model_a, metadata = whisperx.load_align_model(language_code=results["language"], device=device)
model_a, metadata = whisperx.load_align_model(language_code=language, device=device)
# align whisper output
result_aligned = whisperx.align(results["segments"], model_a, metadata, audio, device)
print(result_aligned["segments"]) # after alignment
print(result_aligned["word_segments"]) # after alignment
result_aligned = whisperx.align(result["segments"], model_a, metadata, audio, device, return_char_alignments=False)
transcribed_data = []
for segment in results["segments"]:
# todo:
# if sentence != 'segments':
# continue
# to class
for segment in result_aligned["segments"]:
for obj in segment["words"]:
vtd = TranscribedData(obj) # create custom Word object
vtd.word = vtd.word + ' '
-1
View File
@@ -9,7 +9,6 @@ tensorflow~=2.12.0
matplotlib~=3.7.1
yt_dlp~=2023.3.4
numpy~=1.23.5
git+https://github.com/linto-ai/whisper-timestamped
git+https://github.com/m-bain/whisperx.git
PyHyphen~=4.0.3
demucs~=4.0.0