mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
- Replaced Whisper-timestamp with whisperX
This commit is contained in:
1 parent
a15a1e6d5c
commit
21b136ccae
4 files changed
+15
-70
No files matched your search
@@ -83,7 +83,8 @@ _Not all options working now!_
|
||||
|
||||
[transcription]
|
||||
# Default is whisper
|
||||
--whisper tiny|base|small|medium|large >> ((default) is large)
|
||||
--whisper Multilingual model > tiny|base|small|medium|large-v1|large-v2 >> ((default) is large-v2
|
||||
English-only model > tiny.en|base.en|small.en|medium.en
|
||||
--vosk Needs model
|
||||
|
||||
[extra]
|
||||
@@ -132,7 +133,7 @@ Also keep in mind that while a larger model is more accurate, it also takes long
|
||||
|
||||
#### Whisper
|
||||
|
||||
For the first test run, use the `tiny`, to be accurate use the `large` model
|
||||
For the first test run, use the `tiny`, to be accurate use the `large-v2` model
|
||||
|
||||
```commandline
|
||||
-i XYZ --whisper large
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ class Settings:
|
||||
|
||||
# Whisper
|
||||
transcriber = 'whisper' # whisper|vosk
|
||||
whisper_model = 'large' # Multilingual model tiny|base|small|medium|large
|
||||
whisper_model = 'large-v2' # Multilingual model tiny|base|small|medium|large-v1|large-v2
|
||||
# English-only model tiny.en|base.en|small.en|medium.en
|
||||
|
||||
# Vosk
|
||||
|
||||
@@ -1,88 +1,33 @@
|
||||
import whisper_timestamped as whisper
|
||||
import whisperx
|
||||
from moduls.Speech_Recognition.TranscribedData import TranscribedData
|
||||
from moduls.Log import PRINT_ULTRASTAR
|
||||
from moduls.Log import print_blue_highlighted_text, print_red_highlighted_text
|
||||
|
||||
|
||||
def transcribe_with_whisper_old(audioPath, model, device="cpu"):
|
||||
def transcribe_with_whisper(audio_path, model, device="cpu"):
|
||||
print(f"{PRINT_ULTRASTAR} Loading {print_blue_highlighted_text('whisper')} with model {print_blue_highlighted_text(model)} and {print_red_highlighted_text(device)} as worker")
|
||||
|
||||
model = whisper.load_model(model, device=device)
|
||||
batch_size = 16 # reduce if low on GPU mem
|
||||
compute_type = "float16" if device == "cuda" else "int8" # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
|
||||
# load audio and pad/trim it to fit 30 seconds
|
||||
audio = whisper.load_audio(audioPath)
|
||||
audio_30 = whisper.pad_or_trim(audio)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Start detecting language")
|
||||
|
||||
# make log-Mel spectrogram and move to the same device as the model
|
||||
mel = whisper.log_mel_spectrogram(audio_30).to(model.device)
|
||||
|
||||
# detect the spoken language
|
||||
_, probs = model.detect_language(mel)
|
||||
language = max(probs, key=probs.get)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Detected language: {print_blue_highlighted_text(language)}")
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Transcribing {audioPath}")
|
||||
results = whisper.transcribe(model, audio, language=language)
|
||||
|
||||
transcribed_data = []
|
||||
|
||||
for segment in results["segments"]:
|
||||
# todo:
|
||||
# if sentence != 'segments':
|
||||
# continue
|
||||
# to class
|
||||
for obj in segment["words"]:
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
vtd.word = vtd.word + ' '
|
||||
transcribed_data.append(vtd) # and add it to list
|
||||
|
||||
return transcribed_data, language
|
||||
|
||||
|
||||
def transcribe_with_whisper(audioPath, model, device="cpu"):
|
||||
print("{} Loading {} with model {} and {} as worker".format(PRINT_ULTRASTAR, print_blue_highlighted_text("whisper"),
|
||||
print_blue_highlighted_text(model),
|
||||
print_red_highlighted_text(device)))
|
||||
# transcribe with original whisper
|
||||
# todo compute_type int8 for cpu?
|
||||
model = whisperx.load_model("tiny", device=device, compute_type="int8")
|
||||
loaded_whisper_model = whisperx.load_model(model, device=device, compute_type=compute_type)
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
# load audio and pad/trim it to fit 30 seconds
|
||||
audio = whisperx.load_audio(audioPath)
|
||||
audio_30 = whisperx.pad_or_trim(audio)
|
||||
print(f"{PRINT_ULTRASTAR} Transcribing {audio_path}")
|
||||
|
||||
# make log-Mel spectrogram and move to the same device as the model
|
||||
mel = whisperx.log_mel_spectrogram(audio_30).to(model.device)
|
||||
|
||||
# detect the spoken language
|
||||
_, probs = model.detect_language(mel)
|
||||
language = max(probs, key=probs.get)
|
||||
|
||||
print(f"{PRINT_ULTRASTAR} Detected language: {print_blue_highlighted_text(language)}")
|
||||
print("{} Transcribing {}".format(PRINT_ULTRASTAR, audioPath))
|
||||
|
||||
results = whisperx.transcribe(model, audio, language=language)
|
||||
result = loaded_whisper_model.transcribe(audio, batch_size=batch_size)
|
||||
language = result["language"]
|
||||
|
||||
# load alignment model and metadata
|
||||
model_a, metadata = whisperx.load_align_model(language_code=results["language"], device=device)
|
||||
model_a, metadata = whisperx.load_align_model(language_code=language, device=device)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(results["segments"], model_a, metadata, audio, device)
|
||||
|
||||
print(result_aligned["segments"]) # after alignment
|
||||
print(result_aligned["word_segments"]) # after alignment
|
||||
result_aligned = whisperx.align(result["segments"], model_a, metadata, audio, device, return_char_alignments=False)
|
||||
|
||||
transcribed_data = []
|
||||
|
||||
for segment in results["segments"]:
|
||||
# todo:
|
||||
# if sentence != 'segments':
|
||||
# continue
|
||||
# to class
|
||||
for segment in result_aligned["segments"]:
|
||||
for obj in segment["words"]:
|
||||
vtd = TranscribedData(obj) # create custom Word object
|
||||
vtd.word = vtd.word + ' '
|
||||
|
||||
@@ -9,7 +9,6 @@ tensorflow~=2.12.0
|
||||
matplotlib~=3.7.1
|
||||
yt_dlp~=2023.3.4
|
||||
numpy~=1.23.5
|
||||
git+https://github.com/linto-ai/whisper-timestamped
|
||||
git+https://github.com/m-bain/whisperx.git
|
||||
PyHyphen~=4.0.3
|
||||
demucs~=4.0.0
|
||||
|
||||
Reference in new issue
Block a user