mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Merge pull request #197 from agwosdz/feature_lowconfidence_option
MonkeyPatch FasterWhisperPipeline to detect low confidence
This commit is contained in:
4 files changed
+82
-5
No files matched your search
+2
-1
@@ -30,4 +30,5 @@ music21
|
||||
unidecode
|
||||
dataclasses
|
||||
dataclasses_json
|
||||
num2words
|
||||
num2words
|
||||
inputimeout
|
||||
+5
-1
@@ -71,7 +71,9 @@ contourpy==1.2.1
|
||||
crepe==0.0.15
|
||||
# via -r requirements.in
|
||||
ctranslate2==4.4.0
|
||||
# via faster-whisper
|
||||
# via
|
||||
# faster-whisper
|
||||
# whisperx
|
||||
cycler==0.12.1
|
||||
# via matplotlib
|
||||
dataclasses==0.6
|
||||
@@ -166,6 +168,8 @@ imageio==2.34.2
|
||||
# via crepe
|
||||
iniconfig==2.0.0
|
||||
# via pytest
|
||||
inputimeout==1.0.4
|
||||
# via -r requirements.in
|
||||
isort==5.13.2
|
||||
# via
|
||||
# -r requirements.in
|
||||
|
||||
@@ -11,6 +11,8 @@ from modules.Ultrastar.ultrastar_txt import FormatVersion
|
||||
@dataclass
|
||||
class Settings:
|
||||
APP_VERSION = "0.0.13-dev1"
|
||||
CONFIDENCE_THRESHOLD = 0.6
|
||||
CONFIDENCE_PROMPT_TIMEOUT = 4
|
||||
|
||||
create_midi = True
|
||||
create_plot = False
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Whisper Speech Recognition Module"""
|
||||
|
||||
import sys
|
||||
|
||||
import inspect
|
||||
import textwrap
|
||||
import torch
|
||||
import whisperx
|
||||
from enum import Enum
|
||||
@@ -52,6 +51,24 @@ def number_to_words(line,language='en'):
|
||||
out_tokens.append(token)
|
||||
return ''.join(out_tokens)
|
||||
|
||||
def replace_code_lines(source, start_token, end_token,
|
||||
replacement, escape_tokens=True):
|
||||
"""Replace the source code between `start_token` and `end_token`
|
||||
in `source` with `replacement`. The `start_token` portion is included
|
||||
in the replaced code. If `escape_tokens` is True (default),
|
||||
escape the tokens to avoid them being treated as a regular expression."""
|
||||
|
||||
if escape_tokens:
|
||||
start_token = re.escape(start_token)
|
||||
end_token = re.escape(end_token)
|
||||
|
||||
def replace_with_indent(match):
|
||||
indent = match.group(1)
|
||||
return textwrap.indent(replacement, indent)
|
||||
|
||||
return re.sub(r"^(\s+)({}[\s\S]+?)(?=^\1{})".format(start_token, end_token),
|
||||
replace_with_indent, source, flags=re.MULTILINE)
|
||||
|
||||
def transcribe_with_whisper(
|
||||
audio_path: str,
|
||||
model: WhisperModel,
|
||||
@@ -63,6 +80,59 @@ def transcribe_with_whisper(
|
||||
keep_numbers: bool = False,
|
||||
) -> TranscriptionResult:
|
||||
"""Transcribe with whisper"""
|
||||
# Info: Monkey Patch FasterWhisperPipeline.detect_language to include error handling for low confidence
|
||||
src = textwrap.dedent(inspect.getsource(whisperx.asr.FasterWhisperPipeline.detect_language))
|
||||
# Replace the relevant part of the method
|
||||
start_token = "if audio.shape[0] < N_SAMPLES:"
|
||||
end_token = "return language"
|
||||
replacement = """\
|
||||
#Added imports
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from Settings import Settings
|
||||
from inputimeout import inputimeout, TimeoutOccurred
|
||||
#End Import addition
|
||||
if audio.shape[0] < N_SAMPLES:
|
||||
print("Warning: audio is shorter than 30s, language detection may be inaccurate.")
|
||||
model_n_mels = self.model.feat_kwargs.get("feature_size")
|
||||
segment = log_mel_spectrogram(audio[: N_SAMPLES],
|
||||
n_mels=model_n_mels if model_n_mels is not None else 80,
|
||||
padding=0 if audio.shape[0] >= N_SAMPLES else N_SAMPLES - audio.shape[0])
|
||||
encoder_output = self.model.encode(segment)
|
||||
results = self.model.model.detect_language(encoder_output)
|
||||
language_token, language_probability = results[0][0]
|
||||
language = language_token[2:-2]
|
||||
print(f"Detected language: {language} ({language_probability:.2f}) in first 30s of audio...")
|
||||
#Added handling for low detection probability
|
||||
if language_probability < Settings.CONFIDENCE_THRESHOLD:
|
||||
print(f"{ULTRASINGER_HEAD} {red_highlighted('Warning:')} Language detection probability for detected language {language} is below {Settings.CONFIDENCE_THRESHOLD}, results may be inaccurate.")
|
||||
print(f"{ULTRASINGER_HEAD} Override the language below or re-run with parameter {blue_highlighted('--language xx')} to specify the song language...")
|
||||
try:
|
||||
response = inputimeout(
|
||||
prompt=f"{ULTRASINGER_HEAD} Do you want to continue with {language} (default) or override with another language (y)? (y/n): ",
|
||||
timeout=Settings.CONFIDENCE_PROMPT_TIMEOUT
|
||||
).strip().lower()
|
||||
except TimeoutOccurred:
|
||||
import locale
|
||||
print(f"{ULTRASINGER_HEAD} No user input received in {Settings.CONFIDENCE_PROMPT_TIMEOUT} seconds. Attempting automatic override with system locale.")
|
||||
print(f"{ULTRASINGER_HEAD} Trying to get language from default locale...")
|
||||
current_locale = locale.getlocale()
|
||||
if current_locale[0]:
|
||||
language_code = current_locale[0][:2].strip().lower()
|
||||
print(f"{ULTRASINGER_HEAD} Found language code: {language_code} in locale. Setting language to {blue_highlighted(language_code)}...")
|
||||
language = language_code
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} No locale is set.")
|
||||
response = 'n'
|
||||
language_response = response == 'y'
|
||||
if language_response:
|
||||
language = input(f"{ULTRASINGER_HEAD} Please enter the language code for the language you want to use (e.g. 'en', 'de', 'es', etc.): ").strip().lower()
|
||||
#End addition
|
||||
"""
|
||||
new_src = replace_code_lines(src, start_token, end_token, replacement)
|
||||
# Compile it and execute it in the target module's namespace
|
||||
exec(compile(new_src, "<string>", "exec"), whisperx.asr.__dict__)
|
||||
whisperx.asr.FasterWhisperPipeline.detect_language = whisperx.asr.detect_language
|
||||
#End Monkey Patch
|
||||
|
||||
# Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work.
|
||||
|
||||
|
||||
Reference in new issue
Block a user