mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
New feature: Multilingual num2words
This commit is contained in:
1 parent
e394fe5f56
commit
cfdbfd2edf
7 files changed
+94
-11
No files matched your search
@@ -2,8 +2,7 @@
|
||||
|
||||
import unittest
|
||||
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
from src.modules.Speech_Recognition.Whisper import convert_to_transcribed_data
|
||||
|
||||
from src.modules.Speech_Recognition.Whisper import convert_to_transcribed_data, number_to_words
|
||||
|
||||
class ConvertToTranscribedDataTest(unittest.TestCase):
|
||||
def test_convert_to_transcribed_data(self):
|
||||
@@ -50,6 +49,36 @@ class ConvertToTranscribedDataTest(unittest.TestCase):
|
||||
self.assertEqual(transcribed_data[i].start, expected_output[i].start)
|
||||
self.assertEqual(transcribed_data[i].is_hyphen, expected_output[i].is_hyphen)
|
||||
|
||||
def test_number_to_words_converts(self):
|
||||
#Original, test with no language passed
|
||||
self.act_and_assert("I have 1 million dollars and 2 cents.", "I have one million dollars and two cents.")
|
||||
self.act_and_assert("1 2 3 4 5", "one two three four five")
|
||||
self.act_and_assert("1, 2, 3, 4, 5,", "one, two, three, four, five,")
|
||||
self.act_and_assert("Hello world 1, 2!. 3. 4? Test 100#",
|
||||
"Hello world one, two!. three. four? Test one hundred#")
|
||||
#Test English
|
||||
self.act_and_assert("I have 1 million dollars and 2 cents.", "I have one million dollars and two cents.", "en")
|
||||
self.act_and_assert("1 2 3 4 5", "one two three four five", "en")
|
||||
self.act_and_assert("1, 2, 3, 4, 5,", "one, two, three, four, five,", "en")
|
||||
self.act_and_assert("Hello world 1, 2!. 3. 4? Test 100#",
|
||||
"Hello world one, two!. three. four? Test one hundred#", "en")
|
||||
#Test German
|
||||
self.act_and_assert("1 2 3 4 5", "eins zwei drei vier fünf" ,"de")
|
||||
self.act_and_assert("1, 2, 3, 4, 5","eins, zwei, drei, vier, fünf","de")
|
||||
self.act_and_assert("Ich habe 1 Million Dollar und 2 Cent.","Ich habe eins Million Dollar und zwei Cent.","de")
|
||||
self.act_and_assert("Hallo Welt 1, 2!. 3. 4? Test 100#","Hallo Welt eins, zwei!. drei. vier? Test einhundert#","de")
|
||||
#Test Spanish
|
||||
self.act_and_assert("1 2 3 4 5","uno dos tres cuatro cinco","es")
|
||||
self.act_and_assert("1, 2, 3, 4, 5","uno, dos, tres, cuatro, cinco","es")
|
||||
self.act_and_assert("Tengo un millón de dólares y 2 centavos","Tengo un millón de dólares y dos centavos","es")
|
||||
self.act_and_assert("Hola mundo 1, 2!. 3. 4? Prueba 100#","Hola mundo uno, dos!. tres. cuatro? Prueba cien#","es")
|
||||
|
||||
|
||||
def act_and_assert(self, text, expected_output, language="en"):
|
||||
# Act
|
||||
result = number_to_words(text, language)
|
||||
|
||||
# Assert
|
||||
self.assertEqual(result, expected_output)
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+2
-1
@@ -29,4 +29,5 @@ packaging
|
||||
music21
|
||||
unidecode
|
||||
dataclasses
|
||||
dataclasses_json
|
||||
dataclasses_json
|
||||
num2words
|
||||
@@ -86,6 +86,7 @@ dill==0.3.8
|
||||
# via pylint
|
||||
docopt==0.6.2
|
||||
# via
|
||||
# num2words
|
||||
# pyannote-metrics
|
||||
# pyannote-pipeline
|
||||
dora-search==0.1.12
|
||||
@@ -264,6 +265,8 @@ networkx==3.3
|
||||
# via torch
|
||||
nltk==3.8.1
|
||||
# via whisperx
|
||||
num2words==0.5.13
|
||||
# via -r requirements.in
|
||||
numba==0.60.0
|
||||
# via
|
||||
# librosa
|
||||
|
||||
@@ -39,6 +39,7 @@ class Settings:
|
||||
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
|
||||
whisper_batch_size = 16 # reduce if low on GPU mem
|
||||
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
keep_numbers = False
|
||||
|
||||
# Pitch
|
||||
crepe_model_capacity = "full" # tiny|small|medium|large|full
|
||||
|
||||
+10
-3
@@ -129,9 +129,11 @@ def remove_unecessary_punctuations(transcribed_data: list[TranscribedData]) -> N
|
||||
|
||||
def run() -> tuple[str, Score, Score]:
|
||||
"""The processing function of this program"""
|
||||
|
||||
#List selected options (can add more later)
|
||||
if settings.keep_numbers:
|
||||
print(f"{ULTRASINGER_HEAD} {cyan_highlighted('Option:')} Numbers will be transcribed as numerics (i.e. 1, 2, 3, etc.)")
|
||||
process_data = InitProcessData()
|
||||
|
||||
|
||||
process_data.process_data_paths.cache_folder_path = (
|
||||
os.path.join(settings.output_folder_path, "cache")
|
||||
if settings.cache_override_path is None
|
||||
@@ -479,6 +481,7 @@ def transcribe_audio(cache_folder_path: str, processing_audio_path: str) -> Tran
|
||||
settings.whisper_batch_size,
|
||||
settings.whisper_compute_type,
|
||||
settings.language,
|
||||
settings.keep_numbers,
|
||||
)
|
||||
with open(transcription_path, "w", encoding=FILE_ENCODING) as file:
|
||||
file.write(transcription_result.to_json())
|
||||
@@ -603,6 +606,8 @@ def init_settings(argv: list[str]) -> Settings:
|
||||
settings.whisper_batch_size = int(arg)
|
||||
elif opt in ("--whisper_compute_type"):
|
||||
settings.whisper_compute_type = arg
|
||||
elif opt in ("--keep_numbers"):
|
||||
settings.keep_numbers = arg in ["True", "true"]
|
||||
elif opt in ("--language"):
|
||||
settings.language = arg
|
||||
elif opt in ("--crepe"):
|
||||
@@ -667,6 +672,7 @@ def arg_options():
|
||||
"ofile=",
|
||||
"crepe=",
|
||||
"crepe_step_size=",
|
||||
"demucs=",
|
||||
"whisper=",
|
||||
"whisper_align_model=",
|
||||
"whisper_batch_size=",
|
||||
@@ -684,7 +690,8 @@ def arg_options():
|
||||
"force_crepe_cpu=",
|
||||
"format_version=",
|
||||
"keep_cache",
|
||||
"musescore_path="
|
||||
"musescore_path=",
|
||||
"keep_numbers="
|
||||
]
|
||||
return long, short
|
||||
|
||||
|
||||
@@ -11,6 +11,14 @@ from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData, from_whisper
|
||||
|
||||
#Addition for numbers to words
|
||||
import re
|
||||
import ast
|
||||
from num2words import num2words
|
||||
|
||||
#Addition for numbers to words
|
||||
re_split_preserve_space = re.compile(r'(\d+|\W+|\w+)')
|
||||
|
||||
|
||||
MEMORY_ERROR_MESSAGE = f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu"
|
||||
|
||||
@@ -24,6 +32,26 @@ class WhisperModel(Enum):
|
||||
LARGE_V2 = "large-v2"
|
||||
LARGE_V3 = "large-v3"
|
||||
|
||||
#Addition for numbers to words (Using previous code from louispan in PR#135)
|
||||
def number_to_words(line,language='en'):
|
||||
# https://github.com/m-bain/whisperX
|
||||
# Transcript words which do not contain characters in the alignment models dictionary e.g. "2014." or "£13.60" cannot be aligned and therefore are not given a timing.
|
||||
# Therefore, convert numbers to words
|
||||
out_tokens = []
|
||||
in_tokens = re_split_preserve_space.findall(line)
|
||||
for token in in_tokens:
|
||||
try:
|
||||
num = ast.literal_eval(token)
|
||||
try:
|
||||
out_tokens.append(num2words(num, lang=language))
|
||||
except NotImplementedError:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language for number transcription. Keeping number as numeric characters for line: {line}, token: {token}"
|
||||
)
|
||||
except Exception:
|
||||
out_tokens.append(token)
|
||||
return ''.join(out_tokens)
|
||||
|
||||
def transcribe_with_whisper(
|
||||
audio_path: str,
|
||||
model: WhisperModel,
|
||||
@@ -32,6 +60,7 @@ def transcribe_with_whisper(
|
||||
batch_size: int = 16,
|
||||
compute_type: str = None,
|
||||
language: str = None,
|
||||
keep_numbers: bool = False,
|
||||
) -> TranscriptionResult:
|
||||
"""Transcribe with whisper"""
|
||||
|
||||
@@ -78,6 +107,11 @@ def transcribe_with_whisper(
|
||||
)
|
||||
raise ve
|
||||
|
||||
#Addition for numbers to words (Using previous code from louispan in PR#135)
|
||||
if keep_numbers == False:
|
||||
for obj in result["segments"]:
|
||||
obj["text"] = number_to_words(obj["text"],language)
|
||||
|
||||
# align whisper output
|
||||
result_aligned = whisperx.align(
|
||||
result["segments"],
|
||||
@@ -120,11 +154,18 @@ def convert_to_transcribed_data(result_aligned):
|
||||
vtd = from_whisper(obj) # create custom Word object
|
||||
vtd.word = vtd.word + " " # add space to end of word
|
||||
if len(obj) < 4:
|
||||
previous = transcribed_data[-1] if len(transcribed_data) != 0 else TranscribedData()
|
||||
vtd.start = previous.end + 0.1
|
||||
vtd.end = previous.end + 0.2
|
||||
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
|
||||
f'Fixing it by placing it after the previous word: "{previous.word}". At start: {vtd.start} end: {vtd.end}. Fix it manually!'
|
||||
#Addition for numbers to words (Using previous code from louispan in PR#135)
|
||||
if len(transcribed_data) == 0: # if the first word doesn't have any timing data
|
||||
vtd.start = 0.0
|
||||
vtd.end = 0.1
|
||||
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
|
||||
f'Fixing it by placing it at beginning. At start: {vtd.start} end: {vtd.end}. Fix it manually!'
|
||||
else:
|
||||
previous = transcribed_data[-1] if len(transcribed_data) != 0 else TranscribedData()
|
||||
vtd.start = previous.end + 0.1
|
||||
vtd.end = previous.end + 0.2
|
||||
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
|
||||
f'Fixing it by placing it after the previous word: "{previous.word}". At start: {vtd.start} end: {vtd.end}. Fix it manually!'
|
||||
print(f"{red_highlighted(msg)}")
|
||||
transcribed_data.append(vtd) # and add it to list
|
||||
return transcribed_data
|
||||
@@ -39,6 +39,7 @@ def print_help() -> None:
|
||||
--language Override the language detected by whisper, does not affect transcription but steps after transcription
|
||||
--whisper_batch_size Reduce if low on GPU mem >> ((default) is 16)
|
||||
--whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
|
||||
--keep_numbers Numbers will be transcribed as numerics instead of as words >> True|False >> ((default) is False)
|
||||
|
||||
[pitcher]
|
||||
# Default is crepe
|
||||
|
||||
Reference in new issue
Block a user