New feature: Multilingual num2words

This commit is contained in:
TheZet committed 2024-12-13 15:07:58 -05:00
1 parent e394fe5f56
commit cfdbfd2edf
7 files changed
+94 -11

No files matched your search

@@ -2,8 +2,7 @@
import unittest
from src.modules.Speech_Recognition.TranscribedData import TranscribedData
from src.modules.Speech_Recognition.Whisper import convert_to_transcribed_data
from src.modules.Speech_Recognition.Whisper import convert_to_transcribed_data, number_to_words
class ConvertToTranscribedDataTest(unittest.TestCase):
def test_convert_to_transcribed_data(self):
@@ -50,6 +49,36 @@ class ConvertToTranscribedDataTest(unittest.TestCase):
self.assertEqual(transcribed_data[i].start, expected_output[i].start)
self.assertEqual(transcribed_data[i].is_hyphen, expected_output[i].is_hyphen)
def test_number_to_words_converts(self):
#Original, test with no language passed
self.act_and_assert("I have 1 million dollars and 2 cents.", "I have one million dollars and two cents.")
self.act_and_assert("1 2 3 4 5", "one two three four five")
self.act_and_assert("1, 2, 3, 4, 5,", "one, two, three, four, five,")
self.act_and_assert("Hello world 1, 2!. 3. 4? Test 100#",
"Hello world one, two!. three. four? Test one hundred#")
#Test English
self.act_and_assert("I have 1 million dollars and 2 cents.", "I have one million dollars and two cents.", "en")
self.act_and_assert("1 2 3 4 5", "one two three four five", "en")
self.act_and_assert("1, 2, 3, 4, 5,", "one, two, three, four, five,", "en")
self.act_and_assert("Hello world 1, 2!. 3. 4? Test 100#",
"Hello world one, two!. three. four? Test one hundred#", "en")
#Test German
self.act_and_assert("1 2 3 4 5", "eins zwei drei vier fünf" ,"de")
self.act_and_assert("1, 2, 3, 4, 5","eins, zwei, drei, vier, fünf","de")
self.act_and_assert("Ich habe 1 Million Dollar und 2 Cent.","Ich habe eins Million Dollar und zwei Cent.","de")
self.act_and_assert("Hallo Welt 1, 2!. 3. 4? Test 100#","Hallo Welt eins, zwei!. drei. vier? Test einhundert#","de")
#Test Spanish
self.act_and_assert("1 2 3 4 5","uno dos tres cuatro cinco","es")
self.act_and_assert("1, 2, 3, 4, 5","uno, dos, tres, cuatro, cinco","es")
self.act_and_assert("Tengo un millón de dólares y 2 centavos","Tengo un millón de dólares y dos centavos","es")
self.act_and_assert("Hola mundo 1, 2!. 3. 4? Prueba 100#","Hola mundo uno, dos!. tres. cuatro? Prueba cien#","es")
def act_and_assert(self, text, expected_output, language="en"):
# Act
result = number_to_words(text, language)
# Assert
self.assertEqual(result, expected_output)
if __name__ == "__main__":
unittest.main()
+2 -1
View File
@@ -29,4 +29,5 @@ packaging
music21
unidecode
dataclasses
dataclasses_json
dataclasses_json
num2words
+3
View File
@@ -86,6 +86,7 @@ dill==0.3.8
# via pylint
docopt==0.6.2
# via
# num2words
# pyannote-metrics
# pyannote-pipeline
dora-search==0.1.12
@@ -264,6 +265,8 @@ networkx==3.3
# via torch
nltk==3.8.1
# via whisperx
num2words==0.5.13
# via -r requirements.in
numba==0.60.0
# via
# librosa
+1
View File
@@ -39,6 +39,7 @@ class Settings:
whisper_align_model = None # Model for other languages from huggingface.co e.g -> "gigant/romanian-wav2vec2"
whisper_batch_size = 16 # reduce if low on GPU mem
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
keep_numbers = False
# Pitch
crepe_model_capacity = "full" # tiny|small|medium|large|full
+10 -3
View File
@@ -129,9 +129,11 @@ def remove_unecessary_punctuations(transcribed_data: list[TranscribedData]) -> N
def run() -> tuple[str, Score, Score]:
"""The processing function of this program"""
#List selected options (can add more later)
if settings.keep_numbers:
print(f"{ULTRASINGER_HEAD} {cyan_highlighted('Option:')} Numbers will be transcribed as numerics (i.e. 1, 2, 3, etc.)")
process_data = InitProcessData()
process_data.process_data_paths.cache_folder_path = (
os.path.join(settings.output_folder_path, "cache")
if settings.cache_override_path is None
@@ -479,6 +481,7 @@ def transcribe_audio(cache_folder_path: str, processing_audio_path: str) -> Tran
settings.whisper_batch_size,
settings.whisper_compute_type,
settings.language,
settings.keep_numbers,
)
with open(transcription_path, "w", encoding=FILE_ENCODING) as file:
file.write(transcription_result.to_json())
@@ -603,6 +606,8 @@ def init_settings(argv: list[str]) -> Settings:
settings.whisper_batch_size = int(arg)
elif opt in ("--whisper_compute_type"):
settings.whisper_compute_type = arg
elif opt in ("--keep_numbers"):
settings.keep_numbers = arg in ["True", "true"]
elif opt in ("--language"):
settings.language = arg
elif opt in ("--crepe"):
@@ -667,6 +672,7 @@ def arg_options():
"ofile=",
"crepe=",
"crepe_step_size=",
"demucs=",
"whisper=",
"whisper_align_model=",
"whisper_batch_size=",
@@ -684,7 +690,8 @@ def arg_options():
"force_crepe_cpu=",
"format_version=",
"keep_cache",
"musescore_path="
"musescore_path=",
"keep_numbers="
]
return long, short
+46 -5
View File
@@ -11,6 +11,14 @@ from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
from modules.Speech_Recognition.TranscribedData import TranscribedData, from_whisper
#Addition for numbers to words
import re
import ast
from num2words import num2words
#Addition for numbers to words
re_split_preserve_space = re.compile(r'(\d+|\W+|\w+)')
MEMORY_ERROR_MESSAGE = f"{ULTRASINGER_HEAD} {blue_highlighted('whisper')} ran out of GPU memory; reduce --whisper_batch_size or force usage of cpu with --force_cpu"
@@ -24,6 +32,26 @@ class WhisperModel(Enum):
LARGE_V2 = "large-v2"
LARGE_V3 = "large-v3"
#Addition for numbers to words (Using previous code from louispan in PR#135)
def number_to_words(line,language='en'):
# https://github.com/m-bain/whisperX
# Transcript words which do not contain characters in the alignment models dictionary e.g. "2014." or "£13.60" cannot be aligned and therefore are not given a timing.
# Therefore, convert numbers to words
out_tokens = []
in_tokens = re_split_preserve_space.findall(line)
for token in in_tokens:
try:
num = ast.literal_eval(token)
try:
out_tokens.append(num2words(num, lang=language))
except NotImplementedError:
print(
f"{ULTRASINGER_HEAD} {red_highlighted('Error:')} Unknown language for number transcription. Keeping number as numeric characters for line: {line}, token: {token}"
)
except Exception:
out_tokens.append(token)
return ''.join(out_tokens)
def transcribe_with_whisper(
audio_path: str,
model: WhisperModel,
@@ -32,6 +60,7 @@ def transcribe_with_whisper(
batch_size: int = 16,
compute_type: str = None,
language: str = None,
keep_numbers: bool = False,
) -> TranscriptionResult:
"""Transcribe with whisper"""
@@ -78,6 +107,11 @@ def transcribe_with_whisper(
)
raise ve
#Addition for numbers to words (Using previous code from louispan in PR#135)
if keep_numbers == False:
for obj in result["segments"]:
obj["text"] = number_to_words(obj["text"],language)
# align whisper output
result_aligned = whisperx.align(
result["segments"],
@@ -120,11 +154,18 @@ def convert_to_transcribed_data(result_aligned):
vtd = from_whisper(obj) # create custom Word object
vtd.word = vtd.word + " " # add space to end of word
if len(obj) < 4:
previous = transcribed_data[-1] if len(transcribed_data) != 0 else TranscribedData()
vtd.start = previous.end + 0.1
vtd.end = previous.end + 0.2
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
f'Fixing it by placing it after the previous word: "{previous.word}". At start: {vtd.start} end: {vtd.end}. Fix it manually!'
#Addition for numbers to words (Using previous code from louispan in PR#135)
if len(transcribed_data) == 0: # if the first word doesn't have any timing data
vtd.start = 0.0
vtd.end = 0.1
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
f'Fixing it by placing it at beginning. At start: {vtd.start} end: {vtd.end}. Fix it manually!'
else:
previous = transcribed_data[-1] if len(transcribed_data) != 0 else TranscribedData()
vtd.start = previous.end + 0.1
vtd.end = previous.end + 0.2
msg = f'Error: There is no timestamp for word: "{obj["word"]}". ' \
f'Fixing it by placing it after the previous word: "{previous.word}". At start: {vtd.start} end: {vtd.end}. Fix it manually!'
print(f"{red_highlighted(msg)}")
transcribed_data.append(vtd) # and add it to list
return transcribed_data
+1
View File
@@ -39,6 +39,7 @@ def print_help() -> None:
--language Override the language detected by whisper, does not affect transcription but steps after transcription
--whisper_batch_size Reduce if low on GPU mem >> ((default) is 16)
--whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
--keep_numbers Numbers will be transcribed as numerics instead of as words >> True|False >> ((default) is False)
[pitcher]
# Default is crepe