mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Add word end detection for linebreaks
This commit is contained in:
1 parent
849c281544
commit
7301304085
5 files changed
+42
-79
No files matched your search
@@ -1,7 +1,7 @@
|
||||
"""Tests for silence_processing.py"""
|
||||
|
||||
import unittest
|
||||
from src.modules.Audio.silence_processing import remove_silence2
|
||||
from src.modules.Audio.silence_processing import remove_silence
|
||||
from modules.Speech_Recognition.TranscribedData import TranscribedData
|
||||
|
||||
|
||||
@@ -60,41 +60,41 @@ class SilenceProcessingTest(unittest.TestCase):
|
||||
]
|
||||
|
||||
# Act
|
||||
result = remove_silence2(silence_parts_list, transcribed_data)
|
||||
result = remove_silence(silence_parts_list, transcribed_data)
|
||||
|
||||
# Assert
|
||||
self.assertEqual(result[0].word, "Before silence ")
|
||||
self.assertEqual(result[0].start, 1.0)
|
||||
self.assertEqual(result[0].end, 2.0)
|
||||
self.assertEqual(result[0].is_hyphen, None)
|
||||
self.assertEqual(result[0].is_word_end, True)
|
||||
|
||||
self.assertEqual(result[1].word, "End is in silence ")
|
||||
self.assertEqual(result[1].start, 1.0)
|
||||
self.assertEqual(result[1].end, 2.0)
|
||||
self.assertEqual(result[1].is_hyphen, None)
|
||||
self.assertEqual(result[1].is_word_end, True)
|
||||
|
||||
self.assertEqual(len(result), 6) # "in silence" is removed
|
||||
|
||||
self.assertEqual(result[2].word, "Start is in silence ")
|
||||
self.assertEqual(result[2].start, 9.0)
|
||||
self.assertEqual(result[2].end, 10.0)
|
||||
self.assertEqual(result[2].is_hyphen, None)
|
||||
self.assertEqual(result[2].is_word_end, True)
|
||||
|
||||
self.assertEqual(result[3].word, "After silence ")
|
||||
self.assertEqual(result[3].start, 9.0)
|
||||
self.assertEqual(result[3].end, 10.0)
|
||||
self.assertEqual(result[3].is_hyphen, None)
|
||||
self.assertEqual(result[3].is_word_end, True)
|
||||
|
||||
# Split to "Silence is inside~ "
|
||||
self.assertEqual(result[4].word, "Silence is inside")
|
||||
self.assertEqual(result[4].start, 1.0)
|
||||
self.assertEqual(result[4].end, 2.0)
|
||||
self.assertEqual(result[4].is_hyphen, True)
|
||||
self.assertEqual(result[4].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[5].word, "~ ")
|
||||
self.assertEqual(result[5].start, 9.0)
|
||||
self.assertEqual(result[5].end, 12.0)
|
||||
self.assertEqual(result[5].is_hyphen, False)
|
||||
self.assertEqual(result[5].is_word_end, True)
|
||||
|
||||
def test_remove_multiple_silence_in_between_silence2(self):
|
||||
#
|
||||
@@ -115,33 +115,33 @@ class SilenceProcessingTest(unittest.TestCase):
|
||||
]
|
||||
|
||||
# Act
|
||||
result = remove_silence2(silence_parts_list, transcribed_data)
|
||||
result = remove_silence(silence_parts_list, transcribed_data)
|
||||
|
||||
# Assert
|
||||
self.assertEqual(result[0].word, "Silence is inside")
|
||||
self.assertEqual(result[0].start, 1.0)
|
||||
self.assertEqual(result[0].end, 2.0)
|
||||
self.assertEqual(result[0].is_hyphen, True)
|
||||
self.assertEqual(result[0].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[1].word, "~")
|
||||
self.assertEqual(result[1].start, 3.0)
|
||||
self.assertEqual(result[1].end, 4.0)
|
||||
self.assertEqual(result[1].is_hyphen, True)
|
||||
self.assertEqual(result[1].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[2].word, "~")
|
||||
self.assertEqual(result[2].start, 5.0)
|
||||
self.assertEqual(result[2].end, 6.0)
|
||||
self.assertEqual(result[2].is_hyphen, True)
|
||||
self.assertEqual(result[2].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[3].word, "~")
|
||||
self.assertEqual(result[3].start, 7.0)
|
||||
self.assertEqual(result[3].end, 8.0)
|
||||
self.assertEqual(result[3].is_hyphen, True)
|
||||
self.assertEqual(result[3].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[4].word, "~ ")
|
||||
self.assertEqual(result[4].start, 9.0)
|
||||
self.assertEqual(result[4].end, 12.0)
|
||||
self.assertEqual(result[4].is_hyphen, False)
|
||||
self.assertEqual(result[4].is_word_end, True)
|
||||
|
||||
|
||||
def test_remove_multiple_silence_till_end_silence2(self):
|
||||
@@ -163,28 +163,28 @@ class SilenceProcessingTest(unittest.TestCase):
|
||||
]
|
||||
|
||||
# Act
|
||||
result = remove_silence2(silence_parts_list, transcribed_data)
|
||||
result = remove_silence(silence_parts_list, transcribed_data)
|
||||
|
||||
# Assert
|
||||
self.assertEqual(result[0].word, "Silence is inside")
|
||||
self.assertEqual(result[0].start, 1.0)
|
||||
self.assertEqual(result[0].end, 2.0)
|
||||
self.assertEqual(result[0].is_hyphen, True)
|
||||
self.assertEqual(result[0].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[1].word, "~")
|
||||
self.assertEqual(result[1].start, 3.0)
|
||||
self.assertEqual(result[1].end, 4.0)
|
||||
self.assertEqual(result[1].is_hyphen, True)
|
||||
self.assertEqual(result[1].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[2].word, "~")
|
||||
self.assertEqual(result[2].start, 5.0)
|
||||
self.assertEqual(result[2].end, 6.0)
|
||||
self.assertEqual(result[2].is_hyphen, True)
|
||||
self.assertEqual(result[2].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[3].word, "~ ")
|
||||
self.assertEqual(result[3].start, 7.0)
|
||||
self.assertEqual(result[3].end, 8.0)
|
||||
self.assertEqual(result[3].is_hyphen, False)
|
||||
self.assertEqual(result[3].is_word_end, True)
|
||||
|
||||
def test_remove_no_duration_silence2(self):
|
||||
#
|
||||
@@ -217,30 +217,30 @@ class SilenceProcessingTest(unittest.TestCase):
|
||||
]
|
||||
|
||||
# Act
|
||||
result = remove_silence2(silence_parts_list, transcribed_data)
|
||||
result = remove_silence(silence_parts_list, transcribed_data)
|
||||
|
||||
# Assert
|
||||
self.assertEqual(result[0].word, "Remove split ")
|
||||
self.assertEqual(result[0].start, 1.0)
|
||||
self.assertEqual(result[0].end, 2.0)
|
||||
self.assertEqual(result[0].is_hyphen, None)
|
||||
self.assertEqual(result[0].is_word_end, True)
|
||||
|
||||
self.assertEqual(result[1].word, "Is split ")
|
||||
self.assertEqual(result[1].start, 3.1)
|
||||
self.assertEqual(result[1].end, 4.0)
|
||||
self.assertEqual(result[1].is_hyphen, None)
|
||||
self.assertEqual(result[1].is_word_end, True)
|
||||
|
||||
self.assertEqual(len(result), 4) # "removed " is removed
|
||||
|
||||
self.assertEqual(result[2].word, "Is split2")
|
||||
self.assertEqual(result[2].start, 5.09)
|
||||
self.assertEqual(result[2].end, 6.0)
|
||||
self.assertEqual(result[2].is_hyphen, True)
|
||||
self.assertEqual(result[2].is_word_end, False)
|
||||
|
||||
self.assertEqual(result[3].word, "~ ")
|
||||
self.assertEqual(result[3].start, 6.09)
|
||||
self.assertEqual(result[3].end, 7.0)
|
||||
self.assertEqual(result[3].is_hyphen, False)
|
||||
self.assertEqual(result[3].is_word_end, True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -132,6 +132,10 @@ def add_hyphen_to_data(transcribed_data: list[TranscribedData], hyphen_words: li
|
||||
dup.end = next_start
|
||||
dup.word = hyphen_words[i][hyphenated_word_index]
|
||||
dup.is_hyphen = True
|
||||
if hyphenated_word_index == len(hyphen_words[i]) - 1:
|
||||
dup.is_word_end = True
|
||||
else:
|
||||
dup.is_word_end = False
|
||||
new_data.append(dup)
|
||||
|
||||
return new_data
|
||||
|
||||
@@ -11,16 +11,12 @@ def remove_silence_from_transcription_data(audio_path: str, transcribed_data: li
|
||||
"""Remove silence from given transcription data"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Removing silent start and ending, from transcription data"
|
||||
f"{ULTRASINGER_HEAD} Removing silent parts from transcription data"
|
||||
)
|
||||
|
||||
silence_timestamps = get_silence_sections(audio_path)
|
||||
a = remove_silence2(silence_timestamps, transcribed_data)
|
||||
return a
|
||||
|
||||
#audio, sample_rate = librosa.load(audio_path, sr=None)
|
||||
#remove_silence(audio, sample_rate, transcribed_data)
|
||||
#return transcribed_data
|
||||
data = remove_silence(silence_timestamps, transcribed_data)
|
||||
return data
|
||||
|
||||
|
||||
def get_silence_sections(audio_path: str,
|
||||
@@ -32,7 +28,7 @@ def get_silence_sections(audio_path: str,
|
||||
return s
|
||||
|
||||
|
||||
def remove_silence2(silence_parts_list: list[tuple[float, float]], transcribed_data: list[TranscribedData]):
|
||||
def remove_silence(silence_parts_list: list[tuple[float, float]], transcribed_data: list[TranscribedData]):
|
||||
new_transcribed_data = []
|
||||
|
||||
for data in transcribed_data:
|
||||
@@ -59,16 +55,16 @@ def remove_silence2(silence_parts_list: list[tuple[float, float]], transcribed_d
|
||||
|
||||
if silence_parts_list[next_index][1] >= origin_end:
|
||||
split_word = "~ "
|
||||
is_hyphen = False
|
||||
is_word_end = True
|
||||
else:
|
||||
split_word = "~"
|
||||
is_hyphen = True
|
||||
is_word_end = False
|
||||
else:
|
||||
split_end = origin_end
|
||||
split_word = "~ "
|
||||
is_hyphen = False
|
||||
is_word_end = True
|
||||
|
||||
split_data = TranscribedData({"conf": data.conf, "word": split_word, "end": split_end, "start": silence_end, "is_hyphen": is_hyphen})
|
||||
split_data = TranscribedData({"conf": data.conf, "word": split_word, "end": split_end, "start": silence_end, "is_word_end": is_word_end})
|
||||
|
||||
if not was_split:
|
||||
data.end = silence_start
|
||||
@@ -81,7 +77,7 @@ def remove_silence2(silence_parts_list: list[tuple[float, float]], transcribed_d
|
||||
if split_data.end - split_data.start <= 0.1:
|
||||
continue
|
||||
|
||||
data.is_hyphen = True
|
||||
data.is_word_end = False
|
||||
|
||||
# Remove last whitespace from the data.word
|
||||
if data.word[-1] == " ":
|
||||
@@ -90,10 +86,10 @@ def remove_silence2(silence_parts_list: list[tuple[float, float]], transcribed_d
|
||||
if split_data.end - split_data.start > 0.1:
|
||||
was_split = True
|
||||
new_transcribed_data.append(split_data)
|
||||
elif split_word == "~ " and data.is_hyphen:
|
||||
elif split_word == "~ " and not data.is_word_end:
|
||||
if new_transcribed_data[-1].word[-1] != " ":
|
||||
new_transcribed_data[-1].word += " "
|
||||
new_transcribed_data[-1].is_hyphen = False
|
||||
new_transcribed_data[-1].is_word_end = True
|
||||
|
||||
continue
|
||||
|
||||
@@ -122,42 +118,4 @@ def remove_silence2(silence_parts_list: list[tuple[float, float]], transcribed_d
|
||||
if silence_start > origin_end:
|
||||
# Nothing to do with this word anymore, go to next word
|
||||
break
|
||||
return new_transcribed_data
|
||||
|
||||
|
||||
def remove_silence(audio, sample_rate, transcribed_data: list[TranscribedData]):
|
||||
for i, data in enumerate(transcribed_data):
|
||||
start_time = data.start
|
||||
end_time = data.end
|
||||
start_sample = int(start_time * sample_rate)
|
||||
end_sample = int(end_time * sample_rate)
|
||||
chunk = audio[start_sample:end_sample]
|
||||
|
||||
# todo: why 5 works good? It should be 40db ?!?
|
||||
# max_dB = librosa.amplitude_to_db(chunk, ref=np.max)
|
||||
silence_threshold = 5
|
||||
onsets = librosa.effects.split(
|
||||
chunk, top_db=silence_threshold, frame_length=2048, hop_length=100
|
||||
)
|
||||
|
||||
# Get the duration of the first and last silent intervals
|
||||
if len(onsets) > 0:
|
||||
first_silence = onsets[0][0]
|
||||
last_silence = len(chunk) - onsets[-1][1]
|
||||
|
||||
first_silence_duration = librosa.samples_to_time(
|
||||
first_silence, sr=sample_rate
|
||||
)
|
||||
last_silence_duration = librosa.samples_to_time(
|
||||
last_silence, sr=sample_rate
|
||||
)
|
||||
else:
|
||||
first_silence_duration = 0
|
||||
last_silence_duration = 0
|
||||
|
||||
data.start = (
|
||||
data.start + first_silence_duration
|
||||
)
|
||||
data.end = (
|
||||
data.end - last_silence_duration
|
||||
)
|
||||
return new_transcribed_data
|
||||
@@ -16,3 +16,4 @@ class TranscribedData:
|
||||
self.end = transcribed_json.get("end", None)
|
||||
self.start = transcribed_json.get("start", None)
|
||||
self.is_hyphen = transcribed_json.get("is_hyphen", None)
|
||||
self.is_word_end = transcribed_json.get("is_word_end", True)
|
||||
@@ -121,7 +121,7 @@ def create_ultrastar_txt_from_automation(
|
||||
file.write(line)
|
||||
|
||||
# detect silence between words
|
||||
if transcribed_data[i].is_hyphen:
|
||||
if not transcribed_data[i].is_word_end:
|
||||
continue
|
||||
|
||||
if i < len(transcribed_data) - 1:
|
||||
|
||||
Reference in new issue
Block a user