Phoneme level transcription. The math is mathing.
This commit is contained in:
1 parent
a62bdb5896
commit
7415f99152
2 files changed
+144
-82
No files matched your search
@@ -9,16 +9,45 @@ def gpu_reset():
|
||||
"""
|
||||
Re-enable GPU acceleration for all services without restarting the container.
|
||||
|
||||
After a CUDA OOM or 'unknown error', the lyric transcriber marks the GPU
|
||||
context as failed and falls back to CPU. The separator may also fail with
|
||||
a CUDA context error. Calling this endpoint clears the CUDA allocation
|
||||
cache and resets the Whisper device flag, so both services can retry on GPU.
|
||||
Resets:
|
||||
- Lyric transcriber: clears _cuda_ok flag and Whisper model cache
|
||||
- All services: gc.collect() + torch.cuda.empty_cache() to release
|
||||
any stranded allocations left by separators or alignment models
|
||||
|
||||
NOTE: if the CUDA context is truly corrupted (persistent 'unknown error'),
|
||||
a container restart is still required.
|
||||
"""
|
||||
from ..services.lyric_transcriber_service import reset_cuda
|
||||
return reset_cuda()
|
||||
import gc
|
||||
|
||||
# 1. Reset transcriber-specific CUDA state (_cuda_ok flag + Whisper cache).
|
||||
transcriber_result: dict = {"ok": False, "error": None}
|
||||
try:
|
||||
from ..services.lyric_transcriber_service import reset_cuda
|
||||
transcriber_result = reset_cuda()
|
||||
except Exception as exc:
|
||||
transcriber_result = {"ok": False, "error": str(exc)}
|
||||
|
||||
# 2. Global GPU flush — releases allocations from separators, alignment
|
||||
# models, and any other PyTorch tensors still resident in VRAM.
|
||||
cuda_available = False
|
||||
flush_error: str | None = None
|
||||
try:
|
||||
import torch
|
||||
cuda_available = torch.cuda.is_available()
|
||||
gc.collect()
|
||||
if cuda_available:
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
except Exception as exc:
|
||||
flush_error = str(exc)
|
||||
|
||||
return {
|
||||
"ok": transcriber_result.get("ok", False),
|
||||
"cuda_available": cuda_available,
|
||||
"cuda_ok": transcriber_result.get("cuda_ok", False),
|
||||
"transcriber_reset": transcriber_result,
|
||||
"flush_error": flush_error,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/gpu-info")
|
||||
|
||||
@@ -410,10 +410,11 @@ def result_to_lrc(result: LyricTranscriptionResult) -> str:
|
||||
|
||||
|
||||
def result_to_elrc(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to ELRC format (word-level timestamps).
|
||||
"""Convert transcription result to ELRC format.
|
||||
|
||||
Speaker info is encoded the same way as in result_to_lrc — |sp:SPEAKER_00|
|
||||
is prepended to the word-timestamp content of each line.
|
||||
Uses syllable-level timestamps when phoneme alignment data is present
|
||||
(word.syllables populated), otherwise falls back to word-level timestamps.
|
||||
Speaker info is encoded as |sp:SPEAKER_00| prepended to each line.
|
||||
"""
|
||||
lines = []
|
||||
for seg in result.segments:
|
||||
@@ -432,18 +433,28 @@ def result_to_elrc(result: LyricTranscriptionResult) -> str:
|
||||
line_sec = int(line_start % 60)
|
||||
line_centi = int((line_start % 1) * 100)
|
||||
|
||||
# Build word-level timestamps.
|
||||
# Strip each word's surrounding whitespace before embedding — Whisper/WhisperX
|
||||
# sometimes include a leading space (e.g. " world") which would produce
|
||||
# double-spaces when tokens are joined.
|
||||
word_parts = []
|
||||
word_parts: list = []
|
||||
for word in seg.words:
|
||||
w_min = int(word.start // 60)
|
||||
w_sec = int(word.start % 60)
|
||||
w_centi = int((word.start % 1) * 100)
|
||||
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}")
|
||||
if word.syllables:
|
||||
# Syllable-level: split the orthographic word into n chunks and
|
||||
# pair each with its phoneme timestamp.
|
||||
ortho = _ortho_split(word.word, len(word.syllables))
|
||||
for syl, display_text in zip(word.syllables, ortho):
|
||||
s = syl["start"]
|
||||
s_min = int(s // 60)
|
||||
s_sec = int(s % 60)
|
||||
s_centi = int((s % 1) * 100)
|
||||
word_parts.append(f"<{s_min:02d}:{s_sec:02d}.{s_centi:02d}>{display_text}")
|
||||
else:
|
||||
# Word-level fallback.
|
||||
w_min = int(word.start // 60)
|
||||
w_sec = int(word.start % 60)
|
||||
w_centi = int((word.start % 1) * 100)
|
||||
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}")
|
||||
word_parts.append(" ")
|
||||
|
||||
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{' '.join(word_parts)}")
|
||||
line_content = "".join(word_parts).rstrip()
|
||||
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{line_content}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -549,56 +560,6 @@ def _ortho_split(word_text: str, n: int) -> List[str]:
|
||||
return [p for p in parts if p] or [text]
|
||||
|
||||
|
||||
def result_to_sylrc(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to syllable-level ELRC format.
|
||||
|
||||
Each syllable within a word gets its own <MM:SS.cc> timestamp tag.
|
||||
The display text is an orthographic split of the original word (not IPA),
|
||||
so the karaoke preview shows readable English text at syllable granularity.
|
||||
Falls back to word-level timestamps when no syllable data is present.
|
||||
|
||||
The speaker tag (|sp:...|) is encoded the same way as in result_to_elrc.
|
||||
"""
|
||||
lines = []
|
||||
for seg in result.segments:
|
||||
speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else ""
|
||||
if not seg.words:
|
||||
minutes = int(seg.start // 60)
|
||||
seconds = int(seg.start % 60)
|
||||
centis = int((seg.start % 1) * 100)
|
||||
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}")
|
||||
continue
|
||||
|
||||
line_start = seg.words[0].start
|
||||
line_min = int(line_start // 60)
|
||||
line_sec = int(line_start % 60)
|
||||
line_centi = int((line_start % 1) * 100)
|
||||
|
||||
word_parts: list = []
|
||||
for word in seg.words:
|
||||
if word.syllables:
|
||||
# Split the original English word text into n orthographic
|
||||
# chunks and pair each chunk with a phoneme syllable timestamp.
|
||||
ortho = _ortho_split(word.word, len(word.syllables))
|
||||
for syl, display_text in zip(word.syllables, ortho):
|
||||
s = syl["start"]
|
||||
s_min = int(s // 60)
|
||||
s_sec = int(s % 60)
|
||||
s_centi = int((s % 1) * 100)
|
||||
word_parts.append(f"<{s_min:02d}:{s_sec:02d}.{s_centi:02d}>{display_text}")
|
||||
else:
|
||||
# Fall back to word-level timestamp.
|
||||
w_min = int(word.start // 60)
|
||||
w_sec = int(word.start % 60)
|
||||
w_centi = int((word.start % 1) * 100)
|
||||
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word}")
|
||||
word_parts.append(" ")
|
||||
|
||||
line_content = "".join(word_parts).rstrip()
|
||||
lines.append(
|
||||
f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{line_content}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
_PUNCT_BREAK = frozenset('.!?,;')
|
||||
@@ -646,7 +607,7 @@ def _split_long_segments(segments: list, max_words: int = 10) -> list:
|
||||
|
||||
for ch in chunks:
|
||||
first, last = ch[0], ch[-1]
|
||||
sub_text = "".join(w.get("word", "") for w in ch).strip()
|
||||
sub_text = " ".join(w.get("word", "").strip() for w in ch).strip()
|
||||
result.append({
|
||||
"start": first.get("start") if first.get("start") is not None else seg["start"],
|
||||
"end": last.get("end") if last.get("end") is not None else seg["end"],
|
||||
@@ -834,20 +795,92 @@ def _group_chars_to_syllables(chars: list) -> list:
|
||||
|
||||
def _reconstruct_with_phonemes(phoneme_aligned: dict, original_words_per_seg: list) -> dict:
|
||||
"""After phoneme-level alignment, restore original word texts and attach
|
||||
character-level phoneme timestamps and syllable groupings to each word."""
|
||||
character-level phoneme timestamps and syllable groupings to each word.
|
||||
|
||||
WhisperX stores char alignments on the *segment* dict (not on each word),
|
||||
so we partition the segment-level chars list into per-word slices by time.
|
||||
"""
|
||||
for seg_idx, seg in enumerate(phoneme_aligned.get("segments", [])):
|
||||
orig = (original_words_per_seg[seg_idx]
|
||||
if seg_idx < len(original_words_per_seg) else [])
|
||||
|
||||
# chars live on the segment in this version of whisperx
|
||||
seg_chars = [c for c in seg.get("chars", []) if c.get("start") is not None]
|
||||
char_ptr = 0
|
||||
|
||||
for word_idx, word in enumerate(seg.get("words", [])):
|
||||
if word_idx < len(orig):
|
||||
word["word"] = orig[word_idx]
|
||||
chars = word.get("chars", [])
|
||||
if chars:
|
||||
word["phonemes"] = chars
|
||||
word["syllables"] = _group_chars_to_syllables(chars)
|
||||
|
||||
# Try per-word chars first (older whisperx put them there directly)
|
||||
word_chars = word.get("chars", [])
|
||||
if not word_chars and seg_chars:
|
||||
# Partition segment chars into this word by time window
|
||||
w_start = word.get("start")
|
||||
w_end = word.get("end")
|
||||
if w_start is not None and w_end is not None:
|
||||
word_chars = []
|
||||
while char_ptr < len(seg_chars):
|
||||
c = seg_chars[char_ptr]
|
||||
c_start = c.get("start", w_end + 1)
|
||||
if c_start > w_end + 0.005:
|
||||
break
|
||||
if c_start >= w_start - 0.005:
|
||||
word_chars.append(c)
|
||||
char_ptr += 1
|
||||
|
||||
if word_chars:
|
||||
word["phonemes"] = word_chars
|
||||
word["syllables"] = _group_chars_to_syllables(word_chars)
|
||||
|
||||
return phoneme_aligned
|
||||
|
||||
|
||||
def _apply_pyphen_syllables(segments: list) -> None:
|
||||
"""Populate word["syllables"] using pyphen proportional time splitting.
|
||||
|
||||
Uses the same pyphen library as the lyrics-sync lsync module to identify
|
||||
English syllable boundaries, then distributes each word's [start, end]
|
||||
range proportionally across its syllables.
|
||||
|
||||
Skips words that already have syllable data (e.g. from the IPA phoneme
|
||||
path). Modifies segments in-place. Falls back silently if pyphen is
|
||||
not installed.
|
||||
"""
|
||||
try:
|
||||
import pyphen
|
||||
dic = pyphen.Pyphen(lang='en_US')
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
for seg in segments:
|
||||
for word_info in seg.get("words", []):
|
||||
if word_info.get("syllables"):
|
||||
continue # already populated by phoneme path
|
||||
text = word_info.get("word", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
start = float(word_info.get("start") or 0.0)
|
||||
end = float(word_info.get("end") or start)
|
||||
duration = max(end - start, 0.0)
|
||||
|
||||
positions = dic.positions(text.lower())
|
||||
if not positions:
|
||||
word_info["syllables"] = [{"text": text, "start": start, "end": end}]
|
||||
continue
|
||||
|
||||
boundaries = [0] + list(positions) + [len(text)]
|
||||
total_chars = len(text)
|
||||
syllables = []
|
||||
for i in range(len(boundaries) - 1):
|
||||
s_char = boundaries[i]
|
||||
e_char = boundaries[i + 1]
|
||||
syl_start = start + (s_char / total_chars) * duration
|
||||
syl_end = start + (e_char / total_chars) * duration
|
||||
syllables.append({"text": text[s_char:e_char], "start": syl_start, "end": syl_end})
|
||||
word_info["syllables"] = syllables
|
||||
|
||||
|
||||
# Batch size and compute precision per power tier.
|
||||
# 'compute_type_cuda' is used on GPU; CPU always falls back to 'int8'.
|
||||
_WHISPERX_POWER_PRESETS: dict = {
|
||||
@@ -993,6 +1026,13 @@ def run_lyric_transcription(
|
||||
return_char_alignments=False,
|
||||
)
|
||||
|
||||
# If syllable timing was requested, fill any words that don't yet have
|
||||
# syllable data using pyphen proportional splitting (same approach used
|
||||
# by the lsync library). This covers both the IPA-phoneme fallback case
|
||||
# and the standard word-level path when phoneme_align=True.
|
||||
if phoneme_align:
|
||||
_apply_pyphen_syllables(wx_result["segments"])
|
||||
|
||||
# Free alignment model before diarization (pyannote is also VRAM-heavy).
|
||||
del model_a, metadata
|
||||
gc.collect()
|
||||
@@ -1194,13 +1234,6 @@ def run_lyric_transcription(
|
||||
vtt_path.write_text(vtt_content, encoding="utf-8")
|
||||
output_files.append({"name": vtt_path.name, "path": str(vtt_path), "type": "vtt"})
|
||||
|
||||
# Syllable-level ELRC (phoneme alignment only)
|
||||
if "sylrc" in output_formats:
|
||||
sylrc_path = out_path / f"{base_name}.sylrc"
|
||||
sylrc_content = result_to_sylrc(result)
|
||||
sylrc_path.write_text(sylrc_content, encoding="utf-8")
|
||||
output_files.append({"name": sylrc_path.name, "path": str(sylrc_path), "type": "sylrc"})
|
||||
|
||||
# Update job with results
|
||||
result_summary = {
|
||||
"model": result.model,
|
||||
|
||||
Reference in new issue
Block a user