Phoneme level transcription. The math is mathing.

This commit is contained in:
seclusionenthusiast committed 2026-03-04 21:37:03 +09:00
1 parent a62bdb5896
commit 7415f99152
2 files changed
+144 -82

No files matched your search

+35 -6
View File
@@ -9,16 +9,45 @@ def gpu_reset():
"""
Re-enable GPU acceleration for all services without restarting the container.
After a CUDA OOM or 'unknown error', the lyric transcriber marks the GPU
context as failed and falls back to CPU. The separator may also fail with
a CUDA context error. Calling this endpoint clears the CUDA allocation
cache and resets the Whisper device flag, so both services can retry on GPU.
Resets:
- Lyric transcriber: clears _cuda_ok flag and Whisper model cache
- All services: gc.collect() + torch.cuda.empty_cache() to release
any stranded allocations left by separators or alignment models
NOTE: if the CUDA context is truly corrupted (persistent 'unknown error'),
a container restart is still required.
"""
from ..services.lyric_transcriber_service import reset_cuda
return reset_cuda()
import gc
# 1. Reset transcriber-specific CUDA state (_cuda_ok flag + Whisper cache).
transcriber_result: dict = {"ok": False, "error": None}
try:
from ..services.lyric_transcriber_service import reset_cuda
transcriber_result = reset_cuda()
except Exception as exc:
transcriber_result = {"ok": False, "error": str(exc)}
# 2. Global GPU flush — releases allocations from separators, alignment
# models, and any other PyTorch tensors still resident in VRAM.
cuda_available = False
flush_error: str | None = None
try:
import torch
cuda_available = torch.cuda.is_available()
gc.collect()
if cuda_available:
torch.cuda.empty_cache()
torch.cuda.reset_peak_memory_stats()
except Exception as exc:
flush_error = str(exc)
return {
"ok": transcriber_result.get("ok", False),
"cuda_available": cuda_available,
"cuda_ok": transcriber_result.get("cuda_ok", False),
"transcriber_reset": transcriber_result,
"flush_error": flush_error,
}
@router.get("/gpu-info")
+109 -76
View File
@@ -410,10 +410,11 @@ def result_to_lrc(result: LyricTranscriptionResult) -> str:
def result_to_elrc(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to ELRC format (word-level timestamps).
"""Convert transcription result to ELRC format.
Speaker info is encoded the same way as in result_to_lrc — |sp:SPEAKER_00|
is prepended to the word-timestamp content of each line.
Uses syllable-level timestamps when phoneme alignment data is present
(word.syllables populated), otherwise falls back to word-level timestamps.
Speaker info is encoded as |sp:SPEAKER_00| prepended to each line.
"""
lines = []
for seg in result.segments:
@@ -432,18 +433,28 @@ def result_to_elrc(result: LyricTranscriptionResult) -> str:
line_sec = int(line_start % 60)
line_centi = int((line_start % 1) * 100)
# Build word-level timestamps.
# Strip each word's surrounding whitespace before embedding — Whisper/WhisperX
# sometimes include a leading space (e.g. " world") which would produce
# double-spaces when tokens are joined.
word_parts = []
word_parts: list = []
for word in seg.words:
w_min = int(word.start // 60)
w_sec = int(word.start % 60)
w_centi = int((word.start % 1) * 100)
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}")
if word.syllables:
# Syllable-level: split the orthographic word into n chunks and
# pair each with its phoneme timestamp.
ortho = _ortho_split(word.word, len(word.syllables))
for syl, display_text in zip(word.syllables, ortho):
s = syl["start"]
s_min = int(s // 60)
s_sec = int(s % 60)
s_centi = int((s % 1) * 100)
word_parts.append(f"<{s_min:02d}:{s_sec:02d}.{s_centi:02d}>{display_text}")
else:
# Word-level fallback.
w_min = int(word.start // 60)
w_sec = int(word.start % 60)
w_centi = int((word.start % 1) * 100)
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}")
word_parts.append(" ")
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{' '.join(word_parts)}")
line_content = "".join(word_parts).rstrip()
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{line_content}")
return "\n".join(lines)
@@ -549,56 +560,6 @@ def _ortho_split(word_text: str, n: int) -> List[str]:
return [p for p in parts if p] or [text]
def result_to_sylrc(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to syllable-level ELRC format.
Each syllable within a word gets its own <MM:SS.cc> timestamp tag.
The display text is an orthographic split of the original word (not IPA),
so the karaoke preview shows readable English text at syllable granularity.
Falls back to word-level timestamps when no syllable data is present.
The speaker tag (|sp:...|) is encoded the same way as in result_to_elrc.
"""
lines = []
for seg in result.segments:
speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else ""
if not seg.words:
minutes = int(seg.start // 60)
seconds = int(seg.start % 60)
centis = int((seg.start % 1) * 100)
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}")
continue
line_start = seg.words[0].start
line_min = int(line_start // 60)
line_sec = int(line_start % 60)
line_centi = int((line_start % 1) * 100)
word_parts: list = []
for word in seg.words:
if word.syllables:
# Split the original English word text into n orthographic
# chunks and pair each chunk with a phoneme syllable timestamp.
ortho = _ortho_split(word.word, len(word.syllables))
for syl, display_text in zip(word.syllables, ortho):
s = syl["start"]
s_min = int(s // 60)
s_sec = int(s % 60)
s_centi = int((s % 1) * 100)
word_parts.append(f"<{s_min:02d}:{s_sec:02d}.{s_centi:02d}>{display_text}")
else:
# Fall back to word-level timestamp.
w_min = int(word.start // 60)
w_sec = int(word.start % 60)
w_centi = int((word.start % 1) * 100)
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word}")
word_parts.append(" ")
line_content = "".join(word_parts).rstrip()
lines.append(
f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{line_content}"
)
return "\n".join(lines)
_PUNCT_BREAK = frozenset('.!?,;')
@@ -646,7 +607,7 @@ def _split_long_segments(segments: list, max_words: int = 10) -> list:
for ch in chunks:
first, last = ch[0], ch[-1]
sub_text = "".join(w.get("word", "") for w in ch).strip()
sub_text = " ".join(w.get("word", "").strip() for w in ch).strip()
result.append({
"start": first.get("start") if first.get("start") is not None else seg["start"],
"end": last.get("end") if last.get("end") is not None else seg["end"],
@@ -834,20 +795,92 @@ def _group_chars_to_syllables(chars: list) -> list:
def _reconstruct_with_phonemes(phoneme_aligned: dict, original_words_per_seg: list) -> dict:
"""After phoneme-level alignment, restore original word texts and attach
character-level phoneme timestamps and syllable groupings to each word."""
character-level phoneme timestamps and syllable groupings to each word.
WhisperX stores char alignments on the *segment* dict (not on each word),
so we partition the segment-level chars list into per-word slices by time.
"""
for seg_idx, seg in enumerate(phoneme_aligned.get("segments", [])):
orig = (original_words_per_seg[seg_idx]
if seg_idx < len(original_words_per_seg) else [])
# chars live on the segment in this version of whisperx
seg_chars = [c for c in seg.get("chars", []) if c.get("start") is not None]
char_ptr = 0
for word_idx, word in enumerate(seg.get("words", [])):
if word_idx < len(orig):
word["word"] = orig[word_idx]
chars = word.get("chars", [])
if chars:
word["phonemes"] = chars
word["syllables"] = _group_chars_to_syllables(chars)
# Try per-word chars first (older whisperx put them there directly)
word_chars = word.get("chars", [])
if not word_chars and seg_chars:
# Partition segment chars into this word by time window
w_start = word.get("start")
w_end = word.get("end")
if w_start is not None and w_end is not None:
word_chars = []
while char_ptr < len(seg_chars):
c = seg_chars[char_ptr]
c_start = c.get("start", w_end + 1)
if c_start > w_end + 0.005:
break
if c_start >= w_start - 0.005:
word_chars.append(c)
char_ptr += 1
if word_chars:
word["phonemes"] = word_chars
word["syllables"] = _group_chars_to_syllables(word_chars)
return phoneme_aligned
def _apply_pyphen_syllables(segments: list) -> None:
"""Populate word["syllables"] using pyphen proportional time splitting.
Uses the same pyphen library as the lyrics-sync lsync module to identify
English syllable boundaries, then distributes each word's [start, end]
range proportionally across its syllables.
Skips words that already have syllable data (e.g. from the IPA phoneme
path). Modifies segments in-place. Falls back silently if pyphen is
not installed.
"""
try:
import pyphen
dic = pyphen.Pyphen(lang='en_US')
except ImportError:
return
for seg in segments:
for word_info in seg.get("words", []):
if word_info.get("syllables"):
continue # already populated by phoneme path
text = word_info.get("word", "").strip()
if not text:
continue
start = float(word_info.get("start") or 0.0)
end = float(word_info.get("end") or start)
duration = max(end - start, 0.0)
positions = dic.positions(text.lower())
if not positions:
word_info["syllables"] = [{"text": text, "start": start, "end": end}]
continue
boundaries = [0] + list(positions) + [len(text)]
total_chars = len(text)
syllables = []
for i in range(len(boundaries) - 1):
s_char = boundaries[i]
e_char = boundaries[i + 1]
syl_start = start + (s_char / total_chars) * duration
syl_end = start + (e_char / total_chars) * duration
syllables.append({"text": text[s_char:e_char], "start": syl_start, "end": syl_end})
word_info["syllables"] = syllables
# Batch size and compute precision per power tier.
# 'compute_type_cuda' is used on GPU; CPU always falls back to 'int8'.
_WHISPERX_POWER_PRESETS: dict = {
@@ -993,6 +1026,13 @@ def run_lyric_transcription(
return_char_alignments=False,
)
# If syllable timing was requested, fill any words that don't yet have
# syllable data using pyphen proportional splitting (same approach used
# by the lsync library). This covers both the IPA-phoneme fallback case
# and the standard word-level path when phoneme_align=True.
if phoneme_align:
_apply_pyphen_syllables(wx_result["segments"])
# Free alignment model before diarization (pyannote is also VRAM-heavy).
del model_a, metadata
gc.collect()
@@ -1194,13 +1234,6 @@ def run_lyric_transcription(
vtt_path.write_text(vtt_content, encoding="utf-8")
output_files.append({"name": vtt_path.name, "path": str(vtt_path), "type": "vtt"})
# Syllable-level ELRC (phoneme alignment only)
if "sylrc" in output_formats:
sylrc_path = out_path / f"{base_name}.sylrc"
sylrc_content = result_to_sylrc(result)
sylrc_path.write_text(sylrc_content, encoding="utf-8")
output_files.append({"name": sylrc_path.name, "path": str(sylrc_path), "type": "sylrc"})
# Update job with results
result_summary = {
"model": result.model,