Added lsync syllable engine.

This commit is contained in:
translator committed 2026-03-05 21:01:16 +09:00
1 parent 3d61f5c377
commit 86d182af41
6 files changed
+1685 -8

No files matched your search

+2
View File
@@ -78,6 +78,7 @@ class LyricTranscribeRequest(BaseModel):
diarize: bool = False # run speaker diarization (WhisperX only)
lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance'
phoneme_align: bool = False # use phonemizer + phoneme model for syllable-level timestamps
lsync_align: bool = False # use lsync wav2vec2 CTC + pyphen for syllable-level timestamps
two_pass: bool = False # run a second VAD pass to fill gaps > 2 s
lyrics_source: str = "transcribe" # 'transcribe' | 'auto' | 'tag' | 'file' — when not 'transcribe',
# skip Whisper and force-align existing lyrics instead
@@ -500,6 +501,7 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
diarize=body.diarize,
power_preset=body.lrc_power_preset,
phoneme_align=body.phoneme_align,
lsync_align=body.lsync_align,
two_pass=body.two_pass,
existing_lyrics=resolved_lyrics,
)
File diff suppressed because it is too large. Load diff
+127 -7
View File
@@ -852,6 +852,119 @@ def _reconstruct_with_phonemes(phoneme_aligned: dict, original_words_per_seg: li
return phoneme_aligned
_LSYNC_DIR = str(Path(__file__).parent.parent.parent / "lyrics-sync")
_lsync_recognizer: dict = {} # device → cached PhonemeRecognizer
def _lsync_syllabify(segments: list, audio_arr, device: str) -> bool:
"""Add syllable timestamps to words using lsync CTC forced alignment.
Uses facebook/wav2vec2-large-960h-lv60-self to obtain character-level CTC
emissions for the full audio, then pyphen (via lsync's LyricsProcessor) to
assign syllable boundaries with per-character timing accuracy.
Modifies segments in-place setting word["syllables"] on each aligned word.
Words that already have syllables (e.g. from a prior phoneme pass) are
skipped. Returns True if at least one word was syllabified, False if the
entire pipeline failed (caller should fall back to _apply_pyphen_syllables).
"""
import re
import sys
if _LSYNC_DIR not in sys.path:
sys.path.insert(0, _LSYNC_DIR)
try:
from lsync.phoneme_recognizer import PhonemeRecognizer
from lsync.alignment import Aligner
from lsync.lyrics_processor import LyricsProcessor, SEPARATOR
except ImportError:
return False
# Collect words that still need syllables
all_words: list = []
word_texts: list = []
for seg in segments:
for w in seg.get("words", []):
if w.get("syllables"):
continue # already populated
raw = (w.get("word") or "").strip()
if not raw:
continue
clean = re.sub(r"[^\w']", "", raw)
if not clean:
continue
all_words.append(w)
word_texts.append(clean.upper())
if not all_words:
return False
# Build lsync-format transcript: WORD1|WORD2|...|WORDN|
transcript = SEPARATOR.join(word_texts) + SEPARATOR
# Load / cache model
try:
if device not in _lsync_recognizer:
_lsync_recognizer[device] = PhonemeRecognizer(lang="en-US", device=device)
recognizer = _lsync_recognizer[device]
lp = LyricsProcessor(lang="en-US")
except Exception:
return False
# CTC emissions from full audio (lsync splits into 15-s frames internally)
try:
emission, _pred, _tx, frame_duration = recognizer.recognize(audio_arr)
except Exception:
return False
# Forced CTC alignment
try:
tokens = recognizer.processor.tokenizer(transcript).input_ids
path = Aligner.align(emission, tokens)
except Exception:
return False
# Syllable-level Word objects
try:
syl_words = lp.get_syllables_from_path(transcript, path, frame_duration)
except Exception:
return False
# Group syllable tokens into per-word groups (is_word_end marks word boundary)
groups: list = []
current: list = []
for s in syl_words:
current.append(s)
if s.is_word_end:
groups.append(current)
current = []
if current:
groups.append(current)
if len(groups) != len(all_words):
return False # word-count mismatch — can't safely assign
assigned = 0
for word_info, syls in zip(all_words, groups):
orig = (word_info.get("word") or "").strip()
clean = re.sub(r"[^\w']", "", orig)
# Reconstruct display text from original casing using lsync label lengths
pos = 0
display_parts: list = []
for syl in syls:
n = len(syl.label)
part = clean[pos:pos + n] if pos + n <= len(clean) else syl.label.lower()
display_parts.append(part)
pos += n
word_info["syllables"] = [
{"text": txt, "start": s.start, "end": s.end}
for txt, s in zip(display_parts, syls)
]
assigned += 1
return assigned > 0
def _apply_pyphen_syllables(segments: list) -> None:
"""Populate word["syllables"] using pyphen proportional time splitting.
@@ -1054,6 +1167,7 @@ def run_lyric_transcription(
diarize: bool = False,
power_preset: str = "standard",
phoneme_align: bool = False,
lsync_align: bool = False,
two_pass: bool = False,
existing_lyrics: Optional[str] = None,
):
@@ -1085,8 +1199,8 @@ def run_lyric_transcription(
else:
out_path = LRC_SYNC_DIR / job_id
out_path.mkdir(parents=True, exist_ok=True)
log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s two_pass=%s",
job_id, model, engine, phoneme_align, two_pass)
log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s lsync=%s two_pass=%s",
job_id, model, engine, phoneme_align, lsync_align, two_pass)
try:
if engine == "whisperx":
@@ -1249,11 +1363,17 @@ def run_lyric_transcription(
# timestamps (e.g. last word of a phrase held until the next VAD chunk).
_clamp_word_end_times(wx_result["segments"])
# If syllable timing was requested, fill any words that don't yet have
# syllable data using pyphen proportional splitting (same approach used
# by the lsync library). This covers both the IPA-phoneme fallback case
# and the standard word-level path when phoneme_align=True.
if phoneme_align:
# Syllable timing —————————————————————————————————————————————
# lsync path: CTC forced alignment via wav2vec2 + pyphen syllable
# boundaries (more accurate than proportional distribution).
# Phoneme path: IPA alignment already populated word["syllables"];
# pyphen fills any words that couldn't be aligned.
if lsync_align:
jm.update_job(job_id, progress="Running lsync syllable alignment (wav2vec2 CTC)...")
_lsync_syllabify(wx_result["segments"], audio_arr, device)
# Fill any words lsync couldn't align (proportional fallback)
_apply_pyphen_syllables(wx_result["segments"])
elif phoneme_align:
_apply_pyphen_syllables(wx_result["segments"])
# Free alignment model before diarization (pyannote is also VRAM-heavy).
+1 -1
View File
@@ -19,7 +19,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="StemEngine" />
<link rel="apple-touch-icon" href="/icon.svg" />
<script type="module" crossorigin src="/assets/index-CjpeI2XT.js"></script>
<script type="module" crossorigin src="/assets/index-CkVcWz3N.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DVL4P0ML.css">
</head>
<body>
+2
View File
@@ -389,6 +389,7 @@ export const lyricTranscribeProject = (
diarize?: boolean
lrcPowerPreset?: string
phonemeAlign?: boolean
lsyncAlign?: boolean
twoPass?: boolean
}
) => {
@@ -415,6 +416,7 @@ export const lyricTranscribeProject = (
diarize: opts?.diarize ?? false,
lrc_power_preset: opts?.lrcPowerPreset ?? 'standard',
phoneme_align: phonemeAlign,
lsync_align: opts?.lsyncAlign ?? false,
two_pass: opts?.twoPass ?? false,
}),
})
+11
View File
@@ -1689,6 +1689,7 @@ function StemLyricsRow({
// Transcribe state — initialise from Power settings stored in localStorage
const [diarize, setDiarize] = useState(false)
const [phonemeAlign, setPhonemeAlign] = useState(false)
const [lsyncAlign, setLsyncAlign] = useState(false)
const [whisperModel, setWhisperModel] = useState(
() => localStorage.getItem('stemengine-whisper-model') ?? 'large-v2'
)
@@ -1822,6 +1823,7 @@ function StemLyricsRow({
diarize,
lrcPowerPreset: powerPreset,
phonemeAlign,
lsyncAlign,
twoPass,
})
sessionJobIds.current.add(res.job_id)
@@ -1957,6 +1959,15 @@ function StemLyricsRow({
/>
<span className="text-xs text-slate-600 dark:text-gray-400">Phonemes</span>
</label>
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Use lsync wav2vec2 CTC forced alignment + pyphen for syllable-level timestamps (more accurate than Phonemes mode)">
<input
type="checkbox"
checked={lsyncAlign}
onChange={(e) => setLsyncAlign(e.target.checked)}
className="w-3.5 h-3.5 rounded accent-violet-500"
/>
<span className="text-xs text-slate-600 dark:text-gray-400">Syllables</span>
</label>
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Run a second VAD pass to transcribe gaps larger than 2 seconds">
<input
type="checkbox"