Added lsync syllable engine.
This commit is contained in:
1 parent
3d61f5c377
commit
86d182af41
6 files changed
+1685
-8
No files matched your search
@@ -78,6 +78,7 @@ class LyricTranscribeRequest(BaseModel):
|
||||
diarize: bool = False # run speaker diarization (WhisperX only)
|
||||
lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance'
|
||||
phoneme_align: bool = False # use phonemizer + phoneme model for syllable-level timestamps
|
||||
lsync_align: bool = False # use lsync wav2vec2 CTC + pyphen for syllable-level timestamps
|
||||
two_pass: bool = False # run a second VAD pass to fill gaps > 2 s
|
||||
lyrics_source: str = "transcribe" # 'transcribe' | 'auto' | 'tag' | 'file' — when not 'transcribe',
|
||||
# skip Whisper and force-align existing lyrics instead
|
||||
@@ -500,6 +501,7 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
|
||||
diarize=body.diarize,
|
||||
power_preset=body.lrc_power_preset,
|
||||
phoneme_align=body.phoneme_align,
|
||||
lsync_align=body.lsync_align,
|
||||
two_pass=body.two_pass,
|
||||
existing_lyrics=resolved_lyrics,
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -852,6 +852,119 @@ def _reconstruct_with_phonemes(phoneme_aligned: dict, original_words_per_seg: li
|
||||
return phoneme_aligned
|
||||
|
||||
|
||||
_LSYNC_DIR = str(Path(__file__).parent.parent.parent / "lyrics-sync")
|
||||
_lsync_recognizer: dict = {} # device → cached PhonemeRecognizer
|
||||
|
||||
|
||||
def _lsync_syllabify(segments: list, audio_arr, device: str) -> bool:
|
||||
"""Add syllable timestamps to words using lsync CTC forced alignment.
|
||||
|
||||
Uses facebook/wav2vec2-large-960h-lv60-self to obtain character-level CTC
|
||||
emissions for the full audio, then pyphen (via lsync's LyricsProcessor) to
|
||||
assign syllable boundaries with per-character timing accuracy.
|
||||
|
||||
Modifies segments in-place setting word["syllables"] on each aligned word.
|
||||
Words that already have syllables (e.g. from a prior phoneme pass) are
|
||||
skipped. Returns True if at least one word was syllabified, False if the
|
||||
entire pipeline failed (caller should fall back to _apply_pyphen_syllables).
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
if _LSYNC_DIR not in sys.path:
|
||||
sys.path.insert(0, _LSYNC_DIR)
|
||||
try:
|
||||
from lsync.phoneme_recognizer import PhonemeRecognizer
|
||||
from lsync.alignment import Aligner
|
||||
from lsync.lyrics_processor import LyricsProcessor, SEPARATOR
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
# Collect words that still need syllables
|
||||
all_words: list = []
|
||||
word_texts: list = []
|
||||
for seg in segments:
|
||||
for w in seg.get("words", []):
|
||||
if w.get("syllables"):
|
||||
continue # already populated
|
||||
raw = (w.get("word") or "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
clean = re.sub(r"[^\w']", "", raw)
|
||||
if not clean:
|
||||
continue
|
||||
all_words.append(w)
|
||||
word_texts.append(clean.upper())
|
||||
|
||||
if not all_words:
|
||||
return False
|
||||
|
||||
# Build lsync-format transcript: WORD1|WORD2|...|WORDN|
|
||||
transcript = SEPARATOR.join(word_texts) + SEPARATOR
|
||||
|
||||
# Load / cache model
|
||||
try:
|
||||
if device not in _lsync_recognizer:
|
||||
_lsync_recognizer[device] = PhonemeRecognizer(lang="en-US", device=device)
|
||||
recognizer = _lsync_recognizer[device]
|
||||
lp = LyricsProcessor(lang="en-US")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# CTC emissions from full audio (lsync splits into 15-s frames internally)
|
||||
try:
|
||||
emission, _pred, _tx, frame_duration = recognizer.recognize(audio_arr)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Forced CTC alignment
|
||||
try:
|
||||
tokens = recognizer.processor.tokenizer(transcript).input_ids
|
||||
path = Aligner.align(emission, tokens)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Syllable-level Word objects
|
||||
try:
|
||||
syl_words = lp.get_syllables_from_path(transcript, path, frame_duration)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Group syllable tokens into per-word groups (is_word_end marks word boundary)
|
||||
groups: list = []
|
||||
current: list = []
|
||||
for s in syl_words:
|
||||
current.append(s)
|
||||
if s.is_word_end:
|
||||
groups.append(current)
|
||||
current = []
|
||||
if current:
|
||||
groups.append(current)
|
||||
|
||||
if len(groups) != len(all_words):
|
||||
return False # word-count mismatch — can't safely assign
|
||||
|
||||
assigned = 0
|
||||
for word_info, syls in zip(all_words, groups):
|
||||
orig = (word_info.get("word") or "").strip()
|
||||
clean = re.sub(r"[^\w']", "", orig)
|
||||
# Reconstruct display text from original casing using lsync label lengths
|
||||
pos = 0
|
||||
display_parts: list = []
|
||||
for syl in syls:
|
||||
n = len(syl.label)
|
||||
part = clean[pos:pos + n] if pos + n <= len(clean) else syl.label.lower()
|
||||
display_parts.append(part)
|
||||
pos += n
|
||||
word_info["syllables"] = [
|
||||
{"text": txt, "start": s.start, "end": s.end}
|
||||
for txt, s in zip(display_parts, syls)
|
||||
]
|
||||
assigned += 1
|
||||
|
||||
return assigned > 0
|
||||
|
||||
|
||||
def _apply_pyphen_syllables(segments: list) -> None:
|
||||
"""Populate word["syllables"] using pyphen proportional time splitting.
|
||||
|
||||
@@ -1054,6 +1167,7 @@ def run_lyric_transcription(
|
||||
diarize: bool = False,
|
||||
power_preset: str = "standard",
|
||||
phoneme_align: bool = False,
|
||||
lsync_align: bool = False,
|
||||
two_pass: bool = False,
|
||||
existing_lyrics: Optional[str] = None,
|
||||
):
|
||||
@@ -1085,8 +1199,8 @@ def run_lyric_transcription(
|
||||
else:
|
||||
out_path = LRC_SYNC_DIR / job_id
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s two_pass=%s",
|
||||
job_id, model, engine, phoneme_align, two_pass)
|
||||
log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s lsync=%s two_pass=%s",
|
||||
job_id, model, engine, phoneme_align, lsync_align, two_pass)
|
||||
|
||||
try:
|
||||
if engine == "whisperx":
|
||||
@@ -1249,11 +1363,17 @@ def run_lyric_transcription(
|
||||
# timestamps (e.g. last word of a phrase held until the next VAD chunk).
|
||||
_clamp_word_end_times(wx_result["segments"])
|
||||
|
||||
# If syllable timing was requested, fill any words that don't yet have
|
||||
# syllable data using pyphen proportional splitting (same approach used
|
||||
# by the lsync library). This covers both the IPA-phoneme fallback case
|
||||
# and the standard word-level path when phoneme_align=True.
|
||||
if phoneme_align:
|
||||
# Syllable timing —————————————————————————————————————————————
|
||||
# lsync path: CTC forced alignment via wav2vec2 + pyphen syllable
|
||||
# boundaries (more accurate than proportional distribution).
|
||||
# Phoneme path: IPA alignment already populated word["syllables"];
|
||||
# pyphen fills any words that couldn't be aligned.
|
||||
if lsync_align:
|
||||
jm.update_job(job_id, progress="Running lsync syllable alignment (wav2vec2 CTC)...")
|
||||
_lsync_syllabify(wx_result["segments"], audio_arr, device)
|
||||
# Fill any words lsync couldn't align (proportional fallback)
|
||||
_apply_pyphen_syllables(wx_result["segments"])
|
||||
elif phoneme_align:
|
||||
_apply_pyphen_syllables(wx_result["segments"])
|
||||
|
||||
# Free alignment model before diarization (pyannote is also VRAM-heavy).
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="StemEngine" />
|
||||
<link rel="apple-touch-icon" href="/icon.svg" />
|
||||
<script type="module" crossorigin src="/assets/index-CjpeI2XT.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-CkVcWz3N.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DVL4P0ML.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -389,6 +389,7 @@ export const lyricTranscribeProject = (
|
||||
diarize?: boolean
|
||||
lrcPowerPreset?: string
|
||||
phonemeAlign?: boolean
|
||||
lsyncAlign?: boolean
|
||||
twoPass?: boolean
|
||||
}
|
||||
) => {
|
||||
@@ -415,6 +416,7 @@ export const lyricTranscribeProject = (
|
||||
diarize: opts?.diarize ?? false,
|
||||
lrc_power_preset: opts?.lrcPowerPreset ?? 'standard',
|
||||
phoneme_align: phonemeAlign,
|
||||
lsync_align: opts?.lsyncAlign ?? false,
|
||||
two_pass: opts?.twoPass ?? false,
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -1689,6 +1689,7 @@ function StemLyricsRow({
|
||||
// Transcribe state — initialise from Power settings stored in localStorage
|
||||
const [diarize, setDiarize] = useState(false)
|
||||
const [phonemeAlign, setPhonemeAlign] = useState(false)
|
||||
const [lsyncAlign, setLsyncAlign] = useState(false)
|
||||
const [whisperModel, setWhisperModel] = useState(
|
||||
() => localStorage.getItem('stemengine-whisper-model') ?? 'large-v2'
|
||||
)
|
||||
@@ -1822,6 +1823,7 @@ function StemLyricsRow({
|
||||
diarize,
|
||||
lrcPowerPreset: powerPreset,
|
||||
phonemeAlign,
|
||||
lsyncAlign,
|
||||
twoPass,
|
||||
})
|
||||
sessionJobIds.current.add(res.job_id)
|
||||
@@ -1957,6 +1959,15 @@ function StemLyricsRow({
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Phonemes</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Use lsync wav2vec2 CTC forced alignment + pyphen for syllable-level timestamps (more accurate than Phonemes mode)">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={lsyncAlign}
|
||||
onChange={(e) => setLsyncAlign(e.target.checked)}
|
||||
className="w-3.5 h-3.5 rounded accent-violet-500"
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Syllables</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Run a second VAD pass to transcribe gaps larger than 2 seconds">
|
||||
<input
|
||||
type="checkbox"
|
||||
|
||||
Reference in new issue
Block a user