From 86d182af41152b78aa7df0baf5b98cc8ec48d9ed Mon Sep 17 00:00:00 2001 From: translator Date: Thu, 5 Mar 2026 21:01:16 +0900 Subject: [PATCH] Added lsync syllable engine. --- backend/routers/projects.py | 2 + .../20260304_lyric_transcriber_service.py | 1542 +++++++++++++++++ backend/services/lyric_transcriber_service.py | 134 +- backend/static/index.html | 2 +- frontend/src/api/client.ts | 2 + frontend/src/pages/ProjectDetail.tsx | 11 + 6 files changed, 1685 insertions(+), 8 deletions(-) create mode 100644 backend/services/20260304_lyric_transcriber_service.py diff --git a/backend/routers/projects.py b/backend/routers/projects.py index a3fea58..f6d4d08 100755 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -78,6 +78,7 @@ class LyricTranscribeRequest(BaseModel): diarize: bool = False # run speaker diarization (WhisperX only) lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance' phoneme_align: bool = False # use phonemizer + phoneme model for syllable-level timestamps + lsync_align: bool = False # use lsync wav2vec2 CTC + pyphen for syllable-level timestamps two_pass: bool = False # run a second VAD pass to fill gaps > 2 s lyrics_source: str = "transcribe" # 'transcribe' | 'auto' | 'tag' | 'file' — when not 'transcribe', # skip Whisper and force-align existing lyrics instead @@ -500,6 +501,7 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest): diarize=body.diarize, power_preset=body.lrc_power_preset, phoneme_align=body.phoneme_align, + lsync_align=body.lsync_align, two_pass=body.two_pass, existing_lyrics=resolved_lyrics, ) diff --git a/backend/services/20260304_lyric_transcriber_service.py b/backend/services/20260304_lyric_transcriber_service.py new file mode 100644 index 0000000..74e8ad4 --- /dev/null +++ b/backend/services/20260304_lyric_transcriber_service.py @@ -0,0 +1,1542 @@ +""" +Lyric transcription service using OpenAI Whisper. +Based on Scriberr's transcription approach - provides word-level timestamps +and speaker diarization support for lyrics. + +This service transcribes vocals to text with precise timing information, +suitable for generating LRC and ELRC files. +""" +import contextlib +import threading +import io +import json +import os +import tempfile +import warnings +from pathlib import Path +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union +from dataclasses import dataclass + +# Whisper's timing.py tries to JIT-compile Triton CUDA kernels for DTW alignment. +# This requires the full CUDA toolkit (nvcc), which is not present in the runtime +# image. The fallback (NumPy median kernel) is correct and only marginally slower, +# so suppress the noisy UserWarning. +warnings.filterwarnings( + "ignore", + message="Failed to launch Triton kernels", + category=UserWarning, +) + +# PyTorch 2.6 changed torch.load's default from weights_only=False to True. +# WhisperX / wav2vec2 / pyannote checkpoints contain arbitrary omegaconf and +# other custom types that are not in the default allowlist, and chasing each +# class individually is fragile. Restore the pre-2.6 default (weights_only +# not set by caller → False) by monkey-patching torch.load at startup. +# All checkpoint sources used here (HuggingFace, faster-whisper, pyannote) +# are trusted, so the security trade-off is acceptable. +try: + import torch as _torch + _orig_torch_load = _torch.load + def _patched_torch_load(*args, **kwargs): + # Force weights_only=False regardless of what the caller passed. + # transformers explicitly passes weights_only=True (not just as a + # default), so setdefault is insufficient — we must overwrite it. + # All checkpoint sources used here (HuggingFace, faster-whisper, + # pyannote) are trusted, so the security trade-off is acceptable. + kwargs['weights_only'] = False + return _orig_torch_load(*args, **kwargs) + _torch.load = _patched_torch_load +except Exception: + pass + +from .. import job_manager as jm +from ..config import JOBS_DIR, LRC_SYNC_DIR + + +@dataclass +class WordTimestamp: + """Word-level timestamp information.""" + word: str + start: float + end: float + confidence: Optional[float] = None + phonemes: Optional[List[dict]] = None # [{"char": str, "start": float, "end": float}, ...] + syllables: Optional[List[dict]] = None # [{"text": str, "start": float, "end": float}, ...] + + +@dataclass +class Segment: + """Segment (line/phrase) level timestamp information.""" + text: str + start: float + end: float + words: List[WordTimestamp] + speaker: Optional[str] = None + + +@dataclass +class LyricTranscriptionResult: + """Complete transcription result.""" + text: str + segments: List[Segment] + language: str + duration: float + model: str + + +# Available Whisper models with their characteristics +WHISPER_MODELS = { + "tiny": { + "id": "tiny", + "label": "Tiny (fastest, least accurate)", + "size": "39 MB", + "relative_speed": "~32x", + "english_only": False, + }, + "base": { + "id": "base", + "label": "Base (fast, good accuracy)", + "size": "74 MB", + "relative_speed": "~16x", + "english_only": False, + }, + "small": { + "id": "small", + "label": "Small (balanced)", + "size": "244 MB", + "relative_speed": "~6x", + "english_only": False, + }, + "medium": { + "id": "medium", + "label": "Medium (slower, better accuracy)", + "size": "769 MB", + "relative_speed": "~2x", + "english_only": False, + }, + "large": { + "id": "large", + "label": "Large (slowest, best accuracy)", + "size": "1550 MB", + "relative_speed": "1x", + "english_only": False, + }, + "large-v2": { + "id": "large-v2", + "label": "Large v2 (recommended)", + "size": "1550 MB", + "relative_speed": "1x", + "english_only": False, + }, + "large-v3": { + "id": "large-v3", + "label": "Large v3 (latest)", + "size": "1550 MB", + "relative_speed": "1x", + "english_only": False, + }, +} + +DEFAULT_WHISPER_MODEL = "large-v2" + + +_whisper_cache: Dict[str, Any] = {} + +# Set to False when a CUDA OOM or "unknown error" corrupts the GPU context. +# Once False, all subsequent Whisper loads are forced to CPU so the server +# keeps working without requiring a restart. +_cuda_ok: bool = True + + +def _mark_cuda_failed() -> None: + """Record that the CUDA context is bad and purge any GPU-resident models.""" + global _cuda_ok + _cuda_ok = False + _whisper_cache.clear() + try: + import torch + torch.cuda.empty_cache() + except Exception: + pass + + +def reset_cuda() -> dict: + """ + Re-enable GPU acceleration after a CUDA error recovery. + + The transcriber falls back to CPU when a CUDA OOM or 'unknown error' + occurs. Call this once the GPU is confirmed free to restore GPU usage + without restarting the container. + """ + global _cuda_ok + _cuda_ok = True + _whisper_cache.clear() + cuda_available = False + try: + import torch + if torch.cuda.is_available(): + torch.cuda.empty_cache() + cuda_available = True + except Exception: + pass + return {"ok": True, "cuda_available": cuda_available, "cuda_ok": _cuda_ok} + + +def get_whisper_model(model_name: str = DEFAULT_WHISPER_MODEL, device: Optional[str] = None): + """ + Load and in-process-cache a Whisper model. + + Reuses an already-loaded model of the same name to avoid reloading ~1.5 GB + weights on every transcription call. If a *different* model is requested, + the cached one is evicted and the CUDA allocator cache is flushed first so + that the new model can fit alongside any other resident GPU allocations. + + When `device` is explicitly passed (e.g. "cpu") it overrides the auto-detect + logic. This is used by the CUDA-error recovery path in `transcribe_lyrics`. + """ + import whisper + + cache_key = f"{model_name}:{device or 'auto'}" + + if cache_key in _whisper_cache: + return _whisper_cache[cache_key] + + # Evict any previously cached model and release its GPU memory before + # loading the new one. + if _whisper_cache: + _whisper_cache.clear() + try: + import torch + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + + load_kwargs: dict = {} + if device is not None: + load_kwargs["device"] = device + + model = whisper.load_model(model_name, **load_kwargs) + _whisper_cache[cache_key] = model + return model + + +class _LineCapture(io.RawIOBase): + """ + File-like object that buffers writes and calls `callback` once per complete line. + Used to intercept Whisper's verbose stdout output for live progress reporting. + """ + def __init__(self, callback: Callable[[str], None]) -> None: + self._cb = callback + self._buf = '' + + def write(self, text) -> int: # type: ignore[override] + if isinstance(text, (bytes, bytearray)): + text = text.decode('utf-8', errors='replace') + self._buf += text + while '\n' in self._buf: + line, self._buf = self._buf.split('\n', 1) + cleaned = line.strip() + if cleaned: + self._cb(cleaned) + return len(text) + + def flush(self) -> None: + pass + + # Some libraries call fileno(); raise so they fall back gracefully. + def fileno(self): # type: ignore[override] + raise io.UnsupportedOperation('fileno') + + + +def transcribe_lyrics( + audio_path: str, + model_name: str = DEFAULT_WHISPER_MODEL, + language: Optional[str] = None, + task: Literal["transcribe", "translate"] = "transcribe", + include_word_timestamps: bool = True, + initial_prompt: Optional[str] = None, + temperature: float = 0.0, + condition_on_previous_text: bool = False, + compression_ratio_threshold: float = 1.35, + beam_size: Optional[int] = None, + line_callback: Optional[Callable[[str], None]] = None, +) -> LyricTranscriptionResult: + """ + Transcribe vocals to lyrics with word-level timestamps. + + Anti-hallucination defaults (differ from stock Whisper): + - condition_on_previous_text=False — prevents the model from feeding + a repeated phrase back as context, breaking the repetition loop. + - temperature fallback tuple — when temperature=0.0, Whisper is + automatically given (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) so it can retry + a stuck segment with increasing randomness. + - compression_ratio_threshold=1.35 — flags repetitive segments sooner + (stock default is 2.4) and triggers the temperature fallback. + - _deduplicate_segments post-pass — removes any repeated segments or + internally-looped phrases that still slip through. + + Args: + audio_path: Path to audio file (should be vocals stem for best results) + model_name: Whisper model to use + language: ISO-639-1 language code (auto-detected if None) + task: 'transcribe' for original language, 'translate' for English + include_word_timestamps: Whether to include word-level timestamps + initial_prompt: Optional text to guide the model's style + temperature: Base sampling temperature; 0.0 enables the fallback tuple + condition_on_previous_text: Feed previous segment as context (False prevents loops) + compression_ratio_threshold: Retry segment above this gzip ratio (lower = stricter) + + Returns: + LyricTranscriptionResult with text, segments, and timestamps + """ + import whisper + + # Free any reserved-but-unallocated GPU memory from prior jobs (e.g. Demucs, + # BS-RoFormer) before the Whisper model is loaded / used. + try: + import torch + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + + # If a previous job corrupted the CUDA context (OOM → "unknown error"), + # force CPU so this job still completes rather than failing immediately. + forced_device: Optional[str] = None if _cuda_ok else "cpu" + model = get_whisper_model(model_name, device=forced_device) + + # When temperature is 0 (greedy), pass a fallback sequence so Whisper can + # recover from stuck segments by retrying with increasing randomness. + effective_temperature: Any = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) if temperature == 0.0 else temperature + + decode_options: dict = { + "task": task, + "temperature": effective_temperature, + "condition_on_previous_text": condition_on_previous_text, + "compression_ratio_threshold": compression_ratio_threshold, + } + + if language: + decode_options["language"] = language + if initial_prompt: + decode_options["initial_prompt"] = initial_prompt + if beam_size is not None: + decode_options["beam_size"] = beam_size + + def _run_transcribe(mdl: Any) -> Any: + """Run model.transcribe(), capturing verbose output when a callback is set.""" + if line_callback is not None: + _cap = _LineCapture(line_callback) + with contextlib.redirect_stdout(_cap), contextlib.redirect_stderr(_cap): + return mdl.transcribe( + audio_path, + word_timestamps=include_word_timestamps, + verbose=True, + **decode_options, + ) + return mdl.transcribe( + audio_path, + word_timestamps=include_word_timestamps, + **decode_options, + ) + + # Run transcription with word timestamps. + # On any CUDA error (OOM already happened, or the context was already + # corrupted), mark the GPU as failed, reload the model on CPU, and retry. + # This keeps the server usable without a restart. + try: + result = _run_transcribe(model) + except RuntimeError as exc: + err_str = str(exc) + if "CUDA" in err_str or "cuda" in err_str: + _mark_cuda_failed() + if line_callback: + line_callback("⚠️ CUDA error — retrying on CPU (server restart not required)") + cpu_model = get_whisper_model(model_name, device="cpu") + result = _run_transcribe(cpu_model) + else: + raise + + # Parse results into structured format + raw_segments: List[Segment] = [] + for seg in result["segments"]: + words = [] + if include_word_timestamps and "words" in seg: + for word_info in seg["words"]: + words.append(WordTimestamp( + word=word_info["word"].strip(), + start=word_info["start"], + end=word_info["end"], + confidence=word_info.get("probability") + )) + + raw_segments.append(Segment( + text=seg["text"].strip(), + start=seg["start"], + end=seg["end"], + words=words + )) + + # Calculate total duration + duration = max(seg.end for seg in raw_segments) if raw_segments else 0.0 + + return LyricTranscriptionResult( + text=result["text"].strip(), + segments=raw_segments, + language=result.get("language", "unknown"), + duration=duration, + model=f"whisper-{model_name}" + ) + + +def result_to_lrc(result: LyricTranscriptionResult) -> str: + """Convert transcription result to LRC format (line-level timestamps). + + Speaker info (when present) is encoded as |sp:SPEAKER_00| immediately + after the LRC timestamp so the lyric player can colour-code lines per + speaker while remaining compatible with standard LRC parsers (which + simply ignore the tag as leading text). + """ + lines = [] + for seg in result.segments: + minutes = int(seg.start // 60) + seconds = int(seg.start % 60) + centis = int((seg.start % 1) * 100) + speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else "" + lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}") + return "\n".join(lines) + + +def result_to_elrc(result: LyricTranscriptionResult) -> str: + """Convert transcription result to ELRC format. + + Uses syllable-level timestamps when phoneme alignment data is present + (word.syllables populated), otherwise falls back to word-level timestamps. + Speaker info is encoded as |sp:SPEAKER_00| prepended to each line. + """ + lines = [] + for seg in result.segments: + speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else "" + if not seg.words: + # Fallback to segment-level if no word timestamps + minutes = int(seg.start // 60) + seconds = int(seg.start % 60) + centis = int((seg.start % 1) * 100) + lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}") + continue + + # Line timestamp is start of first word + line_start = seg.words[0].start + line_min = int(line_start // 60) + line_sec = int(line_start % 60) + line_centi = int((line_start % 1) * 100) + + word_parts: list = [] + for word in seg.words: + if word.syllables: + # Syllable-level: split the orthographic word into n chunks and + # pair each with its phoneme timestamp. + ortho = _ortho_split(word.word, len(word.syllables)) + for syl, display_text in zip(word.syllables, ortho): + s = syl["start"] + s_min = int(s // 60) + s_sec = int(s % 60) + s_centi = int((s % 1) * 100) + word_parts.append(f"<{s_min:02d}:{s_sec:02d}.{s_centi:02d}>{display_text}") + else: + # Word-level fallback. + w_min = int(word.start // 60) + w_sec = int(word.start % 60) + w_centi = int((word.start % 1) * 100) + word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}") + word_parts.append(" ") + + line_content = "".join(word_parts).rstrip() + lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{line_content}") + + return "\n".join(lines) + + +def result_to_json(result: LyricTranscriptionResult) -> Dict[str, Any]: + """Convert transcription result to JSON-serializable dict.""" + return { + "text": result.text, + "language": result.language, + "duration": result.duration, + "model": result.model, + "segments": [ + { + "text": seg.text, + "start": seg.start, + "end": seg.end, + "speaker": seg.speaker, + "words": [ + { + "word": w.word, + "start": w.start, + "end": w.end, + "confidence": w.confidence, + "phonemes": w.phonemes or [], + "syllables": w.syllables or [], + } + for w in seg.words + ] + } + for seg in result.segments + ] + } + + +def result_to_txt(result: LyricTranscriptionResult) -> str: + """Convert transcription result to plain text. + + One Whisper segment per line so the output can be fed directly into the + ctc-forced-aligner (lrc_service) which splits on newlines to identify + individual lyric lines. Falls back to the joined text if no segments. + """ + if result.segments: + return '\n'.join(seg.text.strip() for seg in result.segments if seg.text.strip()) + return result.text + + +def result_to_srt(result: LyricTranscriptionResult) -> str: + """Convert transcription result to SRT subtitle format.""" + lines = [] + for i, seg in enumerate(result.segments, 1): + # Format: HH:MM:SS,mmm + def fmt_srt_time(t: float) -> str: + hours = int(t // 3600) + mins = int((t % 3600) // 60) + secs = int(t % 60) + millis = int((t % 1) * 1000) + return f"{hours:02d}:{mins:02d}:{secs:02d},{millis:03d}" + + lines.append(str(i)) + lines.append(f"{fmt_srt_time(seg.start)} --> {fmt_srt_time(seg.end)}") + lines.append(seg.text) + lines.append("") # Empty line between entries + + return "\n".join(lines) + + +def result_to_vtt(result: LyricTranscriptionResult) -> str: + """Convert transcription result to WebVTT subtitle format.""" + lines = ["WEBVTT", ""] + + for seg in result.segments: + def fmt_vtt_time(t: float) -> str: + hours = int(t // 3600) + mins = int((t % 3600) // 60) + secs = int(t % 60) + millis = int((t % 1) * 1000) + return f"{hours:02d}:{mins:02d}:{secs:02d}.{millis:03d}" + + lines.append(f"{fmt_vtt_time(seg.start)} --> {fmt_vtt_time(seg.end)}") + lines.append(seg.text) + lines.append("") + + return "\n".join(lines) + + +def _ortho_split(word_text: str, n: int) -> List[str]: + """Split orthographic word text into n roughly equal character chunks. + + Used by result_to_sylrc to map phoneme-derived syllable count back to + display-friendly English text (e.g. "hello" / 2 → ["hel", "lo"]). + The split is proportional — not linguistically perfect, but readable. + """ + text = word_text.strip() + if n <= 1 or len(text) <= 1: + return [text] if text else [''] + base, rem = divmod(len(text), n) + parts: List[str] = [] + i = 0 + for k in range(n): + size = base + (1 if k < rem else 0) + parts.append(text[i:i + size]) + i += size + return [p for p in parts if p] or [text] + + + + +_PUNCT_BREAK = frozenset('.!?,;') + + +def _split_long_segments(segments: list, max_words: int = 10) -> list: + """ + Split over-long WhisperX segments into shorter lines suitable for LRC display. + + Strategy (applied per segment): + 1. Walk the word list looking for *punctuation boundaries* — words whose + text ends with a character in _PUNCT_BREAK. When such a boundary is + found AND the current chunk already has at least 3 words, emit a new + sub-segment there. + 2. If no punctuation boundary is found within `max_words` words, force- + split at `max_words` regardless. + + Each sub-segment inherits the speaker label and gets timestamps from the + first / last word in the slice. Segments with fewer than 2 words are left + unchanged. + """ + if max_words <= 0: + return segments + + result = [] + for seg in segments: + words = seg.get("words") or [] + if len(words) <= max_words: + result.append(seg) + continue + + # Build sub-segment slices + chunks: list[list] = [] + chunk: list = [] + for w in words: + chunk.append(w) + text = w.get("word", "").rstrip() + at_punct = bool(text) and text[-1] in _PUNCT_BREAK + at_limit = len(chunk) >= max_words + if (at_punct and len(chunk) >= 3) or at_limit: + chunks.append(chunk) + chunk = [] + if chunk: + chunks.append(chunk) + + for ch in chunks: + first, last = ch[0], ch[-1] + sub_text = " ".join(w.get("word", "").strip() for w in ch).strip() + result.append({ + "start": first.get("start") if first.get("start") is not None else seg["start"], + "end": last.get("end") if last.get("end") is not None else seg["end"], + "text": sub_text, + "words": ch, + "speaker": seg.get("speaker"), + }) + + return result + + +_VOCAL_KEYWORDS = frozenset(('vocal', 'vocals', 'voice', 'lead', 'sing')) +_AUDIO_EXTS = frozenset(('.mp3', '.wav', '.flac', '.ogg', '.m4a', '.aac')) + + +def _ensure_vocals(audio_path: str, job_id: str) -> str: + """ + Return the path to the best available vocals audio for transcription. + + Priority: + 1. audio_path already looks like a vocals stem (filename keyword check). + 2. A vocals file exists in the same Stems/ directory — use it. + 3. Run htdemucs --two-stems vocals to extract vocals from audio_path. + Falls back to audio_path unchanged on any failure. + """ + import subprocess, sys + + p = Path(audio_path) + stem_lower = p.stem.lower() + + # 1. Already a vocals stem? + if any(kw in stem_lower for kw in _VOCAL_KEYWORDS): + return audio_path + + # 2. Look for a sibling vocals file in the same directory. + for f in sorted(p.parent.iterdir()): + if f == p or not f.is_file(): + continue + if f.suffix.lower() not in _AUDIO_EXTS: + continue + if any(kw in f.stem.lower() for kw in _VOCAL_KEYWORDS): + jm.update_job(job_id, progress=f"Using existing vocal stem: {f.name}") + return str(f) + + # 3. No vocal stem found — extract with htdemucs --two-stems vocals. + jm.update_job(job_id, progress="No vocal stem found — extracting vocals with Demucs (htdemucs)...") + extract_dir = LRC_SYNC_DIR / job_id / "vocals_extract" + extract_dir.mkdir(parents=True, exist_ok=True) + + cmd = [ + sys.executable, '-m', 'demucs', + '--name', 'htdemucs', + '--two-stems', 'vocals', + '--out', str(extract_dir), + audio_path, + ] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600) + if proc.returncode == 0: + # Output layout: /htdemucs//vocals.wav + for candidate in sorted(extract_dir.rglob('vocals.*')): + if candidate.is_file() and candidate.suffix.lower() in _AUDIO_EXTS: + jm.update_job(job_id, progress=f"Vocals extracted: {candidate.name}") + return str(candidate) + else: + jm.update_job(job_id, progress=f"Demucs exited {proc.returncode} — using original audio.") + except Exception as _e: + jm.update_job(job_id, progress=f"Demucs extraction failed ({_e}) — using original audio.") + + return audio_path + + +# --------------------------------------------------------------------------- +# Phoneme-level alignment helpers +# --------------------------------------------------------------------------- + +# IPA vowel characters that constitute syllable nuclei. +_IPA_VOWELS = frozenset("aeiouæɑɒɔəɛɜɪʊʌɐɘɵɞʉɯɨ") + +# Map Whisper ISO-639-1 language codes → espeak-ng language tags. +_ESPEAK_LANG_MAP: dict = { + 'en': 'en-us', 'fr': 'fr', 'de': 'de', 'es': 'es', + 'it': 'it', 'pt': 'pt', 'nl': 'nl', 'pl': 'pl', + 'ru': 'ru', 'ja': 'ja', 'zh': 'cmn', 'ko': 'ko', + 'ar': 'ar', 'hi': 'hi', 'tr': 'tr', 'sv': 'sv', + 'cs': 'cs', 'ro': 'ro', 'uk': 'uk', 'hu': 'hu', +} + + +def _lang_code_to_espeak(lang_code: str) -> str: + """Map a Whisper ISO-639-1 language code to an espeak-ng language tag.""" + return _ESPEAK_LANG_MAP.get(lang_code.split('-')[0].lower(), 'en-us') + + +def _phonemize_segments(segments: list, lang_code: str) -> tuple: + """Convert each word token in the segments to its espeak IPA phoneme string. + + Uses phonemizer with the espeak backend so the output matches the vocabulary + of the facebook/wav2vec2-xlsr-53-espeak-cv-ft alignment model. + + Returns: + (phoneme_segs, original_words_per_seg) + - phoneme_segs: same structure but each word["word"] is now its IPA + phoneme string (e.g. "hello" → "hɛloʊ"), and segment["text"] is the + space-joined concatenation of all phonemized words. + - original_words_per_seg: list[list[str]] used to restore original text. + + Requires: + pip install phonemizer (plus espeak-ng system package) + """ + from phonemizer.backend import EspeakBackend + from phonemizer.separator import Separator + + espeak_lang = _lang_code_to_espeak(lang_code) + backend = EspeakBackend(espeak_lang, with_stress=False, language_switch='remove-flags') + # phone='' concatenates all phones into one string per word; word='' is + # unused here because each input item is already a single word. + sep = Separator(phone='', word='') + + result_segs: list = [] + original_words_per_seg: list = [] + + for seg in segments: + words = seg.get("words", []) + if not words: + result_segs.append(dict(seg)) + original_words_per_seg.append([]) + continue + + original_words = [w.get("word", "").strip() for w in words] + # Phonemize each word as a separate utterance to keep a 1-to-1 mapping. + phonemized_list = backend.phonemize( + [w or "_" for w in original_words], separator=sep + ) + + new_words = [ + {**w, "word": (ph.strip() or w.get("word", ""))} + for w, ph in zip(words, phonemized_list) + ] + new_seg = { + **seg, + "text": " ".join(nw["word"] for nw in new_words), + "words": new_words, + } + result_segs.append(new_seg) + original_words_per_seg.append(original_words) + + return result_segs, original_words_per_seg + + +def _group_chars_to_syllables(chars: list) -> list: + """Group IPA character timestamps from WhisperX char-level alignment into syllables. + + Rule: each IPA vowel character is a syllable nucleus; preceding consonants + are the onset. Trailing consonants become coda of the preceding syllable. + """ + syllables: list = [] + current: list = [] + + for ch in chars: + c = ch.get("char", "") + current.append(ch) + if c in _IPA_VOWELS: + syllables.append({ + "text": "".join(x["char"] for x in current), + "start": current[0].get("start", ch.get("start", 0.0)), + "end": ch.get("end", 0.0), + }) + current = [] + + # Attach trailing consonants (coda) to the last syllable. + if current: + if syllables: + syllables[-1]["text"] += "".join(x["char"] for x in current) + syllables[-1]["end"] = current[-1].get("end", syllables[-1]["end"]) + else: + syllables.append({ + "text": "".join(x["char"] for x in current), + "start": current[0].get("start", 0.0), + "end": current[-1].get("end", 0.0), + }) + + return syllables + + +def _reconstruct_with_phonemes(phoneme_aligned: dict, original_words_per_seg: list) -> dict: + """After phoneme-level alignment, restore original word texts and attach + character-level phoneme timestamps and syllable groupings to each word. + + WhisperX stores char alignments on the *segment* dict (not on each word), + so we partition the segment-level chars list into per-word slices by time. + """ + for seg_idx, seg in enumerate(phoneme_aligned.get("segments", [])): + orig = (original_words_per_seg[seg_idx] + if seg_idx < len(original_words_per_seg) else []) + + # chars live on the segment in this version of whisperx + seg_chars = [c for c in seg.get("chars", []) if c.get("start") is not None] + char_ptr = 0 + + for word_idx, word in enumerate(seg.get("words", [])): + if word_idx < len(orig): + word["word"] = orig[word_idx] + + # Try per-word chars first (older whisperx put them there directly) + word_chars = word.get("chars", []) + if not word_chars and seg_chars: + # Partition segment chars into this word by time window + w_start = word.get("start") + w_end = word.get("end") + if w_start is not None and w_end is not None: + word_chars = [] + while char_ptr < len(seg_chars): + c = seg_chars[char_ptr] + c_start = c.get("start", w_end + 1) + if c_start > w_end + 0.005: + break + if c_start >= w_start - 0.005: + word_chars.append(c) + char_ptr += 1 + + if word_chars: + word["phonemes"] = word_chars + word["syllables"] = _group_chars_to_syllables(word_chars) + + return phoneme_aligned + + +def _apply_pyphen_syllables(segments: list) -> None: + """Populate word["syllables"] using pyphen proportional time splitting. + + Uses the same pyphen library as the lyrics-sync lsync module to identify + English syllable boundaries, then distributes each word's [start, end] + range proportionally across its syllables. + + Skips words that already have syllable data (e.g. from the IPA phoneme + path). Modifies segments in-place. Falls back silently if pyphen is + not installed. + """ + try: + import pyphen + dic = pyphen.Pyphen(lang='en_US') + except ImportError: + return + + for seg in segments: + for word_info in seg.get("words", []): + if word_info.get("syllables"): + continue # already populated by phoneme path + text = word_info.get("word", "").strip() + if not text: + continue + start = float(word_info.get("start") or 0.0) + end = float(word_info.get("end") or start) + duration = max(end - start, 0.0) + + positions = dic.positions(text.lower()) + if not positions: + word_info["syllables"] = [{"text": text, "start": start, "end": end}] + continue + + boundaries = [0] + list(positions) + [len(text)] + total_chars = len(text) + syllables = [] + for i in range(len(boundaries) - 1): + s_char = boundaries[i] + e_char = boundaries[i + 1] + syl_start = start + (s_char / total_chars) * duration + syl_end = start + (e_char / total_chars) * duration + syllables.append({"text": text[s_char:e_char], "start": syl_start, "end": syl_end}) + word_info["syllables"] = syllables + + +def _clamp_word_end_times(segments: list, max_word_duration: float = 1.5) -> None: + """Prevent silence from inflating word end times. + + After CTC alignment the last word of a VAD chunk often has its 'end' + set to the end of the chunk, which may include a long silent tail. + This function: + 1. Ensures no word extends past the start of the next word. + 2. Caps each word at max_word_duration seconds. + + Modifies segments in-place. + """ + all_words = [] + for seg in segments: + for word in seg.get("words", []): + if word.get("start") is not None and word.get("end") is not None: + all_words.append(word) + + for i, word in enumerate(all_words): + end = word["end"] + if i + 1 < len(all_words): + end = min(end, all_words[i + 1]["start"]) + end = min(end, word["start"] + max_word_duration) + word["end"] = max(end, word["start"]) # never go negative + + +def _fill_transcript_gaps( + wx_model, + audio_arr, + segments: list, + batch_size: int, + job_id: str, + min_gap: float = 2.0, +) -> list: + """Second-pass transcription on silent gaps that VAD missed. + + Finds time regions with no detected speech (gaps > min_gap seconds), + re-runs transcription on just those audio slices, and merges any new + segments back into the original list sorted by start time. + + This catches lyrics in quieter passages or after long instrumental + breaks that the default VAD onset threshold (0.500) filters out. + """ + import numpy as np + if not segments: + return segments + + total_dur = len(audio_arr) / 16000.0 + gaps: list[tuple] = [] + + if segments[0]["start"] > min_gap: + gaps.append((0.0, segments[0]["start"])) + for i in range(len(segments) - 1): + g_start = segments[i]["end"] + g_end = segments[i + 1]["start"] + if g_end - g_start > min_gap: + gaps.append((g_start, g_end)) + if total_dur - segments[-1]["end"] > min_gap: + gaps.append((segments[-1]["end"], total_dur)) + + if not gaps: + return segments + + jm.update_job(job_id, progress=f"Pass 2: scanning {len(gaps)} gap(s) for missed lyrics...") + + new_segs: list = [] + for g_start, g_end in gaps: + s0 = int(g_start * 16000) + s1 = int(g_end * 16000) + audio_slice = audio_arr[s0:s1] + if len(audio_slice) < 16000: # skip slices < 1 s + continue + try: + gap_result = wx_model.transcribe(audio_slice, batch_size=batch_size) + for seg in gap_result.get("segments", []): + if not seg.get("text", "").strip(): + continue + seg["start"] = round(seg["start"] + g_start, 3) + seg["end"] = round(seg["end"] + g_start, 3) + for w in seg.get("words", []): + if w.get("start") is not None: + w["start"] = round(w["start"] + g_start, 3) + if w.get("end") is not None: + w["end"] = round(w["end"] + g_start, 3) + new_segs.append(seg) + except Exception: + continue + + if not new_segs: + return segments + + merged = segments + new_segs + merged.sort(key=lambda s: s["start"]) + return merged + + +# Batch size and compute precision per power tier. +# 'compute_type_cuda' is used on GPU; CPU always falls back to 'int8'. +_WHISPERX_POWER_PRESETS: dict = { + 'eco': {'batch_size': 4, 'compute_type_cuda': 'int8_float16'}, + 'standard': {'batch_size': 8, 'compute_type_cuda': 'float16'}, + 'performance': {'batch_size': 16, 'compute_type_cuda': 'float16'}, +} + + +def run_lyric_transcription( + job_id: str, + audio_path: str, + model: str = DEFAULT_WHISPER_MODEL, + language: Optional[str] = None, + task: str = "transcribe", + output_formats: Optional[List[str]] = None, + initial_prompt: Optional[str] = None, + temperature: float = 0.0, + condition_on_previous_text: bool = False, + word_timestamps: bool = True, + beam_size: Optional[int] = None, + output_dir: Optional[str] = None, + engine: str = "whisperx", + diarize: bool = False, + power_preset: str = "standard", + phoneme_align: bool = False, +): + """ + Background task: transcribe vocals/lyrics from audio. + + Args: + job_id: Job identifier + audio_path: Path to audio file + model: Whisper model name + language: Language code (auto-detect if None) + task: 'transcribe' or 'translate' + output_formats: List of output formats ('lrc', 'elrc', 'json', 'txt', 'srt', 'vtt') + initial_prompt: Optional prompt text + temperature: Sampling temperature + output_dir: Custom output directory + engine: 'whisperx' (faster-whisper + phoneme alignment) + diarize: Run speaker diarization after alignment (WhisperX only) + """ + if output_formats is None: + output_formats = ["elrc", "json", "txt"] + + if output_dir: + out_path = Path(output_dir) + else: + out_path = LRC_SYNC_DIR / job_id + out_path.mkdir(parents=True, exist_ok=True) + + try: + if engine == "whisperx": + # ── WhisperX path: faster-whisper backend + phoneme-level alignment ── + try: + import torch + import whisperx + except ImportError as _imp_err: + jm.update_job( + job_id, + status=jm.JobStatus.ERROR, + error=f"WhisperX is not installed ({_imp_err}). " + "Install with: pip install whisperx", + progress="Error.", + ) + return + + jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading WhisperX model ({model})...") + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + device = "cuda" if torch.cuda.is_available() else "cpu" + _wx_preset = _WHISPERX_POWER_PRESETS.get(power_preset, _WHISPERX_POWER_PRESETS['standard']) + batch_size = _wx_preset['batch_size'] + compute_type = _wx_preset['compute_type_cuda'] if device == "cuda" else "int8" + + # Auto-ensure vocals stem for best transcription quality. + audio_path = _ensure_vocals(audio_path, job_id) + + # Music-tuned ASR options: + # no_speech_threshold 0.45 (< default 0.60) — keep more segments + # compression_ratio_threshold 2.2 (< default 2.4) — reject repetition sooner + # hallucination_silence_threshold — suppress long silence tokens (3.8+) + _asr_opts: dict = { + "no_speech_threshold": 0.45, + "condition_on_previous_text": False, + "compression_ratio_threshold": 2.2, + "temperatures": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0], + "hallucination_silence_threshold": 2.0, # WhisperX ≥3.8 + } + if initial_prompt: + _asr_opts["initial_prompt"] = initial_prompt + + # Music-tuned VAD options: + # lower onset/offset catches quieter vocals that default 0.5/0.363 + # would filter as noise. + _vad_opts: dict = { + "vad_onset": 0.350, + "vad_offset": 0.200, + } + + wx_model = whisperx.load_model( + model, device, + compute_type=compute_type, + language=language or None, + asr_options=_asr_opts, + vad_options=_vad_opts, + ) + audio_arr = whisperx.load_audio(audio_path) + duration = float(len(audio_arr)) / 16000.0 + + jm.update_job(job_id, progress=f"Pass 1: Transcribing with WhisperX ({model}, {power_preset})...") + wx_result = wx_model.transcribe(audio_arr, batch_size=batch_size) + detected_lang = wx_result.get("language") or language or "unknown" + + # Pass 2: re-transcribe any gaps > 2 s that VAD may have missed + # (e.g. quieter verse sections after long instrumentals). + wx_result["segments"] = _fill_transcript_gaps( + wx_model, audio_arr, wx_result.get("segments", []), + batch_size, job_id, min_gap=2.0, + ) + + # Free the Whisper model from VRAM before loading the alignment model. + # large-v2/v3 consume 3–6 GB; keeping both models resident simultaneously + # causes OOM on ≤8 GB cards. + import gc + del wx_model + gc.collect() + if device == "cuda": + torch.cuda.empty_cache() + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + # ── Alignment (word-level or phoneme-level) ───────────────────────── + _use_phoneme = phoneme_align + _phoneme_orig_words: list = [] + + if _use_phoneme: + jm.update_job(job_id, progress="Phonemizing transcript for phoneme-level alignment...") + try: + _phoneme_segs, _phoneme_orig_words = _phonemize_segments( + wx_result["segments"], detected_lang + ) + except Exception as _pe: + jm.update_job( + job_id, + progress=f"Phonemizer failed ({_pe}) — falling back to word-level alignment.", + ) + _use_phoneme = False + + if _use_phoneme: + jm.update_job( + job_id, + progress="Loading phoneme alignment model (facebook/wav2vec2-xlsr-53-espeak-cv-ft)...", + ) + model_a, metadata = whisperx.load_align_model( + language_code=detected_lang, + device=device, + model_name="facebook/wav2vec2-xlsr-53-espeak-cv-ft", + ) + jm.update_job(job_id, progress="Aligning phonemes (WhisperX)...") + wx_result = whisperx.align( + _phoneme_segs, model_a, metadata, audio_arr, device, + interpolate_method="linear", + return_char_alignments=True, + ) + wx_result = _reconstruct_with_phonemes(wx_result, _phoneme_orig_words) + else: + jm.update_job(job_id, progress="Aligning word timestamps (WhisperX)...") + model_a, metadata = whisperx.load_align_model( + language_code=detected_lang, device=device + ) + wx_result = whisperx.align( + wx_result["segments"], model_a, metadata, audio_arr, device, + interpolate_method="linear", + return_char_alignments=False, + ) + + # Clamp word end times: prevent silence tails from inflating end + # timestamps (e.g. last word of a phrase held until the next VAD chunk). + _clamp_word_end_times(wx_result["segments"]) + + # If syllable timing was requested, fill any words that don't yet have + # syllable data using pyphen proportional splitting (same approach used + # by the lsync library). This covers both the IPA-phoneme fallback case + # and the standard word-level path when phoneme_align=True. + if phoneme_align: + _apply_pyphen_syllables(wx_result["segments"]) + + # Free alignment model before diarization (pyannote is also VRAM-heavy). + del model_a, metadata + gc.collect() + if device == "cuda": + torch.cuda.empty_cache() + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + if diarize: + jm.update_job(job_id, progress="Running speaker diarization...") + try: + from ..routers.settings import get_hf_token + hf_token = get_hf_token() + if not hf_token: + raise RuntimeError( + "No Hugging Face token configured. " + "Add your token in Settings → Hugging Face, or set the HF_TOKEN env var." + ) + # Log in globally so pyannote.audio picks up the token regardless of + # which param name the installed version expects (use_auth_token vs token). + try: + from huggingface_hub import login as _hf_login + _hf_login(token=hf_token, add_to_git_credential=False) + except Exception: + pass + diarize_model = whisperx.DiarizationPipeline(use_auth_token=hf_token, device=device) + diarize_segments = diarize_model(audio_arr) + wx_result = whisperx.assign_word_speakers(diarize_segments, wx_result) + except Exception as _diar_err: + jm.update_job(job_id, progress=f"Diarization failed ({_diar_err}), continuing without speaker labels...") + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + # Split over-long segments into shorter lyric lines before converting. + # Punctuation boundaries are preferred; falls back to a hard word-count cut. + raw_segments = _split_long_segments(wx_result["segments"], max_words=10) + + # Convert WhisperX output → LyricTranscriptionResult + segments: List[Segment] = [] + for seg in raw_segments: + words = [] + for w in seg.get("words", []): + _ws = w.get("start") + _we = w.get("end") + words.append(WordTimestamp( + word=w.get("word", ""), + start=float(_ws if _ws is not None else seg["start"]), + end=float(_we if _we is not None else seg["end"]), + confidence=w.get("score"), + phonemes=w.get("phonemes"), + syllables=w.get("syllables"), + )) + segments.append(Segment( + text=seg["text"].strip(), + start=float(seg["start"]), + end=float(seg["end"]), + words=words, + speaker=seg.get("speaker"), + )) + + result = LyricTranscriptionResult( + text=" ".join(seg.text for seg in segments), + segments=segments, + language=detected_lang, + duration=duration, + model=f"whisperx-{model}", + ) + + # ── Release remaining GPU memory ────────────────────────────────────── + # wx_model and model_a are already freed per-phase above. + # Release audio_arr (raw float32 audio buffer — can be hundreds of MB). + del audio_arr + try: + del diarize_model # only defined when diarize=True + except NameError: + pass + try: + torch.cuda.empty_cache() + except Exception: + pass + + else: + # ── Standard Whisper path ───────────────────────────────────────────── + jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading Whisper model ({model})...") + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + jm.update_job(job_id, progress=f"Transcribing lyrics with Whisper ({model})...") + + # Accumulate Whisper's verbose segment output for live progress display. + # Noisy lines (tqdm bars, empty lines) are filtered out. + _log: List[str] = [] + _NOISE_TOKENS = ('%|', 'it/s', 'B/s', '100%') + + def _on_line(line: str) -> None: + if any(tok in line for tok in _NOISE_TOKENS): + return + _log.append(line) + if len(_log) > 30: + _log.pop(0) + jm.update_job(job_id, progress='\n'.join(_log)) + + _TRANSCRIBE_TIMEOUT = 7200 # 2-hour hard cap + _result_box: list = [None] + _exc_box: list = [None] + + def _do_transcribe(): + try: + _result_box[0] = transcribe_lyrics( + audio_path=audio_path, + model_name=model, + language=language, + task=task, # type: ignore + include_word_timestamps=word_timestamps, + initial_prompt=initial_prompt, + temperature=temperature, + condition_on_previous_text=condition_on_previous_text, + beam_size=beam_size, + line_callback=_on_line, + ) + except Exception as _e: + _exc_box[0] = _e + + _t = threading.Thread(target=_do_transcribe, daemon=True) + _t.start() + _t.join(timeout=_TRANSCRIBE_TIMEOUT) + + if _t.is_alive(): + jm.update_job( + job_id, + status=jm.JobStatus.ERROR, + error=f"Transcription timed out after {_TRANSCRIBE_TIMEOUT // 3600}h. " + "The model is still running in the background and will release " + "the GPU when it finishes.", + progress="Timed out.", + ) + return + + if _exc_box[0] is not None: + raise _exc_box[0] + + result = _result_box[0] + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + # Generate output files + jm.update_job(job_id, progress="Generating output files...") + + base_name = Path(audio_path).stem + output_files = [] + + # JSON (full data) + if "json" in output_formats: + json_path = out_path / f"{base_name}_transcription.json" + json_data = result_to_json(result) + json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding="utf-8") + output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"}) + + # LRC (line-level) + if "lrc" in output_formats: + lrc_path = out_path / f"{base_name}.lrc" + lrc_content = result_to_lrc(result) + lrc_path.write_text(lrc_content, encoding="utf-8") + output_files.append({"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"}) + + # ELRC (word-level) + if "elrc" in output_formats: + elrc_path = out_path / f"{base_name}.elrc" + elrc_content = result_to_elrc(result) + elrc_path.write_text(elrc_content, encoding="utf-8") + output_files.append({"name": elrc_path.name, "path": str(elrc_path), "type": "elrc"}) + + # Plain text + if "txt" in output_formats: + txt_path = out_path / f"{base_name}_lyrics.txt" + txt_content = result_to_txt(result) + txt_path.write_text(txt_content, encoding="utf-8") + output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"}) + + # SRT subtitles + if "srt" in output_formats: + srt_path = out_path / f"{base_name}.srt" + srt_content = result_to_srt(result) + srt_path.write_text(srt_content, encoding="utf-8") + output_files.append({"name": srt_path.name, "path": str(srt_path), "type": "srt"}) + + # VTT subtitles + if "vtt" in output_formats: + vtt_path = out_path / f"{base_name}.vtt" + vtt_content = result_to_vtt(result) + vtt_path.write_text(vtt_content, encoding="utf-8") + output_files.append({"name": vtt_path.name, "path": str(vtt_path), "type": "vtt"}) + + # Update job with results + result_summary = { + "model": result.model, + "language": result.language, + "duration": result.duration, + "segment_count": len(result.segments), + "word_count": sum(len(seg.words) for seg in result.segments), + } + + jm.update_job( + job_id, + status=jm.JobStatus.DONE, + progress="Lyric transcription complete.", + files=output_files, + result=result_summary + ) + + except Exception as exc: + jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.") + # Best-effort GPU flush so a failed job doesn't strand VRAM. + try: + import torch + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + + +def run_lyric_transcription_with_diarization( + job_id: str, + audio_path: str, + model: str = DEFAULT_WHISPER_MODEL, + language: Optional[str] = None, + num_speakers: Optional[int] = None, + min_speakers: Optional[int] = None, + max_speakers: Optional[int] = None, + output_formats: Optional[List[str]] = None, + output_dir: Optional[str] = None, +): + """ + Background task: transcribe vocals with speaker diarization. + + This uses WhisperX for diarization which requires additional setup: + - pip install whisperx + - HuggingFace token for pyannote models + + Args: + job_id: Job identifier + audio_path: Path to audio file + model: Whisper model name + language: Language code + num_speakers: Exact number of speakers (optional) + min_speakers: Minimum number of speakers + max_speakers: Maximum number of speakers + output_formats: List of output formats + output_dir: Custom output directory + """ + if output_formats is None: + output_formats = ["json", "txt"] + + if output_dir: + out_path = Path(output_dir) + else: + out_path = LRC_SYNC_DIR / job_id + out_path.mkdir(parents=True, exist_ok=True) + + try: + import whisperx + + jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading WhisperX model...") + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + # Load WhisperX model + device = "cuda" if jm.import_module("torch").cuda.is_available() else "cpu" + compute_type = "float16" if device == "cuda" else "int8" + + model_obj = whisperx.load_model(model, device, compute_type=compute_type) + + # Load audio + audio = whisperx.load_audio(audio_path) + + jm.update_job(job_id, progress="Transcribing with WhisperX...") + + # Transcribe + result = model_obj.transcribe(audio, batch_size=16) + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + # Align timestamps + jm.update_job(job_id, progress="Aligning word timestamps...") + model_a, metadata = whisperx.load_align_model(language_code=result["language"], device=device) + result = whisperx.align(result["segments"], model_a, metadata, audio, device) + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + # Diarization + jm.update_job(job_id, progress="Running speaker diarization...") + diarize_model = whisperx.DiarizationPipeline(device=device) + diarize_segments = diarize_model(audio, num_speakers=num_speakers, + min_speakers=min_speakers, + max_speakers=max_speakers) + + result = whisperx.assign_word_speakers(diarize_segments, result) + + # Generate outputs + jm.update_job(job_id, progress="Generating output files...") + + base_name = Path(audio_path).stem + output_files = [] + + if "json" in output_formats: + json_path = out_path / f"{base_name}_diarized.json" + json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"}) + + if "txt" in output_formats: + txt_path = out_path / f"{base_name}_diarized.txt" + lines = [f"[{seg.get('speaker', 'UNKNOWN')}] {seg['text']}" for seg in result["segments"]] + txt_path.write_text("\n".join(lines), encoding="utf-8") + output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"}) + + result_summary = { + "model": f"whisperx-{model}", + "language": result.get("language", "unknown"), + "segment_count": len(result["segments"]), + "diarization": True, + } + + jm.update_job( + job_id, + status=jm.JobStatus.DONE, + progress="Lyric transcription with diarization complete.", + files=output_files, + result=result_summary + ) + + except ImportError: + jm.update_job( + job_id, + status=jm.JobStatus.ERROR, + error="WhisperX not installed. Install with: pip install whisperx", + progress="Error." + ) + except Exception as exc: + jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.") + + +def is_whisper_available() -> bool: + """Check if Whisper is available.""" + try: + import whisper + return True + except ImportError: + return False + + +def is_whisperx_available() -> bool: + """Check if WhisperX is available.""" + try: + import whisperx + return True + except ImportError: + return False diff --git a/backend/services/lyric_transcriber_service.py b/backend/services/lyric_transcriber_service.py index 5d4b706..d8d5965 100755 --- a/backend/services/lyric_transcriber_service.py +++ b/backend/services/lyric_transcriber_service.py @@ -852,6 +852,119 @@ def _reconstruct_with_phonemes(phoneme_aligned: dict, original_words_per_seg: li return phoneme_aligned +_LSYNC_DIR = str(Path(__file__).parent.parent.parent / "lyrics-sync") +_lsync_recognizer: dict = {} # device → cached PhonemeRecognizer + + +def _lsync_syllabify(segments: list, audio_arr, device: str) -> bool: + """Add syllable timestamps to words using lsync CTC forced alignment. + + Uses facebook/wav2vec2-large-960h-lv60-self to obtain character-level CTC + emissions for the full audio, then pyphen (via lsync's LyricsProcessor) to + assign syllable boundaries with per-character timing accuracy. + + Modifies segments in-place setting word["syllables"] on each aligned word. + Words that already have syllables (e.g. from a prior phoneme pass) are + skipped. Returns True if at least one word was syllabified, False if the + entire pipeline failed (caller should fall back to _apply_pyphen_syllables). + """ + import re + import sys + + if _LSYNC_DIR not in sys.path: + sys.path.insert(0, _LSYNC_DIR) + try: + from lsync.phoneme_recognizer import PhonemeRecognizer + from lsync.alignment import Aligner + from lsync.lyrics_processor import LyricsProcessor, SEPARATOR + except ImportError: + return False + + # Collect words that still need syllables + all_words: list = [] + word_texts: list = [] + for seg in segments: + for w in seg.get("words", []): + if w.get("syllables"): + continue # already populated + raw = (w.get("word") or "").strip() + if not raw: + continue + clean = re.sub(r"[^\w']", "", raw) + if not clean: + continue + all_words.append(w) + word_texts.append(clean.upper()) + + if not all_words: + return False + + # Build lsync-format transcript: WORD1|WORD2|...|WORDN| + transcript = SEPARATOR.join(word_texts) + SEPARATOR + + # Load / cache model + try: + if device not in _lsync_recognizer: + _lsync_recognizer[device] = PhonemeRecognizer(lang="en-US", device=device) + recognizer = _lsync_recognizer[device] + lp = LyricsProcessor(lang="en-US") + except Exception: + return False + + # CTC emissions from full audio (lsync splits into 15-s frames internally) + try: + emission, _pred, _tx, frame_duration = recognizer.recognize(audio_arr) + except Exception: + return False + + # Forced CTC alignment + try: + tokens = recognizer.processor.tokenizer(transcript).input_ids + path = Aligner.align(emission, tokens) + except Exception: + return False + + # Syllable-level Word objects + try: + syl_words = lp.get_syllables_from_path(transcript, path, frame_duration) + except Exception: + return False + + # Group syllable tokens into per-word groups (is_word_end marks word boundary) + groups: list = [] + current: list = [] + for s in syl_words: + current.append(s) + if s.is_word_end: + groups.append(current) + current = [] + if current: + groups.append(current) + + if len(groups) != len(all_words): + return False # word-count mismatch — can't safely assign + + assigned = 0 + for word_info, syls in zip(all_words, groups): + orig = (word_info.get("word") or "").strip() + clean = re.sub(r"[^\w']", "", orig) + # Reconstruct display text from original casing using lsync label lengths + pos = 0 + display_parts: list = [] + for syl in syls: + n = len(syl.label) + part = clean[pos:pos + n] if pos + n <= len(clean) else syl.label.lower() + display_parts.append(part) + pos += n + word_info["syllables"] = [ + {"text": txt, "start": s.start, "end": s.end} + for txt, s in zip(display_parts, syls) + ] + assigned += 1 + + return assigned > 0 + + def _apply_pyphen_syllables(segments: list) -> None: """Populate word["syllables"] using pyphen proportional time splitting. @@ -1054,6 +1167,7 @@ def run_lyric_transcription( diarize: bool = False, power_preset: str = "standard", phoneme_align: bool = False, + lsync_align: bool = False, two_pass: bool = False, existing_lyrics: Optional[str] = None, ): @@ -1085,8 +1199,8 @@ def run_lyric_transcription( else: out_path = LRC_SYNC_DIR / job_id out_path.mkdir(parents=True, exist_ok=True) - log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s two_pass=%s", - job_id, model, engine, phoneme_align, two_pass) + log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s lsync=%s two_pass=%s", + job_id, model, engine, phoneme_align, lsync_align, two_pass) try: if engine == "whisperx": @@ -1249,11 +1363,17 @@ def run_lyric_transcription( # timestamps (e.g. last word of a phrase held until the next VAD chunk). _clamp_word_end_times(wx_result["segments"]) - # If syllable timing was requested, fill any words that don't yet have - # syllable data using pyphen proportional splitting (same approach used - # by the lsync library). This covers both the IPA-phoneme fallback case - # and the standard word-level path when phoneme_align=True. - if phoneme_align: + # Syllable timing ————————————————————————————————————————————— + # lsync path: CTC forced alignment via wav2vec2 + pyphen syllable + # boundaries (more accurate than proportional distribution). + # Phoneme path: IPA alignment already populated word["syllables"]; + # pyphen fills any words that couldn't be aligned. + if lsync_align: + jm.update_job(job_id, progress="Running lsync syllable alignment (wav2vec2 CTC)...") + _lsync_syllabify(wx_result["segments"], audio_arr, device) + # Fill any words lsync couldn't align (proportional fallback) + _apply_pyphen_syllables(wx_result["segments"]) + elif phoneme_align: _apply_pyphen_syllables(wx_result["segments"]) # Free alignment model before diarization (pyannote is also VRAM-heavy). diff --git a/backend/static/index.html b/backend/static/index.html index 6c839cf..1f4a23e 100644 --- a/backend/static/index.html +++ b/backend/static/index.html @@ -19,7 +19,7 @@ - + diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index c07b789..ba385bf 100755 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -389,6 +389,7 @@ export const lyricTranscribeProject = ( diarize?: boolean lrcPowerPreset?: string phonemeAlign?: boolean + lsyncAlign?: boolean twoPass?: boolean } ) => { @@ -415,6 +416,7 @@ export const lyricTranscribeProject = ( diarize: opts?.diarize ?? false, lrc_power_preset: opts?.lrcPowerPreset ?? 'standard', phoneme_align: phonemeAlign, + lsync_align: opts?.lsyncAlign ?? false, two_pass: opts?.twoPass ?? false, }), }) diff --git a/frontend/src/pages/ProjectDetail.tsx b/frontend/src/pages/ProjectDetail.tsx index c86e912..8eaf390 100755 --- a/frontend/src/pages/ProjectDetail.tsx +++ b/frontend/src/pages/ProjectDetail.tsx @@ -1689,6 +1689,7 @@ function StemLyricsRow({ // Transcribe state — initialise from Power settings stored in localStorage const [diarize, setDiarize] = useState(false) const [phonemeAlign, setPhonemeAlign] = useState(false) + const [lsyncAlign, setLsyncAlign] = useState(false) const [whisperModel, setWhisperModel] = useState( () => localStorage.getItem('stemengine-whisper-model') ?? 'large-v2' ) @@ -1822,6 +1823,7 @@ function StemLyricsRow({ diarize, lrcPowerPreset: powerPreset, phonemeAlign, + lsyncAlign, twoPass, }) sessionJobIds.current.add(res.job_id) @@ -1957,6 +1959,15 @@ function StemLyricsRow({ /> Phonemes +