Files
stemengine_old/backend/services/lyric_transcriber_service.py
T
2026-03-03 14:52:19 +09:00

1076 lines
39 KiB
Python
Executable File

"""
Lyric transcription service using OpenAI Whisper.
Based on Scriberr's transcription approach - provides word-level timestamps
and speaker diarization support for lyrics.
This service transcribes vocals to text with precise timing information,
suitable for generating LRC and ELRC files.
"""
import contextlib
import threading
import io
import json
import os
import tempfile
import warnings
from pathlib import Path
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
from dataclasses import dataclass
# Whisper's timing.py tries to JIT-compile Triton CUDA kernels for DTW alignment.
# This requires the full CUDA toolkit (nvcc), which is not present in the runtime
# image. The fallback (NumPy median kernel) is correct and only marginally slower,
# so suppress the noisy UserWarning.
warnings.filterwarnings(
"ignore",
message="Failed to launch Triton kernels",
category=UserWarning,
)
# PyTorch 2.6 changed torch.load's default from weights_only=False to True.
# WhisperX / wav2vec2 / pyannote checkpoints contain arbitrary omegaconf and
# other custom types that are not in the default allowlist, and chasing each
# class individually is fragile. Restore the pre-2.6 default (weights_only
# not set by caller → False) by monkey-patching torch.load at startup.
# All checkpoint sources used here (HuggingFace, faster-whisper, pyannote)
# are trusted, so the security trade-off is acceptable.
try:
import torch as _torch
_orig_torch_load = _torch.load
def _patched_torch_load(*args, **kwargs):
# Force weights_only=False regardless of what the caller passed.
# transformers explicitly passes weights_only=True (not just as a
# default), so setdefault is insufficient — we must overwrite it.
# All checkpoint sources used here (HuggingFace, faster-whisper,
# pyannote) are trusted, so the security trade-off is acceptable.
kwargs['weights_only'] = False
return _orig_torch_load(*args, **kwargs)
_torch.load = _patched_torch_load
except Exception:
pass
from .. import job_manager as jm
from ..config import JOBS_DIR, LRC_SYNC_DIR
@dataclass
class WordTimestamp:
"""Word-level timestamp information."""
word: str
start: float
end: float
confidence: Optional[float] = None
@dataclass
class Segment:
"""Segment (line/phrase) level timestamp information."""
text: str
start: float
end: float
words: List[WordTimestamp]
speaker: Optional[str] = None
@dataclass
class LyricTranscriptionResult:
"""Complete transcription result."""
text: str
segments: List[Segment]
language: str
duration: float
model: str
# Available Whisper models with their characteristics
WHISPER_MODELS = {
"tiny": {
"id": "tiny",
"label": "Tiny (fastest, least accurate)",
"size": "39 MB",
"relative_speed": "~32x",
"english_only": False,
},
"base": {
"id": "base",
"label": "Base (fast, good accuracy)",
"size": "74 MB",
"relative_speed": "~16x",
"english_only": False,
},
"small": {
"id": "small",
"label": "Small (balanced)",
"size": "244 MB",
"relative_speed": "~6x",
"english_only": False,
},
"medium": {
"id": "medium",
"label": "Medium (slower, better accuracy)",
"size": "769 MB",
"relative_speed": "~2x",
"english_only": False,
},
"large": {
"id": "large",
"label": "Large (slowest, best accuracy)",
"size": "1550 MB",
"relative_speed": "1x",
"english_only": False,
},
"large-v2": {
"id": "large-v2",
"label": "Large v2 (recommended)",
"size": "1550 MB",
"relative_speed": "1x",
"english_only": False,
},
"large-v3": {
"id": "large-v3",
"label": "Large v3 (latest)",
"size": "1550 MB",
"relative_speed": "1x",
"english_only": False,
},
}
DEFAULT_WHISPER_MODEL = "large-v2"
_whisper_cache: Dict[str, Any] = {}
# Set to False when a CUDA OOM or "unknown error" corrupts the GPU context.
# Once False, all subsequent Whisper loads are forced to CPU so the server
# keeps working without requiring a restart.
_cuda_ok: bool = True
def _mark_cuda_failed() -> None:
"""Record that the CUDA context is bad and purge any GPU-resident models."""
global _cuda_ok
_cuda_ok = False
_whisper_cache.clear()
try:
import torch
torch.cuda.empty_cache()
except Exception:
pass
def reset_cuda() -> dict:
"""
Re-enable GPU acceleration after a CUDA error recovery.
The transcriber falls back to CPU when a CUDA OOM or 'unknown error'
occurs. Call this once the GPU is confirmed free to restore GPU usage
without restarting the container.
"""
global _cuda_ok
_cuda_ok = True
_whisper_cache.clear()
cuda_available = False
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
cuda_available = True
except Exception:
pass
return {"ok": True, "cuda_available": cuda_available, "cuda_ok": _cuda_ok}
def get_whisper_model(model_name: str = DEFAULT_WHISPER_MODEL, device: Optional[str] = None):
"""
Load and in-process-cache a Whisper model.
Reuses an already-loaded model of the same name to avoid reloading ~1.5 GB
weights on every transcription call. If a *different* model is requested,
the cached one is evicted and the CUDA allocator cache is flushed first so
that the new model can fit alongside any other resident GPU allocations.
When `device` is explicitly passed (e.g. "cpu") it overrides the auto-detect
logic. This is used by the CUDA-error recovery path in `transcribe_lyrics`.
"""
import whisper
cache_key = f"{model_name}:{device or 'auto'}"
if cache_key in _whisper_cache:
return _whisper_cache[cache_key]
# Evict any previously cached model and release its GPU memory before
# loading the new one.
if _whisper_cache:
_whisper_cache.clear()
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
load_kwargs: dict = {}
if device is not None:
load_kwargs["device"] = device
model = whisper.load_model(model_name, **load_kwargs)
_whisper_cache[cache_key] = model
return model
class _LineCapture(io.RawIOBase):
"""
File-like object that buffers writes and calls `callback` once per complete line.
Used to intercept Whisper's verbose stdout output for live progress reporting.
"""
def __init__(self, callback: Callable[[str], None]) -> None:
self._cb = callback
self._buf = ''
def write(self, text) -> int: # type: ignore[override]
if isinstance(text, (bytes, bytearray)):
text = text.decode('utf-8', errors='replace')
self._buf += text
while '\n' in self._buf:
line, self._buf = self._buf.split('\n', 1)
cleaned = line.strip()
if cleaned:
self._cb(cleaned)
return len(text)
def flush(self) -> None:
pass
# Some libraries call fileno(); raise so they fall back gracefully.
def fileno(self): # type: ignore[override]
raise io.UnsupportedOperation('fileno')
def transcribe_lyrics(
audio_path: str,
model_name: str = DEFAULT_WHISPER_MODEL,
language: Optional[str] = None,
task: Literal["transcribe", "translate"] = "transcribe",
include_word_timestamps: bool = True,
initial_prompt: Optional[str] = None,
temperature: float = 0.0,
condition_on_previous_text: bool = False,
compression_ratio_threshold: float = 1.35,
beam_size: Optional[int] = None,
line_callback: Optional[Callable[[str], None]] = None,
) -> LyricTranscriptionResult:
"""
Transcribe vocals to lyrics with word-level timestamps.
Anti-hallucination defaults (differ from stock Whisper):
- condition_on_previous_text=False — prevents the model from feeding
a repeated phrase back as context, breaking the repetition loop.
- temperature fallback tuple — when temperature=0.0, Whisper is
automatically given (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) so it can retry
a stuck segment with increasing randomness.
- compression_ratio_threshold=1.35 — flags repetitive segments sooner
(stock default is 2.4) and triggers the temperature fallback.
- _deduplicate_segments post-pass — removes any repeated segments or
internally-looped phrases that still slip through.
Args:
audio_path: Path to audio file (should be vocals stem for best results)
model_name: Whisper model to use
language: ISO-639-1 language code (auto-detected if None)
task: 'transcribe' for original language, 'translate' for English
include_word_timestamps: Whether to include word-level timestamps
initial_prompt: Optional text to guide the model's style
temperature: Base sampling temperature; 0.0 enables the fallback tuple
condition_on_previous_text: Feed previous segment as context (False prevents loops)
compression_ratio_threshold: Retry segment above this gzip ratio (lower = stricter)
Returns:
LyricTranscriptionResult with text, segments, and timestamps
"""
import whisper
# Free any reserved-but-unallocated GPU memory from prior jobs (e.g. Demucs,
# BS-RoFormer) before the Whisper model is loaded / used.
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
# If a previous job corrupted the CUDA context (OOM → "unknown error"),
# force CPU so this job still completes rather than failing immediately.
forced_device: Optional[str] = None if _cuda_ok else "cpu"
model = get_whisper_model(model_name, device=forced_device)
# When temperature is 0 (greedy), pass a fallback sequence so Whisper can
# recover from stuck segments by retrying with increasing randomness.
effective_temperature: Any = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) if temperature == 0.0 else temperature
decode_options: dict = {
"task": task,
"temperature": effective_temperature,
"condition_on_previous_text": condition_on_previous_text,
"compression_ratio_threshold": compression_ratio_threshold,
}
if language:
decode_options["language"] = language
if initial_prompt:
decode_options["initial_prompt"] = initial_prompt
if beam_size is not None:
decode_options["beam_size"] = beam_size
def _run_transcribe(mdl: Any) -> Any:
"""Run model.transcribe(), capturing verbose output when a callback is set."""
if line_callback is not None:
_cap = _LineCapture(line_callback)
with contextlib.redirect_stdout(_cap), contextlib.redirect_stderr(_cap):
return mdl.transcribe(
audio_path,
word_timestamps=include_word_timestamps,
verbose=True,
**decode_options,
)
return mdl.transcribe(
audio_path,
word_timestamps=include_word_timestamps,
**decode_options,
)
# Run transcription with word timestamps.
# On any CUDA error (OOM already happened, or the context was already
# corrupted), mark the GPU as failed, reload the model on CPU, and retry.
# This keeps the server usable without a restart.
try:
result = _run_transcribe(model)
except RuntimeError as exc:
err_str = str(exc)
if "CUDA" in err_str or "cuda" in err_str:
_mark_cuda_failed()
if line_callback:
line_callback("⚠️ CUDA error — retrying on CPU (server restart not required)")
cpu_model = get_whisper_model(model_name, device="cpu")
result = _run_transcribe(cpu_model)
else:
raise
# Parse results into structured format
raw_segments: List[Segment] = []
for seg in result["segments"]:
words = []
if include_word_timestamps and "words" in seg:
for word_info in seg["words"]:
words.append(WordTimestamp(
word=word_info["word"].strip(),
start=word_info["start"],
end=word_info["end"],
confidence=word_info.get("probability")
))
raw_segments.append(Segment(
text=seg["text"].strip(),
start=seg["start"],
end=seg["end"],
words=words
))
# Calculate total duration
duration = max(seg.end for seg in raw_segments) if raw_segments else 0.0
return LyricTranscriptionResult(
text=result["text"].strip(),
segments=raw_segments,
language=result.get("language", "unknown"),
duration=duration,
model=f"whisper-{model_name}"
)
def result_to_lrc(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to LRC format (line-level timestamps).
Speaker info (when present) is encoded as |sp:SPEAKER_00| immediately
after the LRC timestamp so the lyric player can colour-code lines per
speaker while remaining compatible with standard LRC parsers (which
simply ignore the tag as leading text).
"""
lines = []
for seg in result.segments:
minutes = int(seg.start // 60)
seconds = int(seg.start % 60)
centis = int((seg.start % 1) * 100)
speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else ""
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}")
return "\n".join(lines)
def result_to_elrc(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to ELRC format (word-level timestamps).
Speaker info is encoded the same way as in result_to_lrc — |sp:SPEAKER_00|
is prepended to the word-timestamp content of each line.
"""
lines = []
for seg in result.segments:
speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else ""
if not seg.words:
# Fallback to segment-level if no word timestamps
minutes = int(seg.start // 60)
seconds = int(seg.start % 60)
centis = int((seg.start % 1) * 100)
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}")
continue
# Line timestamp is start of first word
line_start = seg.words[0].start
line_min = int(line_start // 60)
line_sec = int(line_start % 60)
line_centi = int((line_start % 1) * 100)
# Build word-level timestamps.
# Strip each word's surrounding whitespace before embedding — Whisper/WhisperX
# sometimes include a leading space (e.g. " world") which would produce
# double-spaces when tokens are joined.
word_parts = []
for word in seg.words:
w_min = int(word.start // 60)
w_sec = int(word.start % 60)
w_centi = int((word.start % 1) * 100)
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}")
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{' '.join(word_parts)}")
return "\n".join(lines)
def result_to_json(result: LyricTranscriptionResult) -> Dict[str, Any]:
"""Convert transcription result to JSON-serializable dict."""
return {
"text": result.text,
"language": result.language,
"duration": result.duration,
"model": result.model,
"segments": [
{
"text": seg.text,
"start": seg.start,
"end": seg.end,
"speaker": seg.speaker,
"words": [
{
"word": w.word,
"start": w.start,
"end": w.end,
"confidence": w.confidence
}
for w in seg.words
]
}
for seg in result.segments
]
}
def result_to_txt(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to plain text.
One Whisper segment per line so the output can be fed directly into the
ctc-forced-aligner (lrc_service) which splits on newlines to identify
individual lyric lines. Falls back to the joined text if no segments.
"""
if result.segments:
return '\n'.join(seg.text.strip() for seg in result.segments if seg.text.strip())
return result.text
def result_to_srt(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to SRT subtitle format."""
lines = []
for i, seg in enumerate(result.segments, 1):
# Format: HH:MM:SS,mmm
def fmt_srt_time(t: float) -> str:
hours = int(t // 3600)
mins = int((t % 3600) // 60)
secs = int(t % 60)
millis = int((t % 1) * 1000)
return f"{hours:02d}:{mins:02d}:{secs:02d},{millis:03d}"
lines.append(str(i))
lines.append(f"{fmt_srt_time(seg.start)} --> {fmt_srt_time(seg.end)}")
lines.append(seg.text)
lines.append("") # Empty line between entries
return "\n".join(lines)
def result_to_vtt(result: LyricTranscriptionResult) -> str:
"""Convert transcription result to WebVTT subtitle format."""
lines = ["WEBVTT", ""]
for seg in result.segments:
def fmt_vtt_time(t: float) -> str:
hours = int(t // 3600)
mins = int((t % 3600) // 60)
secs = int(t % 60)
millis = int((t % 1) * 1000)
return f"{hours:02d}:{mins:02d}:{secs:02d}.{millis:03d}"
lines.append(f"{fmt_vtt_time(seg.start)} --> {fmt_vtt_time(seg.end)}")
lines.append(seg.text)
lines.append("")
return "\n".join(lines)
_PUNCT_BREAK = frozenset('.!?,;')
def _split_long_segments(segments: list, max_words: int = 10) -> list:
"""
Split over-long WhisperX segments into shorter lines suitable for LRC display.
Strategy (applied per segment):
1. Walk the word list looking for *punctuation boundaries* — words whose
text ends with a character in _PUNCT_BREAK. When such a boundary is
found AND the current chunk already has at least 3 words, emit a new
sub-segment there.
2. If no punctuation boundary is found within `max_words` words, force-
split at `max_words` regardless.
Each sub-segment inherits the speaker label and gets timestamps from the
first / last word in the slice. Segments with fewer than 2 words are left
unchanged.
"""
if max_words <= 0:
return segments
result = []
for seg in segments:
words = seg.get("words") or []
if len(words) <= max_words:
result.append(seg)
continue
# Build sub-segment slices
chunks: list[list] = []
chunk: list = []
for w in words:
chunk.append(w)
text = w.get("word", "").rstrip()
at_punct = bool(text) and text[-1] in _PUNCT_BREAK
at_limit = len(chunk) >= max_words
if (at_punct and len(chunk) >= 3) or at_limit:
chunks.append(chunk)
chunk = []
if chunk:
chunks.append(chunk)
for ch in chunks:
first, last = ch[0], ch[-1]
sub_text = "".join(w.get("word", "") for w in ch).strip()
result.append({
"start": first.get("start") if first.get("start") is not None else seg["start"],
"end": last.get("end") if last.get("end") is not None else seg["end"],
"text": sub_text,
"words": ch,
"speaker": seg.get("speaker"),
})
return result
_VOCAL_KEYWORDS = frozenset(('vocal', 'vocals', 'voice', 'lead', 'sing'))
_AUDIO_EXTS = frozenset(('.mp3', '.wav', '.flac', '.ogg', '.m4a', '.aac'))
def _ensure_vocals(audio_path: str, job_id: str) -> str:
"""
Return the path to the best available vocals audio for transcription.
Priority:
1. audio_path already looks like a vocals stem (filename keyword check).
2. A vocals file exists in the same Stems/ directory — use it.
3. Run htdemucs --two-stems vocals to extract vocals from audio_path.
Falls back to audio_path unchanged on any failure.
"""
import subprocess, sys
p = Path(audio_path)
stem_lower = p.stem.lower()
# 1. Already a vocals stem?
if any(kw in stem_lower for kw in _VOCAL_KEYWORDS):
return audio_path
# 2. Look for a sibling vocals file in the same directory.
for f in sorted(p.parent.iterdir()):
if f == p or not f.is_file():
continue
if f.suffix.lower() not in _AUDIO_EXTS:
continue
if any(kw in f.stem.lower() for kw in _VOCAL_KEYWORDS):
jm.update_job(job_id, progress=f"Using existing vocal stem: {f.name}")
return str(f)
# 3. No vocal stem found — extract with htdemucs --two-stems vocals.
jm.update_job(job_id, progress="No vocal stem found — extracting vocals with Demucs (htdemucs)...")
extract_dir = LRC_SYNC_DIR / job_id / "vocals_extract"
extract_dir.mkdir(parents=True, exist_ok=True)
cmd = [
sys.executable, '-m', 'demucs',
'--name', 'htdemucs',
'--two-stems', 'vocals',
'--out', str(extract_dir),
audio_path,
]
try:
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
if proc.returncode == 0:
# Output layout: <extract_dir>/htdemucs/<input_stem>/vocals.wav
for candidate in sorted(extract_dir.rglob('vocals.*')):
if candidate.is_file() and candidate.suffix.lower() in _AUDIO_EXTS:
jm.update_job(job_id, progress=f"Vocals extracted: {candidate.name}")
return str(candidate)
else:
jm.update_job(job_id, progress=f"Demucs exited {proc.returncode} — using original audio.")
except Exception as _e:
jm.update_job(job_id, progress=f"Demucs extraction failed ({_e}) — using original audio.")
return audio_path
def run_lyric_transcription(
job_id: str,
audio_path: str,
model: str = DEFAULT_WHISPER_MODEL,
language: Optional[str] = None,
task: str = "transcribe",
output_formats: Optional[List[str]] = None,
initial_prompt: Optional[str] = None,
temperature: float = 0.0,
condition_on_previous_text: bool = False,
word_timestamps: bool = True,
beam_size: Optional[int] = None,
output_dir: Optional[str] = None,
engine: str = "whisperx",
diarize: bool = False,
):
"""
Background task: transcribe vocals/lyrics from audio.
Args:
job_id: Job identifier
audio_path: Path to audio file
model: Whisper model name
language: Language code (auto-detect if None)
task: 'transcribe' or 'translate'
output_formats: List of output formats ('lrc', 'elrc', 'json', 'txt', 'srt', 'vtt')
initial_prompt: Optional prompt text
temperature: Sampling temperature
output_dir: Custom output directory
engine: 'whisperx' (faster-whisper + phoneme alignment)
diarize: Run speaker diarization after alignment (WhisperX only)
"""
if output_formats is None:
output_formats = ["elrc", "json", "txt"]
if output_dir:
out_path = Path(output_dir)
else:
out_path = LRC_SYNC_DIR / job_id
out_path.mkdir(parents=True, exist_ok=True)
try:
if engine == "whisperx":
# ── WhisperX path: faster-whisper backend + phoneme-level alignment ──
try:
import torch
import whisperx
except ImportError as _imp_err:
jm.update_job(
job_id,
status=jm.JobStatus.ERROR,
error=f"WhisperX is not installed ({_imp_err}). "
"Install with: pip install whisperx",
progress="Error.",
)
return
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading WhisperX model ({model})...")
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
device = "cuda" if torch.cuda.is_available() else "cpu"
compute_type = "float16" if device == "cuda" else "int8"
# Auto-ensure vocals stem for best transcription quality.
audio_path = _ensure_vocals(audio_path, job_id)
wx_model = whisperx.load_model(model, device, compute_type=compute_type, language=language or None)
audio_arr = whisperx.load_audio(audio_path)
duration = float(len(audio_arr)) / 16000.0
jm.update_job(job_id, progress=f"Transcribing with WhisperX ({model})...")
wx_result = wx_model.transcribe(audio_arr, batch_size=16)
detected_lang = wx_result.get("language") or language or "unknown"
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
jm.update_job(job_id, progress="Aligning word timestamps (WhisperX)...")
model_a, metadata = whisperx.load_align_model(language_code=detected_lang, device=device)
wx_result = whisperx.align(
wx_result["segments"], model_a, metadata, audio_arr, device,
return_char_alignments=False,
)
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
if diarize:
jm.update_job(job_id, progress="Running speaker diarization...")
try:
import os
hf_token = os.environ.get("HF_TOKEN") or None
if not hf_token:
raise RuntimeError(
"HF_TOKEN environment variable is not set. "
"Add HF_TOKEN=<your_token> to your .env file and restart the container."
)
# Log in globally so pyannote.audio picks up the token regardless of
# which param name the installed version expects (use_auth_token vs token).
try:
from huggingface_hub import login as _hf_login
_hf_login(token=hf_token, add_to_git_credential=False)
except Exception:
pass
diarize_model = whisperx.DiarizationPipeline(use_auth_token=hf_token, device=device)
diarize_segments = diarize_model(audio_arr)
wx_result = whisperx.assign_word_speakers(diarize_segments, wx_result)
except Exception as _diar_err:
jm.update_job(job_id, progress=f"Diarization failed ({_diar_err}), continuing without speaker labels...")
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
# Split over-long segments into shorter lyric lines before converting.
# Punctuation boundaries are preferred; falls back to a hard word-count cut.
raw_segments = _split_long_segments(wx_result["segments"], max_words=10)
# Convert WhisperX output → LyricTranscriptionResult
segments: List[Segment] = []
for seg in raw_segments:
words = []
for w in seg.get("words", []):
_ws = w.get("start")
_we = w.get("end")
words.append(WordTimestamp(
word=w.get("word", ""),
start=float(_ws if _ws is not None else seg["start"]),
end=float(_we if _we is not None else seg["end"]),
confidence=w.get("score"),
))
segments.append(Segment(
text=seg["text"].strip(),
start=float(seg["start"]),
end=float(seg["end"]),
words=words,
speaker=seg.get("speaker"),
))
result = LyricTranscriptionResult(
text=" ".join(seg.text for seg in segments),
segments=segments,
language=detected_lang,
duration=duration,
model=f"whisperx-{model}",
)
else:
# ── Standard Whisper path ─────────────────────────────────────────────
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading Whisper model ({model})...")
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
jm.update_job(job_id, progress=f"Transcribing lyrics with Whisper ({model})...")
# Accumulate Whisper's verbose segment output for live progress display.
# Noisy lines (tqdm bars, empty lines) are filtered out.
_log: List[str] = []
_NOISE_TOKENS = ('%|', 'it/s', 'B/s', '100%')
def _on_line(line: str) -> None:
if any(tok in line for tok in _NOISE_TOKENS):
return
_log.append(line)
if len(_log) > 30:
_log.pop(0)
jm.update_job(job_id, progress='\n'.join(_log))
_TRANSCRIBE_TIMEOUT = 7200 # 2-hour hard cap
_result_box: list = [None]
_exc_box: list = [None]
def _do_transcribe():
try:
_result_box[0] = transcribe_lyrics(
audio_path=audio_path,
model_name=model,
language=language,
task=task, # type: ignore
include_word_timestamps=word_timestamps,
initial_prompt=initial_prompt,
temperature=temperature,
condition_on_previous_text=condition_on_previous_text,
beam_size=beam_size,
line_callback=_on_line,
)
except Exception as _e:
_exc_box[0] = _e
_t = threading.Thread(target=_do_transcribe, daemon=True)
_t.start()
_t.join(timeout=_TRANSCRIBE_TIMEOUT)
if _t.is_alive():
jm.update_job(
job_id,
status=jm.JobStatus.ERROR,
error=f"Transcription timed out after {_TRANSCRIBE_TIMEOUT // 3600}h. "
"The model is still running in the background and will release "
"the GPU when it finishes.",
progress="Timed out.",
)
return
if _exc_box[0] is not None:
raise _exc_box[0]
result = _result_box[0]
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
# Generate output files
jm.update_job(job_id, progress="Generating output files...")
base_name = Path(audio_path).stem
output_files = []
# JSON (full data)
if "json" in output_formats:
json_path = out_path / f"{base_name}_transcription.json"
json_data = result_to_json(result)
json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding="utf-8")
output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"})
# LRC (line-level)
if "lrc" in output_formats:
lrc_path = out_path / f"{base_name}.lrc"
lrc_content = result_to_lrc(result)
lrc_path.write_text(lrc_content, encoding="utf-8")
output_files.append({"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"})
# ELRC (word-level)
if "elrc" in output_formats:
elrc_path = out_path / f"{base_name}.elrc"
elrc_content = result_to_elrc(result)
elrc_path.write_text(elrc_content, encoding="utf-8")
output_files.append({"name": elrc_path.name, "path": str(elrc_path), "type": "elrc"})
# Plain text
if "txt" in output_formats:
txt_path = out_path / f"{base_name}_lyrics.txt"
txt_content = result_to_txt(result)
txt_path.write_text(txt_content, encoding="utf-8")
output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"})
# SRT subtitles
if "srt" in output_formats:
srt_path = out_path / f"{base_name}.srt"
srt_content = result_to_srt(result)
srt_path.write_text(srt_content, encoding="utf-8")
output_files.append({"name": srt_path.name, "path": str(srt_path), "type": "srt"})
# VTT subtitles
if "vtt" in output_formats:
vtt_path = out_path / f"{base_name}.vtt"
vtt_content = result_to_vtt(result)
vtt_path.write_text(vtt_content, encoding="utf-8")
output_files.append({"name": vtt_path.name, "path": str(vtt_path), "type": "vtt"})
# Update job with results
result_summary = {
"model": result.model,
"language": result.language,
"duration": result.duration,
"segment_count": len(result.segments),
"word_count": sum(len(seg.words) for seg in result.segments),
}
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
progress="Lyric transcription complete.",
files=output_files,
result=result_summary
)
except Exception as exc:
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
def run_lyric_transcription_with_diarization(
job_id: str,
audio_path: str,
model: str = DEFAULT_WHISPER_MODEL,
language: Optional[str] = None,
num_speakers: Optional[int] = None,
min_speakers: Optional[int] = None,
max_speakers: Optional[int] = None,
output_formats: Optional[List[str]] = None,
output_dir: Optional[str] = None,
):
"""
Background task: transcribe vocals with speaker diarization.
This uses WhisperX for diarization which requires additional setup:
- pip install whisperx
- HuggingFace token for pyannote models
Args:
job_id: Job identifier
audio_path: Path to audio file
model: Whisper model name
language: Language code
num_speakers: Exact number of speakers (optional)
min_speakers: Minimum number of speakers
max_speakers: Maximum number of speakers
output_formats: List of output formats
output_dir: Custom output directory
"""
if output_formats is None:
output_formats = ["json", "txt"]
if output_dir:
out_path = Path(output_dir)
else:
out_path = LRC_SYNC_DIR / job_id
out_path.mkdir(parents=True, exist_ok=True)
try:
import whisperx
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading WhisperX model...")
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
# Load WhisperX model
device = "cuda" if jm.import_module("torch").cuda.is_available() else "cpu"
compute_type = "float16" if device == "cuda" else "int8"
model_obj = whisperx.load_model(model, device, compute_type=compute_type)
# Load audio
audio = whisperx.load_audio(audio_path)
jm.update_job(job_id, progress="Transcribing with WhisperX...")
# Transcribe
result = model_obj.transcribe(audio, batch_size=16)
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
# Align timestamps
jm.update_job(job_id, progress="Aligning word timestamps...")
model_a, metadata = whisperx.load_align_model(language_code=result["language"], device=device)
result = whisperx.align(result["segments"], model_a, metadata, audio, device)
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
# Diarization
jm.update_job(job_id, progress="Running speaker diarization...")
diarize_model = whisperx.DiarizationPipeline(device=device)
diarize_segments = diarize_model(audio, num_speakers=num_speakers,
min_speakers=min_speakers,
max_speakers=max_speakers)
result = whisperx.assign_word_speakers(diarize_segments, result)
# Generate outputs
jm.update_job(job_id, progress="Generating output files...")
base_name = Path(audio_path).stem
output_files = []
if "json" in output_formats:
json_path = out_path / f"{base_name}_diarized.json"
json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"})
if "txt" in output_formats:
txt_path = out_path / f"{base_name}_diarized.txt"
lines = [f"[{seg.get('speaker', 'UNKNOWN')}] {seg['text']}" for seg in result["segments"]]
txt_path.write_text("\n".join(lines), encoding="utf-8")
output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"})
result_summary = {
"model": f"whisperx-{model}",
"language": result.get("language", "unknown"),
"segment_count": len(result["segments"]),
"diarization": True,
}
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
progress="Lyric transcription with diarization complete.",
files=output_files,
result=result_summary
)
except ImportError:
jm.update_job(
job_id,
status=jm.JobStatus.ERROR,
error="WhisperX not installed. Install with: pip install whisperx",
progress="Error."
)
except Exception as exc:
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
def is_whisper_available() -> bool:
"""Check if Whisper is available."""
try:
import whisper
return True
except ImportError:
return False
def is_whisperx_available() -> bool:
"""Check if WhisperX is available."""
try:
import whisperx
return True
except ImportError:
return False