1076 lines
39 KiB
Python
Executable File
1076 lines
39 KiB
Python
Executable File
"""
|
|
Lyric transcription service using OpenAI Whisper.
|
|
Based on Scriberr's transcription approach - provides word-level timestamps
|
|
and speaker diarization support for lyrics.
|
|
|
|
This service transcribes vocals to text with precise timing information,
|
|
suitable for generating LRC and ELRC files.
|
|
"""
|
|
import contextlib
|
|
import threading
|
|
import io
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import warnings
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
|
|
from dataclasses import dataclass
|
|
|
|
# Whisper's timing.py tries to JIT-compile Triton CUDA kernels for DTW alignment.
|
|
# This requires the full CUDA toolkit (nvcc), which is not present in the runtime
|
|
# image. The fallback (NumPy median kernel) is correct and only marginally slower,
|
|
# so suppress the noisy UserWarning.
|
|
warnings.filterwarnings(
|
|
"ignore",
|
|
message="Failed to launch Triton kernels",
|
|
category=UserWarning,
|
|
)
|
|
|
|
# PyTorch 2.6 changed torch.load's default from weights_only=False to True.
|
|
# WhisperX / wav2vec2 / pyannote checkpoints contain arbitrary omegaconf and
|
|
# other custom types that are not in the default allowlist, and chasing each
|
|
# class individually is fragile. Restore the pre-2.6 default (weights_only
|
|
# not set by caller → False) by monkey-patching torch.load at startup.
|
|
# All checkpoint sources used here (HuggingFace, faster-whisper, pyannote)
|
|
# are trusted, so the security trade-off is acceptable.
|
|
try:
|
|
import torch as _torch
|
|
_orig_torch_load = _torch.load
|
|
def _patched_torch_load(*args, **kwargs):
|
|
# Force weights_only=False regardless of what the caller passed.
|
|
# transformers explicitly passes weights_only=True (not just as a
|
|
# default), so setdefault is insufficient — we must overwrite it.
|
|
# All checkpoint sources used here (HuggingFace, faster-whisper,
|
|
# pyannote) are trusted, so the security trade-off is acceptable.
|
|
kwargs['weights_only'] = False
|
|
return _orig_torch_load(*args, **kwargs)
|
|
_torch.load = _patched_torch_load
|
|
except Exception:
|
|
pass
|
|
|
|
from .. import job_manager as jm
|
|
from ..config import JOBS_DIR, LRC_SYNC_DIR
|
|
|
|
|
|
@dataclass
|
|
class WordTimestamp:
|
|
"""Word-level timestamp information."""
|
|
word: str
|
|
start: float
|
|
end: float
|
|
confidence: Optional[float] = None
|
|
|
|
|
|
@dataclass
|
|
class Segment:
|
|
"""Segment (line/phrase) level timestamp information."""
|
|
text: str
|
|
start: float
|
|
end: float
|
|
words: List[WordTimestamp]
|
|
speaker: Optional[str] = None
|
|
|
|
|
|
@dataclass
|
|
class LyricTranscriptionResult:
|
|
"""Complete transcription result."""
|
|
text: str
|
|
segments: List[Segment]
|
|
language: str
|
|
duration: float
|
|
model: str
|
|
|
|
|
|
# Available Whisper models with their characteristics
|
|
WHISPER_MODELS = {
|
|
"tiny": {
|
|
"id": "tiny",
|
|
"label": "Tiny (fastest, least accurate)",
|
|
"size": "39 MB",
|
|
"relative_speed": "~32x",
|
|
"english_only": False,
|
|
},
|
|
"base": {
|
|
"id": "base",
|
|
"label": "Base (fast, good accuracy)",
|
|
"size": "74 MB",
|
|
"relative_speed": "~16x",
|
|
"english_only": False,
|
|
},
|
|
"small": {
|
|
"id": "small",
|
|
"label": "Small (balanced)",
|
|
"size": "244 MB",
|
|
"relative_speed": "~6x",
|
|
"english_only": False,
|
|
},
|
|
"medium": {
|
|
"id": "medium",
|
|
"label": "Medium (slower, better accuracy)",
|
|
"size": "769 MB",
|
|
"relative_speed": "~2x",
|
|
"english_only": False,
|
|
},
|
|
"large": {
|
|
"id": "large",
|
|
"label": "Large (slowest, best accuracy)",
|
|
"size": "1550 MB",
|
|
"relative_speed": "1x",
|
|
"english_only": False,
|
|
},
|
|
"large-v2": {
|
|
"id": "large-v2",
|
|
"label": "Large v2 (recommended)",
|
|
"size": "1550 MB",
|
|
"relative_speed": "1x",
|
|
"english_only": False,
|
|
},
|
|
"large-v3": {
|
|
"id": "large-v3",
|
|
"label": "Large v3 (latest)",
|
|
"size": "1550 MB",
|
|
"relative_speed": "1x",
|
|
"english_only": False,
|
|
},
|
|
}
|
|
|
|
DEFAULT_WHISPER_MODEL = "large-v2"
|
|
|
|
|
|
_whisper_cache: Dict[str, Any] = {}
|
|
|
|
# Set to False when a CUDA OOM or "unknown error" corrupts the GPU context.
|
|
# Once False, all subsequent Whisper loads are forced to CPU so the server
|
|
# keeps working without requiring a restart.
|
|
_cuda_ok: bool = True
|
|
|
|
|
|
def _mark_cuda_failed() -> None:
|
|
"""Record that the CUDA context is bad and purge any GPU-resident models."""
|
|
global _cuda_ok
|
|
_cuda_ok = False
|
|
_whisper_cache.clear()
|
|
try:
|
|
import torch
|
|
torch.cuda.empty_cache()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def reset_cuda() -> dict:
|
|
"""
|
|
Re-enable GPU acceleration after a CUDA error recovery.
|
|
|
|
The transcriber falls back to CPU when a CUDA OOM or 'unknown error'
|
|
occurs. Call this once the GPU is confirmed free to restore GPU usage
|
|
without restarting the container.
|
|
"""
|
|
global _cuda_ok
|
|
_cuda_ok = True
|
|
_whisper_cache.clear()
|
|
cuda_available = False
|
|
try:
|
|
import torch
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
cuda_available = True
|
|
except Exception:
|
|
pass
|
|
return {"ok": True, "cuda_available": cuda_available, "cuda_ok": _cuda_ok}
|
|
|
|
|
|
def get_whisper_model(model_name: str = DEFAULT_WHISPER_MODEL, device: Optional[str] = None):
|
|
"""
|
|
Load and in-process-cache a Whisper model.
|
|
|
|
Reuses an already-loaded model of the same name to avoid reloading ~1.5 GB
|
|
weights on every transcription call. If a *different* model is requested,
|
|
the cached one is evicted and the CUDA allocator cache is flushed first so
|
|
that the new model can fit alongside any other resident GPU allocations.
|
|
|
|
When `device` is explicitly passed (e.g. "cpu") it overrides the auto-detect
|
|
logic. This is used by the CUDA-error recovery path in `transcribe_lyrics`.
|
|
"""
|
|
import whisper
|
|
|
|
cache_key = f"{model_name}:{device or 'auto'}"
|
|
|
|
if cache_key in _whisper_cache:
|
|
return _whisper_cache[cache_key]
|
|
|
|
# Evict any previously cached model and release its GPU memory before
|
|
# loading the new one.
|
|
if _whisper_cache:
|
|
_whisper_cache.clear()
|
|
try:
|
|
import torch
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
except Exception:
|
|
pass
|
|
|
|
load_kwargs: dict = {}
|
|
if device is not None:
|
|
load_kwargs["device"] = device
|
|
|
|
model = whisper.load_model(model_name, **load_kwargs)
|
|
_whisper_cache[cache_key] = model
|
|
return model
|
|
|
|
|
|
class _LineCapture(io.RawIOBase):
|
|
"""
|
|
File-like object that buffers writes and calls `callback` once per complete line.
|
|
Used to intercept Whisper's verbose stdout output for live progress reporting.
|
|
"""
|
|
def __init__(self, callback: Callable[[str], None]) -> None:
|
|
self._cb = callback
|
|
self._buf = ''
|
|
|
|
def write(self, text) -> int: # type: ignore[override]
|
|
if isinstance(text, (bytes, bytearray)):
|
|
text = text.decode('utf-8', errors='replace')
|
|
self._buf += text
|
|
while '\n' in self._buf:
|
|
line, self._buf = self._buf.split('\n', 1)
|
|
cleaned = line.strip()
|
|
if cleaned:
|
|
self._cb(cleaned)
|
|
return len(text)
|
|
|
|
def flush(self) -> None:
|
|
pass
|
|
|
|
# Some libraries call fileno(); raise so they fall back gracefully.
|
|
def fileno(self): # type: ignore[override]
|
|
raise io.UnsupportedOperation('fileno')
|
|
|
|
|
|
|
|
def transcribe_lyrics(
|
|
audio_path: str,
|
|
model_name: str = DEFAULT_WHISPER_MODEL,
|
|
language: Optional[str] = None,
|
|
task: Literal["transcribe", "translate"] = "transcribe",
|
|
include_word_timestamps: bool = True,
|
|
initial_prompt: Optional[str] = None,
|
|
temperature: float = 0.0,
|
|
condition_on_previous_text: bool = False,
|
|
compression_ratio_threshold: float = 1.35,
|
|
beam_size: Optional[int] = None,
|
|
line_callback: Optional[Callable[[str], None]] = None,
|
|
) -> LyricTranscriptionResult:
|
|
"""
|
|
Transcribe vocals to lyrics with word-level timestamps.
|
|
|
|
Anti-hallucination defaults (differ from stock Whisper):
|
|
- condition_on_previous_text=False — prevents the model from feeding
|
|
a repeated phrase back as context, breaking the repetition loop.
|
|
- temperature fallback tuple — when temperature=0.0, Whisper is
|
|
automatically given (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) so it can retry
|
|
a stuck segment with increasing randomness.
|
|
- compression_ratio_threshold=1.35 — flags repetitive segments sooner
|
|
(stock default is 2.4) and triggers the temperature fallback.
|
|
- _deduplicate_segments post-pass — removes any repeated segments or
|
|
internally-looped phrases that still slip through.
|
|
|
|
Args:
|
|
audio_path: Path to audio file (should be vocals stem for best results)
|
|
model_name: Whisper model to use
|
|
language: ISO-639-1 language code (auto-detected if None)
|
|
task: 'transcribe' for original language, 'translate' for English
|
|
include_word_timestamps: Whether to include word-level timestamps
|
|
initial_prompt: Optional text to guide the model's style
|
|
temperature: Base sampling temperature; 0.0 enables the fallback tuple
|
|
condition_on_previous_text: Feed previous segment as context (False prevents loops)
|
|
compression_ratio_threshold: Retry segment above this gzip ratio (lower = stricter)
|
|
|
|
Returns:
|
|
LyricTranscriptionResult with text, segments, and timestamps
|
|
"""
|
|
import whisper
|
|
|
|
# Free any reserved-but-unallocated GPU memory from prior jobs (e.g. Demucs,
|
|
# BS-RoFormer) before the Whisper model is loaded / used.
|
|
try:
|
|
import torch
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
except Exception:
|
|
pass
|
|
|
|
# If a previous job corrupted the CUDA context (OOM → "unknown error"),
|
|
# force CPU so this job still completes rather than failing immediately.
|
|
forced_device: Optional[str] = None if _cuda_ok else "cpu"
|
|
model = get_whisper_model(model_name, device=forced_device)
|
|
|
|
# When temperature is 0 (greedy), pass a fallback sequence so Whisper can
|
|
# recover from stuck segments by retrying with increasing randomness.
|
|
effective_temperature: Any = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) if temperature == 0.0 else temperature
|
|
|
|
decode_options: dict = {
|
|
"task": task,
|
|
"temperature": effective_temperature,
|
|
"condition_on_previous_text": condition_on_previous_text,
|
|
"compression_ratio_threshold": compression_ratio_threshold,
|
|
}
|
|
|
|
if language:
|
|
decode_options["language"] = language
|
|
if initial_prompt:
|
|
decode_options["initial_prompt"] = initial_prompt
|
|
if beam_size is not None:
|
|
decode_options["beam_size"] = beam_size
|
|
|
|
def _run_transcribe(mdl: Any) -> Any:
|
|
"""Run model.transcribe(), capturing verbose output when a callback is set."""
|
|
if line_callback is not None:
|
|
_cap = _LineCapture(line_callback)
|
|
with contextlib.redirect_stdout(_cap), contextlib.redirect_stderr(_cap):
|
|
return mdl.transcribe(
|
|
audio_path,
|
|
word_timestamps=include_word_timestamps,
|
|
verbose=True,
|
|
**decode_options,
|
|
)
|
|
return mdl.transcribe(
|
|
audio_path,
|
|
word_timestamps=include_word_timestamps,
|
|
**decode_options,
|
|
)
|
|
|
|
# Run transcription with word timestamps.
|
|
# On any CUDA error (OOM already happened, or the context was already
|
|
# corrupted), mark the GPU as failed, reload the model on CPU, and retry.
|
|
# This keeps the server usable without a restart.
|
|
try:
|
|
result = _run_transcribe(model)
|
|
except RuntimeError as exc:
|
|
err_str = str(exc)
|
|
if "CUDA" in err_str or "cuda" in err_str:
|
|
_mark_cuda_failed()
|
|
if line_callback:
|
|
line_callback("⚠️ CUDA error — retrying on CPU (server restart not required)")
|
|
cpu_model = get_whisper_model(model_name, device="cpu")
|
|
result = _run_transcribe(cpu_model)
|
|
else:
|
|
raise
|
|
|
|
# Parse results into structured format
|
|
raw_segments: List[Segment] = []
|
|
for seg in result["segments"]:
|
|
words = []
|
|
if include_word_timestamps and "words" in seg:
|
|
for word_info in seg["words"]:
|
|
words.append(WordTimestamp(
|
|
word=word_info["word"].strip(),
|
|
start=word_info["start"],
|
|
end=word_info["end"],
|
|
confidence=word_info.get("probability")
|
|
))
|
|
|
|
raw_segments.append(Segment(
|
|
text=seg["text"].strip(),
|
|
start=seg["start"],
|
|
end=seg["end"],
|
|
words=words
|
|
))
|
|
|
|
# Calculate total duration
|
|
duration = max(seg.end for seg in raw_segments) if raw_segments else 0.0
|
|
|
|
return LyricTranscriptionResult(
|
|
text=result["text"].strip(),
|
|
segments=raw_segments,
|
|
language=result.get("language", "unknown"),
|
|
duration=duration,
|
|
model=f"whisper-{model_name}"
|
|
)
|
|
|
|
|
|
def result_to_lrc(result: LyricTranscriptionResult) -> str:
|
|
"""Convert transcription result to LRC format (line-level timestamps).
|
|
|
|
Speaker info (when present) is encoded as |sp:SPEAKER_00| immediately
|
|
after the LRC timestamp so the lyric player can colour-code lines per
|
|
speaker while remaining compatible with standard LRC parsers (which
|
|
simply ignore the tag as leading text).
|
|
"""
|
|
lines = []
|
|
for seg in result.segments:
|
|
minutes = int(seg.start // 60)
|
|
seconds = int(seg.start % 60)
|
|
centis = int((seg.start % 1) * 100)
|
|
speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else ""
|
|
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def result_to_elrc(result: LyricTranscriptionResult) -> str:
|
|
"""Convert transcription result to ELRC format (word-level timestamps).
|
|
|
|
Speaker info is encoded the same way as in result_to_lrc — |sp:SPEAKER_00|
|
|
is prepended to the word-timestamp content of each line.
|
|
"""
|
|
lines = []
|
|
for seg in result.segments:
|
|
speaker_tag = f"|sp:{seg.speaker}|" if seg.speaker else ""
|
|
if not seg.words:
|
|
# Fallback to segment-level if no word timestamps
|
|
minutes = int(seg.start // 60)
|
|
seconds = int(seg.start % 60)
|
|
centis = int((seg.start % 1) * 100)
|
|
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{speaker_tag}{seg.text}")
|
|
continue
|
|
|
|
# Line timestamp is start of first word
|
|
line_start = seg.words[0].start
|
|
line_min = int(line_start // 60)
|
|
line_sec = int(line_start % 60)
|
|
line_centi = int((line_start % 1) * 100)
|
|
|
|
# Build word-level timestamps.
|
|
# Strip each word's surrounding whitespace before embedding — Whisper/WhisperX
|
|
# sometimes include a leading space (e.g. " world") which would produce
|
|
# double-spaces when tokens are joined.
|
|
word_parts = []
|
|
for word in seg.words:
|
|
w_min = int(word.start // 60)
|
|
w_sec = int(word.start % 60)
|
|
w_centi = int((word.start % 1) * 100)
|
|
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word.strip()}")
|
|
|
|
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{speaker_tag}{' '.join(word_parts)}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def result_to_json(result: LyricTranscriptionResult) -> Dict[str, Any]:
|
|
"""Convert transcription result to JSON-serializable dict."""
|
|
return {
|
|
"text": result.text,
|
|
"language": result.language,
|
|
"duration": result.duration,
|
|
"model": result.model,
|
|
"segments": [
|
|
{
|
|
"text": seg.text,
|
|
"start": seg.start,
|
|
"end": seg.end,
|
|
"speaker": seg.speaker,
|
|
"words": [
|
|
{
|
|
"word": w.word,
|
|
"start": w.start,
|
|
"end": w.end,
|
|
"confidence": w.confidence
|
|
}
|
|
for w in seg.words
|
|
]
|
|
}
|
|
for seg in result.segments
|
|
]
|
|
}
|
|
|
|
|
|
def result_to_txt(result: LyricTranscriptionResult) -> str:
|
|
"""Convert transcription result to plain text.
|
|
|
|
One Whisper segment per line so the output can be fed directly into the
|
|
ctc-forced-aligner (lrc_service) which splits on newlines to identify
|
|
individual lyric lines. Falls back to the joined text if no segments.
|
|
"""
|
|
if result.segments:
|
|
return '\n'.join(seg.text.strip() for seg in result.segments if seg.text.strip())
|
|
return result.text
|
|
|
|
|
|
def result_to_srt(result: LyricTranscriptionResult) -> str:
|
|
"""Convert transcription result to SRT subtitle format."""
|
|
lines = []
|
|
for i, seg in enumerate(result.segments, 1):
|
|
# Format: HH:MM:SS,mmm
|
|
def fmt_srt_time(t: float) -> str:
|
|
hours = int(t // 3600)
|
|
mins = int((t % 3600) // 60)
|
|
secs = int(t % 60)
|
|
millis = int((t % 1) * 1000)
|
|
return f"{hours:02d}:{mins:02d}:{secs:02d},{millis:03d}"
|
|
|
|
lines.append(str(i))
|
|
lines.append(f"{fmt_srt_time(seg.start)} --> {fmt_srt_time(seg.end)}")
|
|
lines.append(seg.text)
|
|
lines.append("") # Empty line between entries
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def result_to_vtt(result: LyricTranscriptionResult) -> str:
|
|
"""Convert transcription result to WebVTT subtitle format."""
|
|
lines = ["WEBVTT", ""]
|
|
|
|
for seg in result.segments:
|
|
def fmt_vtt_time(t: float) -> str:
|
|
hours = int(t // 3600)
|
|
mins = int((t % 3600) // 60)
|
|
secs = int(t % 60)
|
|
millis = int((t % 1) * 1000)
|
|
return f"{hours:02d}:{mins:02d}:{secs:02d}.{millis:03d}"
|
|
|
|
lines.append(f"{fmt_vtt_time(seg.start)} --> {fmt_vtt_time(seg.end)}")
|
|
lines.append(seg.text)
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
_PUNCT_BREAK = frozenset('.!?,;')
|
|
|
|
|
|
def _split_long_segments(segments: list, max_words: int = 10) -> list:
|
|
"""
|
|
Split over-long WhisperX segments into shorter lines suitable for LRC display.
|
|
|
|
Strategy (applied per segment):
|
|
1. Walk the word list looking for *punctuation boundaries* — words whose
|
|
text ends with a character in _PUNCT_BREAK. When such a boundary is
|
|
found AND the current chunk already has at least 3 words, emit a new
|
|
sub-segment there.
|
|
2. If no punctuation boundary is found within `max_words` words, force-
|
|
split at `max_words` regardless.
|
|
|
|
Each sub-segment inherits the speaker label and gets timestamps from the
|
|
first / last word in the slice. Segments with fewer than 2 words are left
|
|
unchanged.
|
|
"""
|
|
if max_words <= 0:
|
|
return segments
|
|
|
|
result = []
|
|
for seg in segments:
|
|
words = seg.get("words") or []
|
|
if len(words) <= max_words:
|
|
result.append(seg)
|
|
continue
|
|
|
|
# Build sub-segment slices
|
|
chunks: list[list] = []
|
|
chunk: list = []
|
|
for w in words:
|
|
chunk.append(w)
|
|
text = w.get("word", "").rstrip()
|
|
at_punct = bool(text) and text[-1] in _PUNCT_BREAK
|
|
at_limit = len(chunk) >= max_words
|
|
if (at_punct and len(chunk) >= 3) or at_limit:
|
|
chunks.append(chunk)
|
|
chunk = []
|
|
if chunk:
|
|
chunks.append(chunk)
|
|
|
|
for ch in chunks:
|
|
first, last = ch[0], ch[-1]
|
|
sub_text = "".join(w.get("word", "") for w in ch).strip()
|
|
result.append({
|
|
"start": first.get("start") if first.get("start") is not None else seg["start"],
|
|
"end": last.get("end") if last.get("end") is not None else seg["end"],
|
|
"text": sub_text,
|
|
"words": ch,
|
|
"speaker": seg.get("speaker"),
|
|
})
|
|
|
|
return result
|
|
|
|
|
|
_VOCAL_KEYWORDS = frozenset(('vocal', 'vocals', 'voice', 'lead', 'sing'))
|
|
_AUDIO_EXTS = frozenset(('.mp3', '.wav', '.flac', '.ogg', '.m4a', '.aac'))
|
|
|
|
|
|
def _ensure_vocals(audio_path: str, job_id: str) -> str:
|
|
"""
|
|
Return the path to the best available vocals audio for transcription.
|
|
|
|
Priority:
|
|
1. audio_path already looks like a vocals stem (filename keyword check).
|
|
2. A vocals file exists in the same Stems/ directory — use it.
|
|
3. Run htdemucs --two-stems vocals to extract vocals from audio_path.
|
|
Falls back to audio_path unchanged on any failure.
|
|
"""
|
|
import subprocess, sys
|
|
|
|
p = Path(audio_path)
|
|
stem_lower = p.stem.lower()
|
|
|
|
# 1. Already a vocals stem?
|
|
if any(kw in stem_lower for kw in _VOCAL_KEYWORDS):
|
|
return audio_path
|
|
|
|
# 2. Look for a sibling vocals file in the same directory.
|
|
for f in sorted(p.parent.iterdir()):
|
|
if f == p or not f.is_file():
|
|
continue
|
|
if f.suffix.lower() not in _AUDIO_EXTS:
|
|
continue
|
|
if any(kw in f.stem.lower() for kw in _VOCAL_KEYWORDS):
|
|
jm.update_job(job_id, progress=f"Using existing vocal stem: {f.name}")
|
|
return str(f)
|
|
|
|
# 3. No vocal stem found — extract with htdemucs --two-stems vocals.
|
|
jm.update_job(job_id, progress="No vocal stem found — extracting vocals with Demucs (htdemucs)...")
|
|
extract_dir = LRC_SYNC_DIR / job_id / "vocals_extract"
|
|
extract_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
cmd = [
|
|
sys.executable, '-m', 'demucs',
|
|
'--name', 'htdemucs',
|
|
'--two-stems', 'vocals',
|
|
'--out', str(extract_dir),
|
|
audio_path,
|
|
]
|
|
try:
|
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
|
|
if proc.returncode == 0:
|
|
# Output layout: <extract_dir>/htdemucs/<input_stem>/vocals.wav
|
|
for candidate in sorted(extract_dir.rglob('vocals.*')):
|
|
if candidate.is_file() and candidate.suffix.lower() in _AUDIO_EXTS:
|
|
jm.update_job(job_id, progress=f"Vocals extracted: {candidate.name}")
|
|
return str(candidate)
|
|
else:
|
|
jm.update_job(job_id, progress=f"Demucs exited {proc.returncode} — using original audio.")
|
|
except Exception as _e:
|
|
jm.update_job(job_id, progress=f"Demucs extraction failed ({_e}) — using original audio.")
|
|
|
|
return audio_path
|
|
|
|
|
|
def run_lyric_transcription(
|
|
job_id: str,
|
|
audio_path: str,
|
|
model: str = DEFAULT_WHISPER_MODEL,
|
|
language: Optional[str] = None,
|
|
task: str = "transcribe",
|
|
output_formats: Optional[List[str]] = None,
|
|
initial_prompt: Optional[str] = None,
|
|
temperature: float = 0.0,
|
|
condition_on_previous_text: bool = False,
|
|
word_timestamps: bool = True,
|
|
beam_size: Optional[int] = None,
|
|
output_dir: Optional[str] = None,
|
|
engine: str = "whisperx",
|
|
diarize: bool = False,
|
|
):
|
|
"""
|
|
Background task: transcribe vocals/lyrics from audio.
|
|
|
|
Args:
|
|
job_id: Job identifier
|
|
audio_path: Path to audio file
|
|
model: Whisper model name
|
|
language: Language code (auto-detect if None)
|
|
task: 'transcribe' or 'translate'
|
|
output_formats: List of output formats ('lrc', 'elrc', 'json', 'txt', 'srt', 'vtt')
|
|
initial_prompt: Optional prompt text
|
|
temperature: Sampling temperature
|
|
output_dir: Custom output directory
|
|
engine: 'whisperx' (faster-whisper + phoneme alignment)
|
|
diarize: Run speaker diarization after alignment (WhisperX only)
|
|
"""
|
|
if output_formats is None:
|
|
output_formats = ["elrc", "json", "txt"]
|
|
|
|
if output_dir:
|
|
out_path = Path(output_dir)
|
|
else:
|
|
out_path = LRC_SYNC_DIR / job_id
|
|
out_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
if engine == "whisperx":
|
|
# ── WhisperX path: faster-whisper backend + phoneme-level alignment ──
|
|
try:
|
|
import torch
|
|
import whisperx
|
|
except ImportError as _imp_err:
|
|
jm.update_job(
|
|
job_id,
|
|
status=jm.JobStatus.ERROR,
|
|
error=f"WhisperX is not installed ({_imp_err}). "
|
|
"Install with: pip install whisperx",
|
|
progress="Error.",
|
|
)
|
|
return
|
|
|
|
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading WhisperX model ({model})...")
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
compute_type = "float16" if device == "cuda" else "int8"
|
|
|
|
# Auto-ensure vocals stem for best transcription quality.
|
|
audio_path = _ensure_vocals(audio_path, job_id)
|
|
|
|
wx_model = whisperx.load_model(model, device, compute_type=compute_type, language=language or None)
|
|
audio_arr = whisperx.load_audio(audio_path)
|
|
duration = float(len(audio_arr)) / 16000.0
|
|
|
|
jm.update_job(job_id, progress=f"Transcribing with WhisperX ({model})...")
|
|
wx_result = wx_model.transcribe(audio_arr, batch_size=16)
|
|
detected_lang = wx_result.get("language") or language or "unknown"
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
jm.update_job(job_id, progress="Aligning word timestamps (WhisperX)...")
|
|
model_a, metadata = whisperx.load_align_model(language_code=detected_lang, device=device)
|
|
wx_result = whisperx.align(
|
|
wx_result["segments"], model_a, metadata, audio_arr, device,
|
|
return_char_alignments=False,
|
|
)
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
if diarize:
|
|
jm.update_job(job_id, progress="Running speaker diarization...")
|
|
try:
|
|
import os
|
|
hf_token = os.environ.get("HF_TOKEN") or None
|
|
if not hf_token:
|
|
raise RuntimeError(
|
|
"HF_TOKEN environment variable is not set. "
|
|
"Add HF_TOKEN=<your_token> to your .env file and restart the container."
|
|
)
|
|
# Log in globally so pyannote.audio picks up the token regardless of
|
|
# which param name the installed version expects (use_auth_token vs token).
|
|
try:
|
|
from huggingface_hub import login as _hf_login
|
|
_hf_login(token=hf_token, add_to_git_credential=False)
|
|
except Exception:
|
|
pass
|
|
diarize_model = whisperx.DiarizationPipeline(use_auth_token=hf_token, device=device)
|
|
diarize_segments = diarize_model(audio_arr)
|
|
wx_result = whisperx.assign_word_speakers(diarize_segments, wx_result)
|
|
except Exception as _diar_err:
|
|
jm.update_job(job_id, progress=f"Diarization failed ({_diar_err}), continuing without speaker labels...")
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
# Split over-long segments into shorter lyric lines before converting.
|
|
# Punctuation boundaries are preferred; falls back to a hard word-count cut.
|
|
raw_segments = _split_long_segments(wx_result["segments"], max_words=10)
|
|
|
|
# Convert WhisperX output → LyricTranscriptionResult
|
|
segments: List[Segment] = []
|
|
for seg in raw_segments:
|
|
words = []
|
|
for w in seg.get("words", []):
|
|
_ws = w.get("start")
|
|
_we = w.get("end")
|
|
words.append(WordTimestamp(
|
|
word=w.get("word", ""),
|
|
start=float(_ws if _ws is not None else seg["start"]),
|
|
end=float(_we if _we is not None else seg["end"]),
|
|
confidence=w.get("score"),
|
|
))
|
|
segments.append(Segment(
|
|
text=seg["text"].strip(),
|
|
start=float(seg["start"]),
|
|
end=float(seg["end"]),
|
|
words=words,
|
|
speaker=seg.get("speaker"),
|
|
))
|
|
|
|
result = LyricTranscriptionResult(
|
|
text=" ".join(seg.text for seg in segments),
|
|
segments=segments,
|
|
language=detected_lang,
|
|
duration=duration,
|
|
model=f"whisperx-{model}",
|
|
)
|
|
|
|
else:
|
|
# ── Standard Whisper path ─────────────────────────────────────────────
|
|
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading Whisper model ({model})...")
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
jm.update_job(job_id, progress=f"Transcribing lyrics with Whisper ({model})...")
|
|
|
|
# Accumulate Whisper's verbose segment output for live progress display.
|
|
# Noisy lines (tqdm bars, empty lines) are filtered out.
|
|
_log: List[str] = []
|
|
_NOISE_TOKENS = ('%|', 'it/s', 'B/s', '100%')
|
|
|
|
def _on_line(line: str) -> None:
|
|
if any(tok in line for tok in _NOISE_TOKENS):
|
|
return
|
|
_log.append(line)
|
|
if len(_log) > 30:
|
|
_log.pop(0)
|
|
jm.update_job(job_id, progress='\n'.join(_log))
|
|
|
|
_TRANSCRIBE_TIMEOUT = 7200 # 2-hour hard cap
|
|
_result_box: list = [None]
|
|
_exc_box: list = [None]
|
|
|
|
def _do_transcribe():
|
|
try:
|
|
_result_box[0] = transcribe_lyrics(
|
|
audio_path=audio_path,
|
|
model_name=model,
|
|
language=language,
|
|
task=task, # type: ignore
|
|
include_word_timestamps=word_timestamps,
|
|
initial_prompt=initial_prompt,
|
|
temperature=temperature,
|
|
condition_on_previous_text=condition_on_previous_text,
|
|
beam_size=beam_size,
|
|
line_callback=_on_line,
|
|
)
|
|
except Exception as _e:
|
|
_exc_box[0] = _e
|
|
|
|
_t = threading.Thread(target=_do_transcribe, daemon=True)
|
|
_t.start()
|
|
_t.join(timeout=_TRANSCRIBE_TIMEOUT)
|
|
|
|
if _t.is_alive():
|
|
jm.update_job(
|
|
job_id,
|
|
status=jm.JobStatus.ERROR,
|
|
error=f"Transcription timed out after {_TRANSCRIBE_TIMEOUT // 3600}h. "
|
|
"The model is still running in the background and will release "
|
|
"the GPU when it finishes.",
|
|
progress="Timed out.",
|
|
)
|
|
return
|
|
|
|
if _exc_box[0] is not None:
|
|
raise _exc_box[0]
|
|
|
|
result = _result_box[0]
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
# Generate output files
|
|
jm.update_job(job_id, progress="Generating output files...")
|
|
|
|
base_name = Path(audio_path).stem
|
|
output_files = []
|
|
|
|
# JSON (full data)
|
|
if "json" in output_formats:
|
|
json_path = out_path / f"{base_name}_transcription.json"
|
|
json_data = result_to_json(result)
|
|
json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"})
|
|
|
|
# LRC (line-level)
|
|
if "lrc" in output_formats:
|
|
lrc_path = out_path / f"{base_name}.lrc"
|
|
lrc_content = result_to_lrc(result)
|
|
lrc_path.write_text(lrc_content, encoding="utf-8")
|
|
output_files.append({"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"})
|
|
|
|
# ELRC (word-level)
|
|
if "elrc" in output_formats:
|
|
elrc_path = out_path / f"{base_name}.elrc"
|
|
elrc_content = result_to_elrc(result)
|
|
elrc_path.write_text(elrc_content, encoding="utf-8")
|
|
output_files.append({"name": elrc_path.name, "path": str(elrc_path), "type": "elrc"})
|
|
|
|
# Plain text
|
|
if "txt" in output_formats:
|
|
txt_path = out_path / f"{base_name}_lyrics.txt"
|
|
txt_content = result_to_txt(result)
|
|
txt_path.write_text(txt_content, encoding="utf-8")
|
|
output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"})
|
|
|
|
# SRT subtitles
|
|
if "srt" in output_formats:
|
|
srt_path = out_path / f"{base_name}.srt"
|
|
srt_content = result_to_srt(result)
|
|
srt_path.write_text(srt_content, encoding="utf-8")
|
|
output_files.append({"name": srt_path.name, "path": str(srt_path), "type": "srt"})
|
|
|
|
# VTT subtitles
|
|
if "vtt" in output_formats:
|
|
vtt_path = out_path / f"{base_name}.vtt"
|
|
vtt_content = result_to_vtt(result)
|
|
vtt_path.write_text(vtt_content, encoding="utf-8")
|
|
output_files.append({"name": vtt_path.name, "path": str(vtt_path), "type": "vtt"})
|
|
|
|
# Update job with results
|
|
result_summary = {
|
|
"model": result.model,
|
|
"language": result.language,
|
|
"duration": result.duration,
|
|
"segment_count": len(result.segments),
|
|
"word_count": sum(len(seg.words) for seg in result.segments),
|
|
}
|
|
|
|
jm.update_job(
|
|
job_id,
|
|
status=jm.JobStatus.DONE,
|
|
progress="Lyric transcription complete.",
|
|
files=output_files,
|
|
result=result_summary
|
|
)
|
|
|
|
except Exception as exc:
|
|
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
|
|
|
|
|
def run_lyric_transcription_with_diarization(
|
|
job_id: str,
|
|
audio_path: str,
|
|
model: str = DEFAULT_WHISPER_MODEL,
|
|
language: Optional[str] = None,
|
|
num_speakers: Optional[int] = None,
|
|
min_speakers: Optional[int] = None,
|
|
max_speakers: Optional[int] = None,
|
|
output_formats: Optional[List[str]] = None,
|
|
output_dir: Optional[str] = None,
|
|
):
|
|
"""
|
|
Background task: transcribe vocals with speaker diarization.
|
|
|
|
This uses WhisperX for diarization which requires additional setup:
|
|
- pip install whisperx
|
|
- HuggingFace token for pyannote models
|
|
|
|
Args:
|
|
job_id: Job identifier
|
|
audio_path: Path to audio file
|
|
model: Whisper model name
|
|
language: Language code
|
|
num_speakers: Exact number of speakers (optional)
|
|
min_speakers: Minimum number of speakers
|
|
max_speakers: Maximum number of speakers
|
|
output_formats: List of output formats
|
|
output_dir: Custom output directory
|
|
"""
|
|
if output_formats is None:
|
|
output_formats = ["json", "txt"]
|
|
|
|
if output_dir:
|
|
out_path = Path(output_dir)
|
|
else:
|
|
out_path = LRC_SYNC_DIR / job_id
|
|
out_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
import whisperx
|
|
|
|
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading WhisperX model...")
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
# Load WhisperX model
|
|
device = "cuda" if jm.import_module("torch").cuda.is_available() else "cpu"
|
|
compute_type = "float16" if device == "cuda" else "int8"
|
|
|
|
model_obj = whisperx.load_model(model, device, compute_type=compute_type)
|
|
|
|
# Load audio
|
|
audio = whisperx.load_audio(audio_path)
|
|
|
|
jm.update_job(job_id, progress="Transcribing with WhisperX...")
|
|
|
|
# Transcribe
|
|
result = model_obj.transcribe(audio, batch_size=16)
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
# Align timestamps
|
|
jm.update_job(job_id, progress="Aligning word timestamps...")
|
|
model_a, metadata = whisperx.load_align_model(language_code=result["language"], device=device)
|
|
result = whisperx.align(result["segments"], model_a, metadata, audio, device)
|
|
|
|
if jm.should_cancel(job_id):
|
|
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
|
return
|
|
|
|
# Diarization
|
|
jm.update_job(job_id, progress="Running speaker diarization...")
|
|
diarize_model = whisperx.DiarizationPipeline(device=device)
|
|
diarize_segments = diarize_model(audio, num_speakers=num_speakers,
|
|
min_speakers=min_speakers,
|
|
max_speakers=max_speakers)
|
|
|
|
result = whisperx.assign_word_speakers(diarize_segments, result)
|
|
|
|
# Generate outputs
|
|
jm.update_job(job_id, progress="Generating output files...")
|
|
|
|
base_name = Path(audio_path).stem
|
|
output_files = []
|
|
|
|
if "json" in output_formats:
|
|
json_path = out_path / f"{base_name}_diarized.json"
|
|
json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"})
|
|
|
|
if "txt" in output_formats:
|
|
txt_path = out_path / f"{base_name}_diarized.txt"
|
|
lines = [f"[{seg.get('speaker', 'UNKNOWN')}] {seg['text']}" for seg in result["segments"]]
|
|
txt_path.write_text("\n".join(lines), encoding="utf-8")
|
|
output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"})
|
|
|
|
result_summary = {
|
|
"model": f"whisperx-{model}",
|
|
"language": result.get("language", "unknown"),
|
|
"segment_count": len(result["segments"]),
|
|
"diarization": True,
|
|
}
|
|
|
|
jm.update_job(
|
|
job_id,
|
|
status=jm.JobStatus.DONE,
|
|
progress="Lyric transcription with diarization complete.",
|
|
files=output_files,
|
|
result=result_summary
|
|
)
|
|
|
|
except ImportError:
|
|
jm.update_job(
|
|
job_id,
|
|
status=jm.JobStatus.ERROR,
|
|
error="WhisperX not installed. Install with: pip install whisperx",
|
|
progress="Error."
|
|
)
|
|
except Exception as exc:
|
|
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
|
|
|
|
|
def is_whisper_available() -> bool:
|
|
"""Check if Whisper is available."""
|
|
try:
|
|
import whisper
|
|
return True
|
|
except ImportError:
|
|
return False
|
|
|
|
|
|
def is_whisperx_available() -> bool:
|
|
"""Check if WhisperX is available."""
|
|
try:
|
|
import whisperx
|
|
return True
|
|
except ImportError:
|
|
return False
|