Files
stemengine_old/backend/routers_root_backup/lyric_transcribe.py
T
2026-03-03 14:52:19 +09:00

543 lines
21 KiB
Python
Executable File

"""
Lyric transcription endpoint - transcribe vocals to text with timestamps.
Based on Scriberr's transcription approach using OpenAI Whisper.
This provides:
- Word-level and line-level timestamps
- Multiple output formats (LRC, ELRC, JSON, SRT, VTT, TXT)
- Speaker diarization support (via WhisperX)
- Multiple Whisper model sizes
"""
import json
import shutil
from pathlib import Path
from typing import Optional, List
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
from pydantic import BaseModel
from .. import job_manager as jm
from ..config import UPLOAD_DIR, PROJECTS_DIR
from ..services.lyric_transcriber_service import (
run_lyric_transcription,
run_lyric_transcription_with_diarization,
WHISPER_MODELS,
DEFAULT_WHISPER_MODEL,
is_whisper_available,
is_whisperx_available,
)
router = APIRouter(prefix="/api/lyric-transcribe", tags=["lyric-transcribe"])
class LyricTranscribeRequest(BaseModel):
"""Request body for project-based lyric transcription."""
stem_name: str
model: str = DEFAULT_WHISPER_MODEL
language: Optional[str] = None
task: str = "transcribe"
output_formats: List[str] = ["elrc", "json", "txt"]
initial_prompt: Optional[str] = None
temperature: float = 0.0
condition_on_previous_text: bool = True
word_timestamps: bool = True
beam_size: Optional[int] = None
class LyricTranscribeWithDiarizationRequest(BaseModel):
"""Request body for transcription with speaker diarization."""
stem_name: str
model: str = DEFAULT_WHISPER_MODEL
language: Optional[str] = None
num_speakers: Optional[int] = None
min_speakers: Optional[int] = None
max_speakers: Optional[int] = None
output_formats: List[str] = ["json", "txt"]
VALID_OUTPUT_FORMATS = {"lrc", "elrc", "json", "txt", "srt", "vtt"}
VALID_TASKS = {"transcribe", "translate"}
_AUDIO_EXTS = {".mp3", ".wav", ".flac", ".m4a", ".ogg", ".wma"}
# ---------------------------------------------------------------------------
# Helper functions
# ---------------------------------------------------------------------------
def _find_vocal_file(project_dir: Path) -> Optional[Path]:
"""Find the vocal file in a project's Stems folder."""
stems_dir = project_dir / "Stems"
if not stems_dir.exists():
return None
# Look for common vocal file names
vocal_patterns = ["vocals", "Vocals", "lead", "Lead", "vocal", "Vocal"]
for pattern in vocal_patterns:
for ext in _AUDIO_EXTS:
candidate = stems_dir / f"{pattern}{ext}"
if candidate.exists():
return candidate
# Fallback: any file with "vocal" in the name
for f in stems_dir.iterdir():
if f.is_file() and f.suffix in _AUDIO_EXTS:
if "vocal" in f.name.lower():
return f
return None
def _find_stem_file(project_dir: Path, stem_name: str) -> Optional[Path]:
"""Find a specific stem file in a project's Stems folder."""
stems_dir = project_dir / "Stems"
if not stems_dir.exists():
return None
# Exact match
for ext in _AUDIO_EXTS:
candidate = stems_dir / f"{stem_name}{ext}"
if candidate.exists():
return candidate
# Case-insensitive match
for f in stems_dir.iterdir():
if f.is_file() and f.suffix in _AUDIO_EXTS:
if f.stem.lower() == stem_name.lower():
return f
return None
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/models")
def get_available_models():
"""Get list of available Whisper models."""
return {
"models": list(WHISPER_MODELS.values()),
"default": DEFAULT_WHISPER_MODEL,
"whisper_available": is_whisper_available(),
"whisperx_available": is_whisperx_available(),
}
@router.get("/languages")
def get_supported_languages():
"""Get list of languages supported by Whisper."""
# Common languages for music/lyrics
languages = [
{"code": None, "name": "Auto-detect", "label": "Auto-detect language"},
{"code": "en", "name": "English", "label": "English"},
{"code": "zh", "name": "Chinese", "label": "Chinese (Mandarin)"},
{"code": "de", "name": "German", "label": "German"},
{"code": "es", "name": "Spanish", "label": "Spanish"},
{"code": "ru", "name": "Russian", "label": "Russian"},
{"code": "ko", "name": "Korean", "label": "Korean"},
{"code": "fr", "name": "French", "label": "French"},
{"code": "ja", "name": "Japanese", "label": "Japanese"},
{"code": "pt", "name": "Portuguese", "label": "Portuguese"},
{"code": "tr", "name": "Turkish", "label": "Turkish"},
{"code": "pl", "name": "Polish", "label": "Polish"},
{"code": "ca", "name": "Catalan", "label": "Catalan"},
{"code": "nl", "name": "Dutch", "label": "Dutch"},
{"code": "ar", "name": "Arabic", "label": "Arabic"},
{"code": "sv", "name": "Swedish", "label": "Swedish"},
{"code": "it", "name": "Italian", "label": "Italian"},
{"code": "id", "name": "Indonesian", "label": "Indonesian"},
{"code": "hi", "name": "Hindi", "label": "Hindi"},
{"code": "fi", "name": "Finnish", "label": "Finnish"},
{"code": "vi", "name": "Vietnamese", "label": "Vietnamese"},
{"code": "he", "name": "Hebrew", "label": "Hebrew"},
{"code": "uk", "name": "Ukrainian", "label": "Ukrainian"},
{"code": "el", "name": "Greek", "label": "Greek"},
{"code": "ms", "name": "Malay", "label": "Malay"},
{"code": "cs", "name": "Czech", "label": "Czech"},
{"code": "ro", "name": "Romanian", "label": "Romanian"},
{"code": "da", "name": "Danish", "label": "Danish"},
{"code": "hu", "name": "Hungarian", "label": "Hungarian"},
{"code": "ta", "name": "Tamil", "label": "Tamil"},
{"code": "no", "name": "Norwegian", "label": "Norwegian"},
{"code": "th", "name": "Thai", "label": "Thai"},
{"code": "ur", "name": "Urdu", "label": "Urdu"},
{"code": "hr", "name": "Croatian", "label": "Croatian"},
{"code": "bg", "name": "Bulgarian", "label": "Bulgarian"},
{"code": "lt", "name": "Lithuanian", "label": "Lithuanian"},
{"code": "la", "name": "Latin", "label": "Latin"},
{"code": "mi", "name": "Maori", "label": "Maori"},
{"code": "ml", "name": "Malayalam", "label": "Malayalam"},
{"code": "cy", "name": "Welsh", "label": "Welsh"},
{"code": "sk", "name": "Slovak", "label": "Slovak"},
{"code": "te", "name": "Telugu", "label": "Telugu"},
{"code": "fa", "name": "Persian", "label": "Persian"},
{"code": "lv", "name": "Latvian", "label": "Latvian"},
{"code": "bn", "name": "Bengali", "label": "Bengali"},
{"code": "sr", "name": "Serbian", "label": "Serbian"},
{"code": "az", "name": "Azerbaijani", "label": "Azerbaijani"},
{"code": "sl", "name": "Slovenian", "label": "Slovenian"},
{"code": "kn", "name": "Kannada", "label": "Kannada"},
{"code": "et", "name": "Estonian", "label": "Estonian"},
{"code": "mk", "name": "Macedonian", "label": "Macedonian"},
{"code": "br", "name": "Breton", "label": "Breton"},
{"code": "eu", "name": "Basque", "label": "Basque"},
{"code": "is", "name": "Icelandic", "label": "Icelandic"},
{"code": "hy", "name": "Armenian", "label": "Armenian"},
{"code": "ne", "name": "Nepali", "label": "Nepali"},
{"code": "mn", "name": "Mongolian", "label": "Mongolian"},
{"code": "bs", "name": "Bosnian", "label": "Bosnian"},
{"code": "kk", "name": "Kazakh", "label": "Kazakh"},
{"code": "sq", "name": "Albanian", "label": "Albanian"},
{"code": "sw", "name": "Swahili", "label": "Swahili"},
{"code": "gl", "name": "Galician", "label": "Galician"},
{"code": "mr", "name": "Marathi", "label": "Marathi"},
{"code": "pa", "name": "Punjabi", "label": "Punjabi"},
{"code": "si", "name": "Sinhala", "label": "Sinhala"},
{"code": "km", "name": "Khmer", "label": "Khmer"},
{"code": "sn", "name": "Shona", "label": "Shona"},
{"code": "yo", "name": "Yoruba", "label": "Yoruba"},
{"code": "so", "name": "Somali", "label": "Somali"},
{"code": "af", "name": "Afrikaans", "label": "Afrikaans"},
{"code": "oc", "name": "Occitan", "label": "Occitan"},
{"code": "ka", "name": "Georgian", "label": "Georgian"},
{"code": "be", "name": "Belarusian", "label": "Belarusian"},
{"code": "tg", "name": "Tajik", "label": "Tajik"},
{"code": "sd", "name": "Sindhi", "label": "Sindhi"},
{"code": "gu", "name": "Gujarati", "label": "Gujarati"},
{"code": "am", "name": "Amharic", "label": "Amharic"},
{"code": "yi", "name": "Yiddish", "label": "Yiddish"},
{"code": "lo", "name": "Lao", "label": "Lao"},
{"code": "uz", "name": "Uzbek", "label": "Uzbek"},
{"code": "fo", "name": "Faroese", "label": "Faroese"},
{"code": "ht", "name": "Haitian Creole", "label": "Haitian Creole"},
{"code": "ps", "name": "Pashto", "label": "Pashto"},
{"code": "tk", "name": "Turkmen", "label": "Turkmen"},
{"code": "nn", "name": "Nynorsk", "label": "Nynorsk"},
{"code": "mt", "name": "Maltese", "label": "Maltese"},
{"code": "sa", "name": "Sanskrit", "label": "Sanskrit"},
{"code": "lb", "name": "Luxembourgish", "label": "Luxembourgish"},
{"code": "my", "name": "Myanmar", "label": "Myanmar"},
{"code": "bo", "name": "Tibetan", "label": "Tibetan"},
{"code": "tl", "name": "Tagalog", "label": "Tagalog"},
{"code": "mg", "name": "Malagasy", "label": "Malagasy"},
{"code": "as", "name": "Assamese", "label": "Assamese"},
{"code": "tt", "name": "Tatar", "label": "Tatar"},
{"code": "haw", "name": "Hawaiian", "label": "Hawaiian"},
{"code": "ln", "name": "Lingala", "label": "Lingala"},
{"code": "ha", "name": "Hausa", "label": "Hausa"},
{"code": "ba", "name": "Bashkir", "label": "Bashkir"},
{"code": "jw", "name": "Javanese", "label": "Javanese"},
{"code": "su", "name": "Sundanese", "label": "Sundanese"},
]
return {"languages": languages}
@router.get("/formats")
def get_output_formats():
"""Get available output formats."""
return {
"formats": [
{"id": "lrc", "label": "LRC (line-level timestamps)", "extension": ".lrc"},
{"id": "elrc", "label": "ELRC (word-level timestamps)", "extension": ".elrc"},
{"id": "json", "label": "JSON (full data)", "extension": ".json"},
{"id": "txt", "label": "Plain Text", "extension": ".txt"},
{"id": "srt", "label": "SRT Subtitles", "extension": ".srt"},
{"id": "vtt", "label": "WebVTT Subtitles", "extension": ".vtt"},
]
}
@router.post("")
async def transcribe_lyrics(
file: UploadFile = File(...),
model: str = Form(DEFAULT_WHISPER_MODEL),
language: Optional[str] = Form(None),
task: str = Form("transcribe"),
output_formats: Optional[str] = Form("elrc,json,txt"),
initial_prompt: Optional[str] = Form(None),
temperature: float = Form(0.0),
):
"""
Transcribe lyrics from an uploaded audio file.
- **file**: Audio file (vocals stem recommended for best results)
- **model**: Whisper model to use (tiny, base, small, medium, large, large-v2, large-v3)
- **language**: ISO-639-1 language code (auto-detect if not provided)
- **task**: 'transcribe' for original language, 'translate' for English
- **output_formats**: Comma-separated list: lrc, elrc, json, txt, srt, vtt
- **initial_prompt**: Optional text to guide the model's style
- **temperature**: Sampling temperature (0.0 = greedy, higher = more random)
"""
if not is_whisper_available():
raise HTTPException(
status_code=503,
detail="Whisper is not installed. Install with: pip install openai-whisper"
)
if model not in WHISPER_MODELS:
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
if task not in VALID_TASKS:
raise HTTPException(status_code=400, detail=f"Invalid task: {task}")
# Parse output formats
formats = [f.strip() for f in (output_formats or "json").split(",")]
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
if invalid_formats:
raise HTTPException(
status_code=400,
detail=f"Invalid output formats: {invalid_formats}"
)
# Create job and save file
job_id = jm.create_job("lyric-transcribe")
upload_dir = UPLOAD_DIR / job_id
upload_dir.mkdir(parents=True, exist_ok=True)
audio_path = upload_dir / file.filename
with open(audio_path, "wb") as f:
shutil.copyfileobj(file.file, f)
# Submit job
jm.submit_gpu(
run_lyric_transcription,
job_id,
str(audio_path),
model=model,
language=language,
task=task,
output_formats=formats,
initial_prompt=initial_prompt,
temperature=temperature,
)
return {"job_id": job_id}
@router.post("/from-project")
async def transcribe_lyrics_from_project(
project_name: str = Form(...),
stem_name: Optional[str] = Form(None),
model: str = Form(DEFAULT_WHISPER_MODEL),
language: Optional[str] = Form(None),
task: str = Form("transcribe"),
output_formats: Optional[str] = Form("elrc,json,txt"),
initial_prompt: Optional[str] = Form(None),
temperature: float = Form(0.0),
):
"""
Transcribe lyrics from a project's vocal stem.
- **project_name**: Name of the project
- **stem_name**: Specific stem to use (default: auto-detect vocals)
- Other parameters same as /api/lyric-transcribe
"""
if not is_whisper_available():
raise HTTPException(
status_code=503,
detail="Whisper is not installed. Install with: pip install openai-whisper"
)
if model not in WHISPER_MODELS:
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
if task not in VALID_TASKS:
raise HTTPException(status_code=400, detail=f"Invalid task: {task}")
project_dir = PROJECTS_DIR / project_name
if not project_dir.exists():
raise HTTPException(status_code=404, detail=f"Project not found: {project_name}")
# Find audio file
if stem_name:
audio_file = _find_stem_file(project_dir, stem_name)
if not audio_file:
raise HTTPException(
status_code=404,
detail=f"Stem not found: {stem_name}"
)
else:
audio_file = _find_vocal_file(project_dir)
if not audio_file:
raise HTTPException(
status_code=404,
detail="No vocal file found in project. Please specify stem_name."
)
# Parse output formats
formats = [f.strip() for f in (output_formats or "json").split(",")]
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
if invalid_formats:
raise HTTPException(
status_code=400,
detail=f"Invalid output formats: {invalid_formats}"
)
# Create job
job_id = jm.create_job("lyric-transcribe")
# Output to project's LRC folder
output_dir = project_dir / "LRC"
output_dir.mkdir(parents=True, exist_ok=True)
# Submit job
jm.submit_gpu(
run_lyric_transcription,
job_id,
str(audio_file),
model=model,
language=language,
task=task,
output_formats=formats,
initial_prompt=initial_prompt,
temperature=temperature,
output_dir=str(output_dir),
)
return {"job_id": job_id, "source": audio_file.name}
@router.post("/with-diarization")
async def transcribe_lyrics_with_diarization(
file: UploadFile = File(...),
model: str = Form(DEFAULT_WHISPER_MODEL),
language: Optional[str] = Form(None),
num_speakers: Optional[int] = Form(None),
min_speakers: Optional[int] = Form(None),
max_speakers: Optional[int] = Form(None),
output_formats: Optional[str] = Form("json,txt"),
):
"""
Transcribe lyrics with speaker diarization (who spoke when).
Requires WhisperX to be installed: pip install whisperx
- **file**: Audio file
- **num_speakers**: Exact number of speakers (optional)
- **min_speakers**: Minimum number of speakers (optional)
- **max_speakers**: Maximum number of speakers (optional)
"""
if not is_whisperx_available():
raise HTTPException(
status_code=503,
detail="WhisperX is not installed. Install with: pip install whisperx"
)
if model not in WHISPER_MODELS:
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
# Parse output formats
formats = [f.strip() for f in (output_formats or "json").split(",")]
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
if invalid_formats:
raise HTTPException(
status_code=400,
detail=f"Invalid output formats: {invalid_formats}"
)
# Create job and save file
job_id = jm.create_job("lyric-transcribe-diarize")
upload_dir = UPLOAD_DIR / job_id
upload_dir.mkdir(parents=True, exist_ok=True)
audio_path = upload_dir / file.filename
with open(audio_path, "wb") as f:
shutil.copyfileobj(file.file, f)
# Submit job
jm.submit_gpu(
run_lyric_transcription_with_diarization,
job_id,
str(audio_path),
model=model,
language=language,
num_speakers=num_speakers,
min_speakers=min_speakers,
max_speakers=max_speakers,
output_formats=formats,
)
return {"job_id": job_id}
@router.post("/project/{project_id}")
def transcribe_project_lyrics(project_id: str, body: LyricTranscribeRequest):
"""
Transcribe lyrics from a specific stem in a project.
Similar to transcribe_lyrics_from_project but uses project_id and JSON body.
"""
if not is_whisper_available():
raise HTTPException(
status_code=503,
detail="Whisper is not installed. Install with: pip install openai-whisper"
)
if body.model not in WHISPER_MODELS:
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
if body.task not in VALID_TASKS:
raise HTTPException(status_code=400, detail=f"Invalid task: {body.task}")
# Get project
project = jm.get_project(project_id)
if not project:
raise HTTPException(status_code=404, detail=f"Project not found: {project_id}")
# Get separation job to find stems
sep_job = jm.get_job(project["sep_job_id"])
if not sep_job:
raise HTTPException(status_code=404, detail="Separation job not found")
# Find the stem file
stem_files = {
f["stem"]: f["path"]
for f in sep_job.get("files", [])
if f.get("type") == "stem" and f.get("stem")
}
if body.stem_name not in stem_files:
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
audio_path = stem_files[body.stem_name]
if not Path(audio_path).exists():
raise HTTPException(status_code=404, detail=f"Stem file not found: {audio_path}")
# Validate output formats
invalid_formats = set(body.output_formats) - VALID_OUTPUT_FORMATS
if invalid_formats:
raise HTTPException(
status_code=400,
detail=f"Invalid output formats: {invalid_formats}"
)
# Create job
job_id = jm.create_job("lyric-transcribe")
# Output to project directory
project_dir = PROJECTS_DIR / project["name"]
output_dir = project_dir / "LRC"
output_dir.mkdir(parents=True, exist_ok=True)
# Store job reference
from ..job_manager import set_project_lrc_job
set_project_lrc_job(project_id, body.stem_name, job_id)
# Submit job
jm.submit_gpu(
run_lyric_transcription,
job_id,
audio_path,
model=body.model,
language=body.language,
task=body.task,
output_formats=body.output_formats,
initial_prompt=body.initial_prompt,
temperature=body.temperature,
condition_on_previous_text=body.condition_on_previous_text,
word_timestamps=body.word_timestamps,
beam_size=body.beam_size,
output_dir=str(output_dir),
)
return {"job_id": job_id, "project_id": project_id, "stem": body.stem_name}