mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Why At a kjbox show on 2026-11-06, the make-it quick version for Sombr "My Body Isn't Ready (Orchestral Version Live From Sound City)" (job `f9764a9f`) rendered with only "(lyrics unavailable)" on screen. The singer heard the backing track with no lyrics. LRCLIB only has this song under the plain title. ## What - `fetch_lrclib_lyrics`: if the exact `/get` and `/search` find nothing, retry `/search` once with the title cleaned (`clean_title`: trailing `(…)`/`[…]` groups and `- Live …`/`- Remaster` style suffixes removed). - A synced hit from the retry keeps its timing only when its length is within **4 s** of the audio, because a different length usually means a different recording (studio vs live). Any other hit is used as plain lyrics (constant crawl). - Plain lyrics found under the exact title still win, and titles with nothing to clean make no extra request. - The quick render now searches with the job's `lyrics_artist`/`lyrics_title` overrides, else the title without a tempo label, the same names the review page uses. The displayed artist/title are unchanged. - Companion: kjbox v0.150.0 (nomadkaraoke/kjbox#314, deployed) lets the KJ add lyrics on the box when a draft still comes back without any. ## Testing - `backend/tests/test_quick_version.py`: 56 pass, including 17 new tests (cleaned-title retry: matching sync, wrong-length sync downgraded to plain, no duration, exact plain wins, failure paths, no retry for plain titles; `clean_title` cases; lyrics-search names incl. tempo label and overrides). - Live LRCLIB check: `fetch_lrclib_lyrics('Sombr', "My Body Isn't Ready (Orchestral Version Live From Sound City)", 224.5)` returns **synced, 37 lines**. Before this change it returned nothing. - Local CodeRabbit review: 0 findings. Full suite left to CI. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVWXLyvTmHNnqQu3rzA Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
429 lines
17 KiB
Python
429 lines
17 KiB
Python
"""Quick-version renderer — a scrolling-lyrics 480p karaoke video in seconds.
|
|
|
|
Ported from kjbox ``fastgen/fastgen.py`` (the local POC Andrew uses at shows).
|
|
Given an instrumental (+ optional vocals guide), artist and title, it:
|
|
|
|
1. fetches lyrics from LRCLIB (line-synced preferred, plain as fallback);
|
|
2. lays the lines out as one tall PIL-rendered PNG;
|
|
3. scrolls that PNG upward with an ffmpeg ``overlay`` whose y expression is a
|
|
piecewise-linear "sum of ramps" anchored so each synced line reaches the
|
|
reading row at its sung time (constant crawl when there is no timing);
|
|
4. muxes it with the instrumental (and the vocals quietly mixed back in as a
|
|
singer guide) in a single ffmpeg pass.
|
|
|
|
Deliberately primitive: no word-level highlighting and no review — it exists
|
|
so a kjbox singer can sing a song minutes after asking, while the full gen job
|
|
keeps working toward the proper NOMAD version.
|
|
|
|
PIL rather than ffmpeg ``drawtext``: ffmpeg 8's always-on harfbuzz shaping
|
|
renders a ``.notdef`` box for every newline in a multi-line textfile.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import textwrap
|
|
from dataclasses import dataclass, field
|
|
from typing import List, Optional, Tuple
|
|
|
|
import requests
|
|
|
|
from karaoke_gen.lyrics_transcriber.lyrics import lrclib_client
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
LRCLIB_BASE = "https://lrclib.net/api"
|
|
USER_AGENT = "nomadkaraoke-quick-version/1.0 (https://nomadkaraoke.com)"
|
|
|
|
_FONT_PATH = os.path.join(
|
|
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))),
|
|
"karaoke_gen", "resources", "Montserrat-Bold.ttf",
|
|
)
|
|
|
|
# (text, anchor_seconds | None). Only the first visual row of a lyric line
|
|
# carries its anchor.
|
|
TimedLine = Tuple[str, Optional[float]]
|
|
|
|
_LRC_RE = re.compile(r"\[(\d+):(\d+(?:\.\d+)?)\]")
|
|
|
|
# A synced LRCLIB result found under a *cleaned* title is often a different
|
|
# recording (studio vs "Live From ..."), so its timing is only trusted when the
|
|
# lengths agree this closely; otherwise its words scroll at a constant pace.
|
|
CLEAN_TITLE_SYNC_TOLERANCE_S = 4.0
|
|
|
|
# Trailing "(...)" / "[...]" groups and " - Live at ..." style suffixes that stop
|
|
# LRCLIB matching the plain song (kjbox quick_lyrics.clean_title mirrors this).
|
|
_TRAILING_GROUP_RE = re.compile(r"\s*[\(\[][^\(\)\[\]]*[\)\]]\s*$")
|
|
_DASH_SUFFIX_RE = re.compile(
|
|
r"\s+-\s+.*\b(live|version|remaster(ed)?|mix|edit|acoustic|orchestral|session|demo|mono|stereo)\b.*$",
|
|
re.IGNORECASE)
|
|
|
|
|
|
def clean_title(title: str) -> str:
|
|
"""'Song (Live From X) [HD]' / 'Song - 2011 Remaster' → 'Song' (never empty)."""
|
|
t = (title or "").strip()
|
|
while True:
|
|
stripped = _TRAILING_GROUP_RE.sub("", t)
|
|
if stripped == t or not stripped.strip():
|
|
break
|
|
t = stripped
|
|
t = _DASH_SUFFIX_RE.sub("", t)
|
|
return t.strip() or (title or "").strip()
|
|
|
|
|
|
@dataclass
|
|
class Lyrics:
|
|
kind: str # "synced" | "plain"
|
|
plain: str
|
|
timed: Optional[List[Tuple[float, str]]] = None
|
|
|
|
|
|
@dataclass
|
|
class RenderSettings:
|
|
height: int = 480
|
|
fps: int = 24
|
|
wrap: int = 34
|
|
reading_frac: float = 0.42
|
|
vocals_level: float = 0.3
|
|
gap_threshold: float = 5.0
|
|
gap_lines: int = 1
|
|
font_path: str = _FONT_PATH
|
|
|
|
|
|
@dataclass
|
|
class RenderResult:
|
|
output_path: str
|
|
lyrics_tier: str # "synced" | "constant" | "none"
|
|
line_count: int
|
|
duration_seconds: float
|
|
extra: dict = field(default_factory=dict)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Lyrics
|
|
# --------------------------------------------------------------------------- #
|
|
def _strip_lrc_timestamps(synced: str) -> str:
|
|
return "\n".join(_LRC_RE.sub("", raw).strip() for raw in synced.splitlines()).strip()
|
|
|
|
|
|
def parse_synced(synced: str) -> List[Tuple[float, str]]:
|
|
"""Parse ``[mm:ss.xx] line`` LRC into sorted (start_seconds, text) pairs.
|
|
|
|
A line may carry several timestamps (``[00:10.00][01:10.00] chorus``); each
|
|
becomes its own entry.
|
|
"""
|
|
out: List[Tuple[float, str]] = []
|
|
for raw in synced.splitlines():
|
|
stamps = _LRC_RE.findall(raw)
|
|
if not stamps:
|
|
continue
|
|
text = _LRC_RE.sub("", raw).strip()
|
|
for mins, secs in stamps:
|
|
out.append((int(mins) * 60 + float(secs), text))
|
|
out.sort(key=lambda p: p[0])
|
|
return out
|
|
|
|
|
|
def lyrics_from_lrclib_payload(data: dict) -> Optional[Lyrics]:
|
|
synced = data.get("syncedLyrics")
|
|
if synced and synced.strip():
|
|
timed = parse_synced(synced)
|
|
if timed:
|
|
return Lyrics("synced", _strip_lrc_timestamps(synced), timed)
|
|
plain = data.get("plainLyrics")
|
|
if plain and plain.strip():
|
|
return Lyrics("plain", plain)
|
|
return None
|
|
|
|
|
|
def fetch_lrclib_lyrics(
|
|
artist: str, title: str, duration: Optional[float], timeout: float = 10.0
|
|
) -> Optional[Lyrics]:
|
|
"""Exact ``/get`` (with duration when known), then ``/search`` preferring synced."""
|
|
session = requests.Session()
|
|
session.headers["User-Agent"] = USER_AGENT
|
|
|
|
params = {"artist_name": artist, "track_name": title}
|
|
if duration:
|
|
params["duration"] = int(round(duration))
|
|
try:
|
|
r = lrclib_client.get(f"{LRCLIB_BASE}/get", params, user_agent=USER_AGENT, timeout=timeout, session=session, retry=False)
|
|
if r.status_code == 200:
|
|
got = lyrics_from_lrclib_payload(r.json())
|
|
if got and got.kind == "synced":
|
|
return got
|
|
exact_plain = got
|
|
else:
|
|
exact_plain = None
|
|
except (requests.RequestException, ValueError) as exc:
|
|
logger.warning(f"LRCLIB get failed: {exc}")
|
|
exact_plain = None
|
|
|
|
try:
|
|
r = lrclib_client.get(f"{LRCLIB_BASE}/search", {"q": f"{artist} {title}"}, user_agent=USER_AGENT, timeout=timeout, session=session, retry=False)
|
|
if r.status_code == 200:
|
|
hits = r.json() or []
|
|
for hit in sorted(hits, key=lambda h: 0 if h.get("syncedLyrics") else 1):
|
|
got = lyrics_from_lrclib_payload(hit)
|
|
if got and got.kind == "synced":
|
|
return got
|
|
if got and exact_plain is None:
|
|
exact_plain = got
|
|
except (requests.RequestException, ValueError) as exc:
|
|
logger.warning(f"LRCLIB search failed: {exc}")
|
|
|
|
# Decorated titles ("My Song (Orchestral Version Live From X)") rarely match
|
|
# on LRCLIB: retry once with the plain title. Exact-title plain lyrics still
|
|
# win over this looser match.
|
|
cleaned = clean_title(title)
|
|
if exact_plain is None and cleaned and cleaned.casefold() != (title or "").strip().casefold():
|
|
exact_plain = _search_cleaned_title(artist, cleaned, duration, timeout, session)
|
|
|
|
return exact_plain
|
|
|
|
|
|
def _search_cleaned_title(artist: str, cleaned: str, duration: Optional[float], timeout: float,
|
|
session: requests.Session) -> Optional[Lyrics]:
|
|
"""Search LRCLIB with a cleaned title. Synced timing is kept only for a hit
|
|
whose length matches the audio; any other hit is used as plain lyrics."""
|
|
try:
|
|
r = lrclib_client.get(f"{LRCLIB_BASE}/search", {"q": f"{artist} {cleaned}"}, user_agent=USER_AGENT,
|
|
timeout=timeout, session=session, retry=False)
|
|
if r.status_code != 200:
|
|
return None
|
|
hits = [h for h in (r.json() or []) if isinstance(h, dict)]
|
|
except (requests.RequestException, ValueError) as exc:
|
|
logger.warning(f"LRCLIB cleaned-title search failed: {exc}")
|
|
return None
|
|
fallback: Optional[Lyrics] = None
|
|
for hit in sorted(hits, key=lambda h: 0 if h.get("syncedLyrics") else 1):
|
|
got = lyrics_from_lrclib_payload(hit)
|
|
if got is None:
|
|
continue
|
|
hit_duration = hit.get("duration")
|
|
if (got.kind == "synced" and duration and isinstance(hit_duration, (int, float))
|
|
and abs(hit_duration - duration) <= CLEAN_TITLE_SYNC_TOLERANCE_S):
|
|
logger.info(f"Quick lyrics: synced match via cleaned title {cleaned!r}")
|
|
return got
|
|
if fallback is None and got.plain.strip():
|
|
fallback = Lyrics("plain", got.plain)
|
|
if fallback is not None:
|
|
logger.info(f"Quick lyrics: plain lyrics via cleaned title {cleaned!r} (no length-matched sync)")
|
|
return fallback
|
|
|
|
|
|
def resolve_timed_lines(lyrics: Optional[Lyrics]) -> Tuple[List[TimedLine], str]:
|
|
"""Lyrics → (line, anchor|None) pairs + tier label (synced / constant / none)."""
|
|
if lyrics is None:
|
|
return [], "none"
|
|
if lyrics.kind == "synced" and lyrics.timed:
|
|
return [(text, t) for t, text in lyrics.timed if text.strip()], "synced"
|
|
return [(raw.strip(), None) for raw in lyrics.plain.splitlines()], "constant"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Layout
|
|
# --------------------------------------------------------------------------- #
|
|
def insert_instrumental_gaps(timed_lines: List[TimedLine], threshold: float, gap_lines: int = 1) -> List[TimedLine]:
|
|
"""Blank spacer rows where consecutive anchored lines are > threshold apart,
|
|
so the crawl visibly opens up during an instrumental break."""
|
|
if threshold <= 0:
|
|
return list(timed_lines)
|
|
out: List[TimedLine] = []
|
|
last_t: Optional[float] = None
|
|
for text, anchor in timed_lines:
|
|
if anchor is not None:
|
|
if last_t is not None and anchor - last_t > threshold:
|
|
out.extend([("", None)] * gap_lines)
|
|
last_t = anchor
|
|
out.append((text, anchor))
|
|
return out
|
|
|
|
|
|
def build_visual_lines(artist: str, title: str, timed_lines: List[TimedLine], wrap: int) -> List[TimedLine]:
|
|
header: List[TimedLine] = [("", None), ("", None), (artist.upper(), None), (title.upper(), None),
|
|
("", None), ("", None)]
|
|
body: List[TimedLine] = []
|
|
for text, anchor in timed_lines:
|
|
text = text.strip()
|
|
if not text:
|
|
body.append(("", None))
|
|
continue
|
|
for i, row in enumerate(textwrap.wrap(text, width=wrap) or [""]):
|
|
body.append((row, anchor if i == 0 else None))
|
|
return header + body + [("", None)] * 3
|
|
|
|
|
|
def render_crawl_png(
|
|
lines: List[TimedLine], width: int, fontsize: int, font_path: str, workdir: str
|
|
) -> Tuple[str, int, List[float]]:
|
|
"""Rasterise the whole crawl to one tall PNG → (path, height, row centres)."""
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
|
|
font = ImageFont.truetype(font_path, fontsize)
|
|
ascent, descent = font.getmetrics()
|
|
line_advance = ascent + descent + round(fontsize * 0.35)
|
|
stroke = max(2, fontsize // 16)
|
|
pad = fontsize
|
|
|
|
height = pad * 2 + line_advance * len(lines)
|
|
img = Image.new("RGB", (width, height), (0, 0, 0))
|
|
draw = ImageDraw.Draw(img)
|
|
|
|
centers: List[float] = []
|
|
y = pad
|
|
for text, _anchor in lines:
|
|
if text:
|
|
line_w = draw.textlength(text, font=font)
|
|
draw.text(((width - line_w) / 2, y), text, font=font, fill=(255, 232, 31),
|
|
stroke_width=stroke, stroke_fill=(0, 0, 0))
|
|
centers.append(y + line_advance / 2)
|
|
y += line_advance
|
|
|
|
png_path = os.path.join(workdir, "crawl.png")
|
|
img.save(png_path)
|
|
return png_path, height, centers
|
|
|
|
|
|
def build_scroll_y_expr(
|
|
lines: List[TimedLine], centers: List[float], duration: float,
|
|
frame_h: int, img_h: int, reading_frac: float,
|
|
) -> Tuple[str, bool]:
|
|
"""ffmpeg ``overlay`` y expression → (expr, is_time_anchored).
|
|
|
|
y is the PNG's top edge relative to the frame top; each anchored line's
|
|
centre ``c_i`` should sit on the reading row ``R`` at ``t_i`` (y = R - c_i).
|
|
Piecewise-linear between anchors, written as a flat sum of ramps
|
|
``Y0 + M0*(t-T0) + Σ (Mk-M[k-1])*max(0,t-Tk)`` with commas escaped for the
|
|
filtergraph parser.
|
|
"""
|
|
reading_row = reading_frac * frame_h
|
|
pts = [(0.0, float(frame_h))]
|
|
for (_text, anchor), c in zip(lines, centers):
|
|
if anchor is not None and 0.0 < anchor < duration:
|
|
pts.append((float(anchor), reading_row - c))
|
|
|
|
timed = len(pts) > 1
|
|
if timed:
|
|
pts.append((duration, pts[-1][1]))
|
|
else:
|
|
pts.append((duration, -float(img_h)))
|
|
|
|
clean = [pts[0]]
|
|
for t, y in pts[1:]:
|
|
if t > clean[-1][0] + 1e-3:
|
|
clean.append((t, y))
|
|
pts = clean
|
|
if len(pts) < 2: # degenerate (zero-length audio) — park the crawl off-screen
|
|
return f"{float(frame_h):.2f}", False
|
|
|
|
slopes = [(pts[i + 1][1] - pts[i][1]) / (pts[i + 1][0] - pts[i][0]) for i in range(len(pts) - 1)]
|
|
terms = [f"{pts[0][1]:.2f}", f"({slopes[0]:.5f})*(t-{pts[0][0]:.3f})"]
|
|
for k in range(1, len(slopes)):
|
|
dm = slopes[k] - slopes[k - 1]
|
|
if abs(dm) < 1e-6:
|
|
continue
|
|
terms.append(f"({dm:.5f})*max(0\\,t-{pts[k][0]:.3f})")
|
|
return "+".join(terms), timed
|
|
|
|
|
|
def build_ffmpeg_cmd(
|
|
instrumental: str, vocals: Optional[str], vocals_level: float,
|
|
crawl_png: str, y_expr: str, out_path: str, width: int, height: int, fps: int,
|
|
duration: float,
|
|
) -> List[str]:
|
|
"""Every input and the output are explicitly bounded to ``duration``.
|
|
|
|
``-shortest`` alone is NOT enough: ffmpeg 4.4 (the GPU image's Ubuntu 22.04
|
|
package) never ends when an ``amix`` output is paired with the infinite
|
|
lavfi colour + looped PNG video — a 20 s song rendered 540 s+ of video until
|
|
the timeout (prod job 405eed71, 2026-09-28). ffmpeg 8 happens to stop."""
|
|
dur = f"{max(duration, 0.1):.3f}"
|
|
overlay = f"[0:v][1:v]overlay=x=(W-w)/2:y={y_expr}[v]"
|
|
inputs = [
|
|
"-f", "lavfi", "-i", f"color=c=black:s={width}x{height}:r={fps}:d={dur}",
|
|
"-loop", "1", "-framerate", str(fps), "-t", dur, "-i", crawl_png,
|
|
"-i", instrumental,
|
|
]
|
|
if vocals and vocals_level > 0:
|
|
# Faint lead vocal as a singer guide (there is no word highlighting).
|
|
inputs += ["-i", vocals]
|
|
audio = (f"[2:a]volume=1[gi];[3:a]volume={vocals_level:.3f}[gv];"
|
|
f"[gi][gv]amix=inputs=2:normalize=0:duration=first[a]")
|
|
filt, amap = overlay + ";" + audio, ["-map", "[a]"]
|
|
else:
|
|
filt, amap = overlay, ["-map", "2:a"]
|
|
return [
|
|
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *inputs,
|
|
"-filter_complex", filt,
|
|
"-map", "[v]", *amap,
|
|
"-c:v", "libx264", "-preset", "veryfast", "-crf", "26", "-pix_fmt", "yuv420p",
|
|
"-c:a", "aac", "-b:a", "192k",
|
|
"-movflags", "+faststart",
|
|
"-t", dur,
|
|
"-shortest",
|
|
out_path,
|
|
]
|
|
|
|
|
|
def probe_duration(audio_path: str) -> float:
|
|
out = subprocess.run(
|
|
["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=nk=1:nw=1", audio_path],
|
|
capture_output=True, text=True, check=True, timeout=60,
|
|
)
|
|
return float(out.stdout.strip())
|
|
|
|
|
|
def render_quick_video(
|
|
*,
|
|
instrumental_path: str,
|
|
vocals_path: Optional[str],
|
|
artist: str,
|
|
title: str,
|
|
out_path: str,
|
|
workdir: str,
|
|
lyrics: Optional[Lyrics] = None,
|
|
fetch_lyrics: bool = True,
|
|
lyrics_artist: Optional[str] = None,
|
|
lyrics_title: Optional[str] = None,
|
|
settings: Optional[RenderSettings] = None,
|
|
ffmpeg_timeout: float = 600.0,
|
|
) -> RenderResult:
|
|
"""Render the quick video. ``lyrics`` overrides the LRCLIB fetch;
|
|
``lyrics_artist``/``lyrics_title`` (the job's lyrics-search overrides) are
|
|
what LRCLIB is searched with, while ``artist``/``title`` are displayed."""
|
|
s = settings or RenderSettings()
|
|
duration = probe_duration(instrumental_path)
|
|
|
|
if lyrics is None and fetch_lyrics:
|
|
lyrics = fetch_lrclib_lyrics(lyrics_artist or artist, lyrics_title or title, duration)
|
|
|
|
timed_lines, tier = resolve_timed_lines(lyrics)
|
|
if tier == "synced":
|
|
timed_lines = insert_instrumental_gaps(timed_lines, s.gap_threshold, s.gap_lines)
|
|
if tier == "none":
|
|
timed_lines = [("(lyrics unavailable)", None)]
|
|
|
|
width = round(s.height * 16 / 9)
|
|
width += width % 2
|
|
fontsize = max(18, round(s.height * 0.07))
|
|
lines = build_visual_lines(artist, title, timed_lines, s.wrap)
|
|
crawl_png, img_h, centers = render_crawl_png(lines, width, fontsize, s.font_path, workdir)
|
|
y_expr, _ = build_scroll_y_expr(lines, centers, duration, s.height, img_h, s.reading_frac)
|
|
|
|
cmd = build_ffmpeg_cmd(instrumental_path, vocals_path, s.vocals_level, crawl_png, y_expr,
|
|
out_path, width, s.height, s.fps, duration)
|
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=ffmpeg_timeout)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"ffmpeg failed ({proc.returncode}): {proc.stderr[-2000:]}")
|
|
|
|
return RenderResult(
|
|
output_path=out_path,
|
|
lyrics_tier=tier,
|
|
line_count=sum(1 for t, _ in timed_lines if t.strip()),
|
|
duration_seconds=duration,
|
|
)
|