Switch to WhisperX engine for alignment. Design change and fullscreen for lyrics preview.

This commit is contained in:
translator committed 2026-03-05 14:37:42 +09:00
1 parent 8d16557e6a
commit ce082059e0
10 files changed
+981 -277

No files matched your search

+11 -4
View File
@@ -44,8 +44,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libespeak-ng-dev \
&& rm -rf /var/lib/apt/lists/*
# Bootstrap pip for Python 3.11
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
# Bootstrap pip for Python 3.11, then install uv (faster resolver, no depth limit)
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \
&& python3.11 -m pip install --no-cache-dir uv
# ---------- Node.js 20.x ----------
# Required for `vite build --watch` which runs at container startup via start.sh.
@@ -57,15 +58,21 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
WORKDIR /app
COPY backend/requirements.txt /app/backend/requirements.txt
RUN python3.11 -m pip install --no-cache-dir -r /app/backend/requirements.txt
RUN python3.11 -m uv pip install --system --no-cache -r /app/backend/requirements.txt
# Install CUDA-enabled PyTorch (CUDA 12.4 wheels; self-contained, does not need system CUDA 12.4)
# torch 2.6+ is required — earlier versions have a serious torch.load vulnerability (even with
# weights_only=True). cu121 wheels were discontinued after torch 2.4; use cu124 going forward.
RUN python3.11 -m pip install --no-cache-dir \
RUN python3.11 -m uv pip install --system --no-cache \
torch==2.6.0 torchaudio==2.6.0 torchvision==0.21.0 \
--index-url https://download.pytorch.org/whl/cu124
# Install basic-pitch without its tensorflow dependency.
# basic-pitch can run via onnxruntime (already installed above); installing with
# --no-deps skips the tensorflow pull that has no cp311 wheels and conflicts with
# numpy>=2.1.0 required by whisperx.
RUN python3.11 -m uv pip install --system --no-cache --no-deps basic-pitch
# Note: Model weights are downloaded on-demand at runtime on first use.
# This keeps the Docker image smaller and avoids build-time network dependencies.
+9 -11
View File
@@ -9,7 +9,7 @@ sqlalchemy==2.0.36
# Audio utilities
pydub==0.25.1
soundfile==0.12.1
soundfile>=0.13.1
librosa==0.10.2
mutagen==1.47.0
@@ -18,7 +18,7 @@ music21==9.3.0
# ML — PyTorch (CUDA 12.1 build; installed via --index-url in Dockerfile)
# torch and torchaudio are installed separately in the Dockerfile with the CUDA wheel index.
numpy==1.26.4
numpy>=2.1.0
# Demucs stem separation
demucs==4.0.1
@@ -27,7 +27,7 @@ demucs==4.0.1
bs-roformer>=1.0.0
omegaconf==2.3.0
pyyaml>=6.0
einops==0.8.0
einops>=0.8.1
rotary-embedding-torch==0.6.5
beartype==0.18.5
tqdm>=4.66
@@ -49,7 +49,11 @@ requests>=2.31.0
cryptography>=42.0
# MIDI transcription via basic-pitch
basic-pitch>=0.3.0
# Installed separately in Dockerfile with --no-deps to avoid TF/numpy conflict.
# Runtime inference uses onnxruntime (above) instead of TensorFlow.
# pretty_midi is basic-pitch's MIDI output library; must be installed separately
# since it's normally pulled in as a dep of basic-pitch.
pretty_midi>=0.2.10
# Syllable hyphenation (pure-Python, for per-syllable ELRC timing)
pyphen>=0.14.0
@@ -63,13 +67,7 @@ accelerate>=0.19.0
# audio-separator (required by SCNet, MedleyVox, Apollo, Aspiration, Bandit separators)
audio-separator[gpu]>=0.28.0
# Lyric transcription via OpenAI Whisper (local, offline)
# Provides word-level timestamps for lyrics transcription.
# Models are downloaded automatically on first use.
openai-whisper>=20231117
# Optional: WhisperX for faster transcription + phoneme-level word alignment
# Uncomment to enable the WhisperX engine option in AI Transcribe:
# Lyric transcription via WhisperX (faster-whisper backend + phoneme-level word alignment)
whisperx>=3.8.1
# Phoneme-level syllable alignment (requires espeak-ng system package)
+63 -6
View File
@@ -8,12 +8,13 @@ import shutil
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, HTTPException, Response
from pydantic import BaseModel
from .. import job_manager as jm
from ..config import PROJECTS_DIR, UPLOAD_DIR, TRANSCRIBE_MODES, DEFAULT_TRANSCRIBE_MODE, STATIC_OPTIONS
from ..services.project_metadata import write_project_metadata, read_project_metadata
from ..services.metadata_util import _read_cover_art, read_metadata
from ..services.transcriber_service import run_transcription
from ..services.abc_service import run_abc_conversion
from ..services.mix_service import run_mix
@@ -77,6 +78,9 @@ class LyricTranscribeRequest(BaseModel):
diarize: bool = False # run speaker diarization (WhisperX only)
lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance'
phoneme_align: bool = False # use phonemizer + phoneme model for syllable-level timestamps
lyrics_source: str = "transcribe" # 'transcribe' | 'auto' | 'tag' | 'file' — when not 'transcribe',
# skip Whisper and force-align existing lyrics instead
existing_lyrics: Optional[str] = None # lyrics text (plain or LRC) to align; overrides lyrics_source
# ---------------------------------------------------------------------------
@@ -162,16 +166,52 @@ def get_project_detail(project_id: str):
stem: jm.get_job(jid)
for stem, jid in (project.get("lrc_jobs") or {}).items()
}
# Read artist from source audio metadata
source_artist = ""
sep_job_data = jm.get_job(project["sep_job_id"]) or {}
source_file = next(
(f for f in sep_job_data.get("files", []) if f.get("type") == "source"),
None,
)
if source_file:
src_path = source_file.get("path", "")
if src_path and Path(src_path).exists():
try:
meta = read_metadata(src_path)
source_artist = meta.get("artist") or meta.get("albumartist") or ""
except Exception:
pass
return {
**project,
"sep_job": jm.get_job(project["sep_job_id"]),
"trans_jobs": trans_jobs_expanded,
"abc_jobs": abc_jobs_expanded,
"lrc_jobs": lrc_jobs_expanded,
"mix_job": _get_job_or_none(project["mix_job_id"]),
"sep_job": jm.get_job(project["sep_job_id"]),
"trans_jobs": trans_jobs_expanded,
"abc_jobs": abc_jobs_expanded,
"lrc_jobs": lrc_jobs_expanded,
"mix_job": _get_job_or_none(project["mix_job_id"]),
"source_artist": source_artist,
}
@router.get("/{project_id}/cover-art")
def get_cover_art(project_id: str):
"""Return cover art image bytes from the source audio file, or 404 if none."""
project = _get_project_or_404(project_id)
sep_job_data = jm.get_job(project["sep_job_id"]) or {}
source_file = next(
(f for f in sep_job_data.get("files", []) if f.get("type") == "source"),
None,
)
if not source_file:
raise HTTPException(status_code=404, detail="No source file")
src_path = source_file.get("path", "")
if not src_path or not Path(src_path).exists():
raise HTTPException(status_code=404, detail="Source file not found")
data, mime = _read_cover_art(src_path)
if not data:
raise HTTPException(status_code=404, detail="No cover art")
return Response(content=data, media_type=mime or "image/jpeg")
@router.delete("/{project_id}")
def delete_project(project_id: str):
"""Remove the project record and all associated files from disk."""
@@ -418,6 +458,22 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
job_id = jm.create_job("lyric-transcribe")
# Resolve existing lyrics: caller may provide them directly, or request
# auto-extraction from the audio file's metadata tags or companion LRC.
resolved_lyrics: Optional[str] = body.existing_lyrics
if resolved_lyrics is None and body.lyrics_source != "transcribe":
from ..services.metadata_util import extract_lyrics_from_file
lyrics_data = extract_lyrics_from_file(stem_path)
if lyrics_data.get("plain"):
# Use raw (with timestamps if synced) so the service can parse LRC windows.
resolved_lyrics = lyrics_data.get("raw") or lyrics_data["plain"]
else:
raise HTTPException(
status_code=422,
detail=f"lyrics_source='{body.lyrics_source}' requested but no lyrics found "
"in file tags or companion LRC.",
)
# Store job reference keyed by stem name; same key for both engines so the
# frontend always shows the most-recent AI transcription for that stem.
set_project_lrc_job(project_id, f"{body.stem_name}_whisper", job_id)
@@ -438,6 +494,7 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
diarize=body.diarize,
power_preset=body.lrc_power_preset,
phoneme_align=body.phoneme_align,
existing_lyrics=resolved_lyrics,
)
return {"job_id": job_id, "output_dir": str(output_dir)}
+84 -25
View File
@@ -985,6 +985,38 @@ _WHISPERX_POWER_PRESETS: dict = {
}
def _lyrics_to_whisperx_segments(lyrics_text: str, audio_duration: float) -> list:
"""Convert existing lyrics (plain text or LRC) to WhisperX segment format.
If the lyrics contain LRC timestamps they are used as segment boundaries so
alignment is performed within each known line window. Plain text lines are
distributed evenly across the audio duration as an initial estimate.
"""
from .metadata_util import is_lrc_synced, parse_lrc
if is_lrc_synced(lyrics_text):
lines = parse_lrc(lyrics_text) # [(timestamp, text), ...]
segments = []
for i, (ts, text) in enumerate(lines):
end = lines[i + 1][0] if i + 1 < len(lines) else audio_duration
segments.append({"text": text, "start": ts, "end": end, "words": []})
return segments
else:
plain_lines = [ln.strip() for ln in lyrics_text.splitlines() if ln.strip()]
if not plain_lines:
return []
step = audio_duration / len(plain_lines)
return [
{
"text": text,
"start": round(i * step, 3),
"end": round((i + 1) * step, 3),
"words": [],
}
for i, text in enumerate(plain_lines)
]
def run_lyric_transcription(
job_id: str,
audio_path: str,
@@ -1002,6 +1034,7 @@ def run_lyric_transcription(
diarize: bool = False,
power_preset: str = "standard",
phoneme_align: bool = False,
existing_lyrics: Optional[str] = None,
):
"""
Background task: transcribe vocals/lyrics from audio.
@@ -1018,6 +1051,10 @@ def run_lyric_transcription(
output_dir: Custom output directory
engine: 'whisperx' (faster-whisper + phoneme alignment)
diarize: Run speaker diarization after alignment (WhisperX only)
existing_lyrics: When set, skip Whisper transcription and force-align
these lyrics instead. Accepts plain text (one lyric line per line)
or LRC format (line-level timestamps are used as alignment windows).
Only effective when engine='whisperx'.
"""
if output_formats is None:
output_formats = ["elrc", "json", "txt"]
@@ -1079,35 +1116,57 @@ def run_lyric_transcription(
"vad_offset": 0.200,
}
wx_model = whisperx.load_model(
model, device,
compute_type=compute_type,
language=language or None,
asr_options=_asr_opts,
vad_options=_vad_opts,
)
import gc
audio_arr = whisperx.load_audio(audio_path)
duration = float(len(audio_arr)) / 16000.0
jm.update_job(job_id, progress=f"Pass 1: Transcribing with WhisperX ({model}, {power_preset})...")
wx_result = wx_model.transcribe(audio_arr, batch_size=batch_size)
detected_lang = wx_result.get("language") or language or "unknown"
if existing_lyrics:
# ── Forced-alignment path: skip Whisper, align provided lyrics ──
jm.update_job(job_id, progress="Preparing existing lyrics for WhisperX alignment...")
detected_lang = language or "en"
wx_result = {
"segments": _lyrics_to_whisperx_segments(existing_lyrics, duration),
"language": detected_lang,
}
if not wx_result["segments"]:
jm.update_job(
job_id,
status=jm.JobStatus.ERROR,
error="No lyric lines found in the provided text.",
progress="Error.",
)
return
_result_model_tag = "whisperx-align"
else:
# ── Normal transcription path ──────────────────────────────────
wx_model = whisperx.load_model(
model, device,
compute_type=compute_type,
language=language or None,
asr_options=_asr_opts,
vad_options=_vad_opts,
)
# Pass 2: re-transcribe any gaps > 2 s that VAD may have missed
# (e.g. quieter verse sections after long instrumentals).
wx_result["segments"] = _fill_transcript_gaps(
wx_model, audio_arr, wx_result.get("segments", []),
batch_size, job_id, min_gap=2.0,
)
jm.update_job(job_id, progress=f"Pass 1: Transcribing with WhisperX ({model}, {power_preset})...")
wx_result = wx_model.transcribe(audio_arr, batch_size=batch_size)
detected_lang = wx_result.get("language") or language or "unknown"
# Free the Whisper model from VRAM before loading the alignment model.
# large-v2/v3 consume 3–6 GB; keeping both models resident simultaneously
# causes OOM on ≤8 GB cards.
import gc
del wx_model
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
# Pass 2: re-transcribe any gaps > 2 s that VAD may have missed
# (e.g. quieter verse sections after long instrumentals).
wx_result["segments"] = _fill_transcript_gaps(
wx_model, audio_arr, wx_result.get("segments", []),
batch_size, job_id, min_gap=2.0,
)
# Free the Whisper model from VRAM before loading the alignment model.
# large-v2/v3 consume 3–6 GB; keeping both models resident simultaneously
# causes OOM on ≤8 GB cards.
del wx_model
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
_result_model_tag = f"whisperx-{model}"
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
@@ -1238,7 +1297,7 @@ def run_lyric_transcription(
segments=segments,
language=detected_lang,
duration=duration,
model=f"whisperx-{model}",
model=_result_model_tag,
)
# ── Release remaining GPU memory ──────────────────────────────────────
+2 -2
View File
@@ -19,8 +19,8 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="StemEngine" />
<link rel="apple-touch-icon" href="/icon.svg" />
<script type="module" crossorigin src="/assets/index-3g7RCqSu.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-C4A4XGsw.css">
<script type="module" crossorigin src="/assets/index-Dq5PnhTp.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DBmWYVxf.css">
</head>
<body>
<div id="root"></div>
+51 -1
View File
@@ -12,7 +12,8 @@
"@tonejs/midi": "^2.0.28",
"lucide-react": "^0.294.0",
"react": "^18.2.0",
"react-dom": "^18.2.0"
"react-dom": "^18.2.0",
"tone": "^14.7.0"
},
"devDependencies": {
"@types/react": "^18.2.37",
@@ -255,6 +256,15 @@
"@babel/core": "^7.0.0-0"
}
},
"node_modules/@babel/runtime": {
"version": "7.28.6",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz",
"integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==",
"license": "MIT",
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/template": {
"version": "7.28.6",
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz",
@@ -1234,6 +1244,19 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz",
"integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA=="
},
"node_modules/automation-events": {
"version": "7.1.15",
"resolved": "https://registry.npmjs.org/automation-events/-/automation-events-7.1.15.tgz",
"integrity": "sha512-NsHJlve3twcgs8IyP4iEYph7Fzpnh6klN7G5LahwvypakBjFbsiGHJxrqTmeHKREdu/Tx6oZboqNI0tD4MnFlA==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.28.6",
"tslib": "^2.8.1"
},
"engines": {
"node": ">=18.2.0"
}
},
"node_modules/autoprefixer": {
"version": "10.4.24",
"resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.24.tgz",
@@ -2267,6 +2290,17 @@
"node": ">=0.10.0"
}
},
"node_modules/standardized-audio-context": {
"version": "25.3.77",
"resolved": "https://registry.npmjs.org/standardized-audio-context/-/standardized-audio-context-25.3.77.tgz",
"integrity": "sha512-Ki9zNz6pKcC5Pi+QPjPyVsD9GwJIJWgryji0XL9cAJXMGyn+dPOf6Qik1AHei0+UNVcc4BOCa0hWLBzlwqsW/A==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.25.6",
"automation-events": "^7.0.9",
"tslib": "^2.7.0"
}
},
"node_modules/sucrase": {
"version": "3.35.1",
"resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz",
@@ -2416,12 +2450,28 @@
"node": ">=8.0"
}
},
"node_modules/tone": {
"version": "14.9.17",
"resolved": "https://registry.npmjs.org/tone/-/tone-14.9.17.tgz",
"integrity": "sha512-+Qb7M4NMua+tb5Z52+MEVmjye0fjJuIFBePx423pqr9E6/lHDqZAG+fUAvo+Ujm48q0s9bVLRAyT1ETJJglNtg==",
"license": "MIT",
"dependencies": {
"standardized-audio-context": "^25.3.70",
"tslib": "^2.3.1"
}
},
"node_modules/ts-interface-checker": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz",
"integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==",
"dev": true
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"license": "0BSD"
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
+1
View File
@@ -11,6 +11,7 @@
"dependencies": {
"@tanstack/react-query": "^5.0.0",
"@tonejs/midi": "^2.0.28",
"tone": "^14.7.0",
"lucide-react": "^0.294.0",
"react": "^18.2.0",
"react-dom": "^18.2.0"
+548 -207
View File
@@ -1,5 +1,5 @@
import { useState, useEffect, useLayoutEffect, useRef, useCallback } from 'react'
import { Play, Pause, SkipBack, Volume2, VolumeX, Palette, Pencil, X, AlignLeft, AlignCenter, AlignRight, Type, Crosshair } from 'lucide-react'
import { Play, Pause, SkipBack, Volume2, VolumeX, Palette, Pencil, X, AlignLeft, AlignCenter, AlignRight, Type, Crosshair, Maximize2, Minimize2 } from 'lucide-react'
import type { LyricsStyleOptions } from '../types'
import { registerPlayback, unregisterPlayback, publishMeta, clearMeta } from '../utils/audioRegistry'
@@ -24,7 +24,7 @@ interface Stem {
url: string
}
export type LyricsVisualStyle = 'scroll' | 'fade' | 'karaoke' | 'glow'
export type LyricsVisualStyle = 'scroll' | 'fade' | 'karaoke' | 'glow' | 'blur'
interface Props {
lrcText: string
@@ -36,6 +36,10 @@ interface Props {
onLrcChange?: (newLrc: string) => void
/** Stable key used to persist speaker names/colours in localStorage. */
storageKey?: string
/** URL of the source audio cover art (served by /api/projects/{id}/cover-art). */
coverArtUrl?: string
/** Artist name from source audio metadata. */
artist?: string
}
// ─── Visual style definitions ─────────────────────────────────────────────────
@@ -44,7 +48,8 @@ const VISUAL_STYLES: { id: LyricsVisualStyle; label: string; desc: string }[] =
{ id: 'scroll', label: 'Scroll', desc: 'Center-scroll with highlighted current line' },
{ id: 'fade', label: 'Fade', desc: 'Spotlight — inactive lines fade away' },
{ id: 'karaoke', label: 'Karaoke', desc: 'Word-by-word left-to-right fill' },
{ id: 'glow', label: 'Glow', desc: 'Words pop with a glow when active' },
{ id: 'glow', label: 'Glow', desc: 'Left-to-right fill with colour glow on active line' },
{ id: 'blur', label: 'Blur', desc: 'Active line sharp and bright, inactive lines blurred' },
]
const FONT_OPTIONS = [
@@ -269,14 +274,17 @@ interface StyleState {
wordHighlight: string
textAlign: 'left' | 'center' | 'right'
lineSpacing: number
dynBg: boolean
}
function StylePicker({
style,
onChange,
hasCoverArt,
}: {
style: StyleState
onChange: (patch: Partial<StyleState>) => void
hasCoverArt?: boolean
}) {
return (
<div className="px-3 py-2.5 border-b border-white/10 space-y-2.5">
@@ -337,6 +345,18 @@ function StylePicker({
/>
</div>
</div>
{/* Dynamic BG from album art — only shown when cover art is available */}
{hasCoverArt && (
<label className="flex items-center gap-2 cursor-pointer select-none">
<input
type="checkbox"
checked={style.dynBg}
onChange={e => onChange({ dynBg: e.target.checked })}
className="w-3 h-3 accent-brand-400 cursor-pointer"
/>
<span className="text-[10px] text-slate-400">Dynamic background color from album art</span>
</label>
)}
</div>
)
}
@@ -461,7 +481,7 @@ function LrcEditor({
// ─── Main component ───────────────────────────────────────────────────────────
export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, styleOptions, defaultStyle, onLrcChange, storageKey }: Props) {
export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, styleOptions, defaultStyle, onLrcChange, storageKey, coverArtUrl, artist }: Props) {
// Internal lrc override from editor
const [editedLrc, setEditedLrc] = useState<string | null>(null)
const activeLrc = editedLrc ?? lrcText
@@ -470,9 +490,11 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
const hasElrc = lines.some(l => l.words.length > 0)
const isMultiStem = stems && stems.length > 0
const [visualStyle, setVisualStyle] = useState<LyricsVisualStyle>(
defaultStyle ?? (hasElrc ? 'karaoke' : 'scroll')
)
const [visualStyle, setVisualStyle] = useState<LyricsVisualStyle>(() => {
const saved = localStorage.getItem('stemengine-lyric-visual-style') as LyricsVisualStyle | null
if (saved && ['scroll', 'fade', 'karaoke', 'glow', 'blur'].includes(saved)) return saved
return defaultStyle ?? (hasElrc ? 'karaoke' : 'scroll')
})
const [currentTime, setCurrentTime] = useState(0)
const [duration, setDuration] = useState(0)
const [isPlaying, setIsPlaying] = useState(false)
@@ -483,21 +505,70 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
const [showStylePicker, setShowStylePicker] = useState(false)
const [showEditor, setShowEditor] = useState(false)
// Editable style state — seeded from styleOptions prop
const [style, setStyle] = useState<StyleState>({
fontFamily: styleOptions?.fontFamily ?? 'system-ui, sans-serif',
fontSize: styleOptions?.fontSize ?? 20,
bgColor: styleOptions?.backgroundColor ?? '#0f172a',
textColor: styleOptions?.textColor ?? '#c8d3e0',
highlightColor: styleOptions?.highlightColor ?? '#3b82f6',
wordHighlight: styleOptions?.wordHighlightColor ?? '#60a5fa',
textAlign: (styleOptions?.textAlign ?? 'center') as 'left' | 'center' | 'right',
lineSpacing: styleOptions?.lineSpacing ?? 1.6,
// Editable style state — seeded from styleOptions prop, then overridden by any saved preference
const [style, setStyle] = useState<StyleState>(() => {
const defaults: StyleState = {
fontFamily: styleOptions?.fontFamily ?? 'system-ui, sans-serif',
fontSize: styleOptions?.fontSize ?? 20,
bgColor: styleOptions?.backgroundColor ?? '#0f172a',
textColor: styleOptions?.textColor ?? '#c8d3e0',
highlightColor: styleOptions?.highlightColor ?? '#3b82f6',
wordHighlight: styleOptions?.wordHighlightColor ?? '#60a5fa',
textAlign: (styleOptions?.textAlign ?? 'center') as 'left' | 'center' | 'right',
lineSpacing: styleOptions?.lineSpacing ?? 1.6,
dynBg: false,
}
try {
const saved = localStorage.getItem('stemengine-lyric-style')
if (saved) return { ...defaults, ...JSON.parse(saved) }
} catch { /* ignore */ }
return defaults
})
const patchStyle = useCallback((patch: Partial<StyleState>) => setStyle(s => ({ ...s, ...patch })), [])
const patchStyle = useCallback((patch: Partial<StyleState>) => {
setStyle(s => {
const next = { ...s, ...patch }
try { localStorage.setItem('stemengine-lyric-style', JSON.stringify(next)) } catch { /* quota */ }
return next
})
}, [])
// Destructure for convenience
const { fontFamily, fontSize, bgColor, textColor, highlightColor, wordHighlight, textAlign, lineSpacing } = style
const { fontFamily, fontSize, bgColor, textColor, highlightColor, wordHighlight, textAlign, lineSpacing, dynBg } = style
// Track whether cover art image loaded successfully
const [artLoaded, setArtLoaded] = useState(false)
// Reset artLoaded when coverArtUrl changes
useEffect(() => { setArtLoaded(false) }, [coverArtUrl])
// ── Dynamic background color extracted from album art (Canvas API) ──────────
const [extractedBg, setExtractedBg] = useState<string | null>(null)
useEffect(() => {
if (!dynBg || !coverArtUrl || !artLoaded) { setExtractedBg(null); return }
const img = new window.Image()
img.crossOrigin = 'anonymous'
img.onload = () => {
try {
const canvas = document.createElement('canvas')
canvas.width = 16; canvas.height = 16
const ctx = canvas.getContext('2d')!
ctx.drawImage(img, 0, 0, 16, 16)
const { data } = ctx.getImageData(0, 0, 16, 16)
let r = 0, g = 0, b = 0
for (let i = 0; i < data.length; i += 4) {
r += data[i]; g += data[i + 1]; b += data[i + 2]
}
const n = data.length / 4
// Darken to ~22% brightness to keep the background dark and readable
setExtractedBg(`rgb(${Math.round(r / n * 0.22)},${Math.round(g / n * 0.22)},${Math.round(b / n * 0.22)})`)
} catch { setExtractedBg(null) }
}
img.onerror = () => setExtractedBg(null)
img.src = coverArtUrl
}, [dynBg, coverArtUrl, artLoaded])
// Effective background color: extracted color when dynBg is enabled, otherwise user-chosen bgColor
const effectiveBg = dynBg && extractedBg ? extractedBg : bgColor
// Per-stem state (multi-stem mode)
const [gains, setGains] = useState<Record<string, number>>({})
@@ -610,9 +681,9 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [activeLine, wordHighlight, textColor, speakerColors])
// ── Auto-detect default style for new content ────────────────────────────
// ── Auto-detect default style for new content (only if user has no saved preference) ──
useEffect(() => {
if (!defaultStyle) {
if (!defaultStyle && !localStorage.getItem('stemengine-lyric-visual-style')) {
setVisualStyle(hasElrc ? 'karaoke' : 'scroll')
}
}, [hasElrc, defaultStyle])
@@ -766,6 +837,21 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
}
}, [isPlaying, duration, isMultiStem])
// ── Fullscreen ────────────────────────────────────────────────────────────
const fullscreenRef = useRef<HTMLDivElement | null>(null)
const [isFullscreen, setIsFullscreen] = useState(false)
useEffect(() => {
function onFsChange() { setIsFullscreen(!!document.fullscreenElement) }
document.addEventListener('fullscreenchange', onFsChange)
return () => document.removeEventListener('fullscreenchange', onFsChange)
}, [])
function toggleFullscreen() {
if (!isFullscreen) fullscreenRef.current?.requestFullscreen()
else document.exitFullscreen()
}
// ── Centre active line ────────────────────────────────────────────────────
// Uses CSS translateY on an inner wrapper rather than scrollTop so that:
// • Windowed modes (karaoke/glow) — only ~7 lines are in the DOM, so
@@ -1148,8 +1234,8 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
if (lines.length === 0) return <p style={{ color: textColor, opacity: 0.4, textAlign }}>No lyrics</p>
const window = 3
const start = Math.max(0, activeLine - window)
const end = Math.min(lines.length - 1, activeLine + window + 1)
const start = Math.max(0, activeLine - window)
const end = Math.min(lines.length - 1, activeLine + window + 1)
const visible = lines.slice(start, end + 1)
return (
@@ -1158,56 +1244,197 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
const i = start + vi
const active = i === activeLine
const past = i < activeLine
const activeWordIdx = active ? findActiveWord(line.words, currentTime) : -1
const nextLineTime = lines[i + 1]?.time
if (!active) {
return (
<div
key={i}
style={{
textAlign,
padding: '0.3em 0.75em',
fontSize: fontSize * 0.85,
lineHeight: lineSpacing,
opacity: past ? 0.5 : 0.2,
transition: 'opacity 0.4s ease',
}}
>
<SpeakerDot line={line} prevSpeaker={lines[i - 1]?.speaker} color={line.speaker ? getSpeakerColor(line.speaker) : undefined} name={line.speaker ? getSpeakerName(line.speaker) : undefined} />
{line.words.length > 0
? line.words.map((w, wi) => (
<span key={wi} style={{ color: past ? (line.speaker ? getSpeakerColor(line.speaker) : wordHighlight) : textColor, marginRight: w.isWordEnd ? '0.25em' : '0' }}>
{w.text}
</span>
))
: <span style={{ color: past ? (line.speaker ? getSpeakerColor(line.speaker) : wordHighlight) : textColor }}>{line.text}</span>
}
</div>
)
}
// Active line — same word-fill as karaoke but with a colour glow filter.
// Using CSS filter: drop-shadow avoids the textShadow + WebkitTextFillColor:transparent
// incompatibility that caused the previous implementation to look broken.
const activeWordIdx = findActiveWord(line.words, currentTime)
const glowColor = line.speaker ? getSpeakerColor(line.speaker) : wordHighlight
const dimColor = line.speaker ? getSpeakerColor(line.speaker) + '60' : textColor + '60'
return (
<div
key={i}
ref={active ? (el) => { activeLineRef.current = el } : undefined}
ref={(el) => { activeLineRef.current = el }}
style={{
textAlign,
padding: '0.35em 0.75em',
fontSize: active ? fontSize * 1.05 : fontSize * 0.88,
padding: '0.4em 0.75em',
fontSize: fontSize * 1.05,
lineHeight: lineSpacing,
opacity: active ? 1 : past ? 0.45 : 0.22,
transition: 'all 0.3s ease',
filter: `drop-shadow(0 0 8px ${glowColor}cc) drop-shadow(0 0 16px ${glowColor}66)`,
transition: 'filter 0.3s ease',
}}
>
<SpeakerDot line={line} prevSpeaker={lines[i - 1]?.speaker} color={line.speaker ? getSpeakerColor(line.speaker) : undefined} name={line.speaker ? getSpeakerName(line.speaker) : undefined} />
{line.words.length > 0
? line.words.map((w, wi) => {
const wordActive = active && wi === activeWordIdx
const wordDone = active ? wi < activeWordIdx : past
const glowColor = line.speaker ? getSpeakerColor(line.speaker) : wordHighlight
const doneColor = line.speaker ? getSpeakerColor(line.speaker) + 'cc' : textColor + 'cc'
const done = wi < activeWordIdx
const fill = wi === activeWordIdx
? wordFillPct(line, wi, nextLineTime, currentTime)
: 0
return (
<span
<KaraokeWord
key={wi}
style={{
color: wordActive
? glowColor
: wordDone
? doneColor
: textColor + '55',
marginRight: w.isWordEnd ? '0.25em' : '0',
display: 'inline-block',
verticalAlign: 'top',
textShadow: wordActive
? `0 0 12px ${glowColor}, 0 0 24px ${glowColor}80`
: 'none',
transform: wordActive ? 'scale(1.06)' : 'scale(1)',
transformOrigin: '50% 50%',
transition: 'all 0.15s ease',
fontWeight: wordActive ? 700 : 400,
}}
>
{w.text}
</span>
word={w.text}
fill={fill}
done={done}
bright={glowColor}
dim={dimColor}
fontSize={fontSize * 1.05}
isWordEnd={w.isWordEnd}
elRef={wi === activeWordIdx ? activeWordSpanRef : undefined}
/>
)
})
: <span style={{ color: active ? (line.speaker ? getSpeakerColor(line.speaker) : highlightColor) : past ? textColor : textColor + '40' }}>
{line.text}
</span>
: (() => {
const nextT = lines[activeLine + 1]?.time ?? (line.time + 4)
const pct = Math.max(0, Math.min(1, (currentTime - line.time) / (nextT - line.time)))
return (
<span
ref={activeWordSpanRef}
style={{
display: 'inline-block',
verticalAlign: 'top',
backgroundImage: `linear-gradient(to right, ${glowColor} ${Math.round(pct * 100)}%, ${dimColor} ${Math.round(pct * 100)}%)`,
WebkitBackgroundClip: 'text',
WebkitTextFillColor: 'transparent',
backgroundClip: 'text',
} as React.CSSProperties}
>
{line.text}
</span>
)
})()
}
</div>
)
})}
</>
)
}
function renderBlurStyle() {
if (lines.length === 0) return <p style={{ color: textColor, opacity: 0.4, textAlign }}>No lyrics</p>
const window = 3
const start = Math.max(0, activeLine - window)
const end = Math.min(lines.length - 1, activeLine + window + 1)
const visible = lines.slice(start, end + 1)
return (
<>
{visible.map((line, vi) => {
const i = start + vi
const active = i === activeLine
const past = i < activeLine
const dist = Math.abs(i - activeLine)
const nextLineTime = lines[i + 1]?.time
if (!active) {
return (
<div
key={i}
style={{
textAlign,
padding: '0.3em 0.75em',
fontSize: fontSize * 0.88,
lineHeight: lineSpacing,
color: past ? (line.speaker ? getSpeakerColor(line.speaker) : wordHighlight) : textColor,
opacity: past ? 0.4 : 0.15,
filter: `blur(${dist * 1.5}px)`,
transition: 'opacity 0.4s ease, filter 0.4s ease',
}}
>
<SpeakerDot line={line} prevSpeaker={lines[i - 1]?.speaker} color={line.speaker ? getSpeakerColor(line.speaker) : undefined} name={line.speaker ? getSpeakerName(line.speaker) : undefined} />
<span>{line.words.length > 0 ? line.words.map(w => w.text + (w.isWordEnd ? ' ' : '')).join('') : line.text}</span>
</div>
)
}
// Active line — sharp, full brightness, word-fill same as karaoke
const activeWordIdx = findActiveWord(line.words, currentTime)
const activeBright = line.speaker ? getSpeakerColor(line.speaker) : wordHighlight
const activeDim = line.speaker ? getSpeakerColor(line.speaker) + '60' : textColor + '60'
return (
<div
key={i}
ref={(el) => { activeLineRef.current = el }}
style={{
textAlign,
padding: '0.4em 0.75em',
fontSize: fontSize * 1.1,
lineHeight: lineSpacing,
filter: 'none',
transition: 'filter 0.4s ease',
}}
>
<SpeakerDot line={line} prevSpeaker={lines[i - 1]?.speaker} color={line.speaker ? getSpeakerColor(line.speaker) : undefined} name={line.speaker ? getSpeakerName(line.speaker) : undefined} />
{line.words.length > 0
? line.words.map((w, wi) => {
const done = wi < activeWordIdx
const fill = wi === activeWordIdx
? wordFillPct(line, wi, nextLineTime, currentTime)
: 0
return (
<KaraokeWord
key={wi}
word={w.text}
fill={fill}
done={done}
bright={activeBright}
dim={activeDim}
fontSize={fontSize * 1.1}
isWordEnd={w.isWordEnd}
elRef={wi === activeWordIdx ? activeWordSpanRef : undefined}
/>
)
})
: (() => {
const nextT = lines[activeLine + 1]?.time ?? (line.time + 4)
const pct = Math.max(0, Math.min(1, (currentTime - line.time) / (nextT - line.time)))
const fillColor = line.speaker ? getSpeakerColor(line.speaker) : highlightColor
return (
<span
ref={activeWordSpanRef}
style={{
display: 'inline-block',
verticalAlign: 'top',
backgroundImage: `linear-gradient(to right, ${fillColor} ${Math.round(pct * 100)}%, ${fillColor}50 ${Math.round(pct * 100)}%)`,
WebkitBackgroundClip: 'text',
WebkitTextFillColor: 'transparent',
backgroundClip: 'text',
} as React.CSSProperties}
>
{line.text}
</span>
)
})()
}
</div>
)
@@ -1224,14 +1451,218 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
const displayDuration = timeSource?.duration || duration || 1
const hasAudio = isMultiStem || !!audioUrl
// Helper: lyrics display area
function renderLyricsArea() {
return (
<div
ref={containerRef}
style={{
backgroundColor: effectiveBg,
fontFamily,
height: isFullscreen ? undefined : 'clamp(200px, 40vh, 340px)',
flex: isFullscreen ? '1 1 0' : undefined,
overflow: 'hidden',
position: 'relative',
}}
>
<div
style={{
transform: `translateY(${translateY}px)`,
transition: 'transform 0.35s cubic-bezier(0.33, 1, 0.68, 1)',
willChange: 'transform',
textAlign,
}}
>
{visualStyle === 'scroll' && renderScrollStyle()}
{visualStyle === 'fade' && renderFadeStyle()}
{visualStyle === 'karaoke' && renderKaraokeStyle()}
{visualStyle === 'glow' && renderGlowStyle()}
{visualStyle === 'blur' && renderBlurStyle()}
</div>
{/* Windowed album art overlay — top-left thumbnail (only shown when art loaded) */}
{!isFullscreen && coverArtUrl && artLoaded && (
<div style={{
position: 'absolute', top: 8, left: 8, zIndex: 10,
display: 'flex', flexDirection: 'column', alignItems: 'flex-start', gap: 3,
background: 'linear-gradient(135deg, rgba(0,0,0,0.55) 0%, transparent 100%)',
padding: '6px 14px 6px 6px',
borderRadius: 8,
maxWidth: '45%',
}}>
<img
src={coverArtUrl}
alt="Cover"
style={{ width: 48, height: 48, objectFit: 'cover', borderRadius: 6, flexShrink: 0 }}
/>
{(artist || title) && (
<div style={{ maxWidth: 120, overflow: 'hidden' }}>
{artist && (
<p style={{ margin: 0, fontSize: 10, color: '#cbd5e1', fontWeight: 600, lineHeight: 1.3,
overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
{artist}
</p>
)}
{title && (
<p style={{ margin: 0, fontSize: 9, color: '#94a3b8', lineHeight: 1.3,
overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
{title}
</p>
)}
</div>
)}
</div>
)}
{/* Hidden image used to detect whether cover art is available */}
{coverArtUrl && !artLoaded && (
<img
src={coverArtUrl}
alt=""
style={{ display: 'none' }}
onLoad={() => setArtLoaded(true)}
onError={() => setArtLoaded(false)}
/>
)}
</div>
)
}
// Helper: transport controls
function renderTransport() {
return (
<div
style={{ backgroundColor: effectiveBg }}
className="flex items-center gap-1.5 px-2.5 py-2 border-t border-white/10"
>
<button
onClick={handleReset}
className="p-1.5 rounded text-slate-400 hover:text-white transition-colors shrink-0"
title="Restart"
>
<SkipBack className="w-4 h-4" />
</button>
<button
onClick={togglePlay}
className="p-1.5 rounded text-white hover:text-brand-300 transition-colors shrink-0"
title={isPlaying ? 'Pause' : 'Play'}
>
{isPlaying
? <Pause className="w-5 h-5" />
: <Play className="w-5 h-5" />
}
</button>
<div className="flex-1 flex items-center min-w-0 py-2 -my-2">
<input
type="range"
min={0}
max={displayDuration}
step={0.1}
value={currentTime}
onChange={handleSeek}
className="w-full h-1.5 accent-brand-500 cursor-pointer"
/>
</div>
<span className="text-[11px] tabular-nums text-slate-400 whitespace-nowrap shrink-0 ml-0.5">
{formatTime(currentTime)}
</span>
{hasAudio && (
<div className="flex items-center gap-1 ml-1 shrink-0">
<button
onClick={toggleMute}
className="p-1.5 rounded text-slate-400 hover:text-white transition-colors"
title={isMuted ? 'Unmute' : 'Mute'}
>
{isMuted || volume === 0
? <VolumeX className="w-4 h-4" />
: <Volume2 className="w-4 h-4" />
}
</button>
<input
type="range"
min={0}
max={1}
step={0.05}
value={isMuted ? 0 : volume}
onChange={handleVolumeChange}
className="w-14 h-1.5 accent-brand-500 cursor-pointer hidden sm:block"
/>
</div>
)}
</div>
)
}
// Helper: per-stem controls
function renderStems() {
if (!isMultiStem || !stems) return null
return (
<div
style={{ backgroundColor: effectiveBg }}
className="space-y-1.5 px-3 pb-2.5 pt-1.5 border-t border-white/10"
>
{stems.map(stem => (
<div key={stem.name} className="flex items-center gap-2 min-h-[36px]">
<span className="text-xs font-medium text-slate-400 w-16 truncate capitalize shrink-0">
{stem.label}
</span>
<button
onClick={() => toggleStemMute(stem.name)}
title={mutedStems[stem.name] ? 'Unmute' : 'Mute'}
className={`w-7 h-7 text-xs font-bold rounded transition-colors flex-shrink-0 ${
mutedStems[stem.name]
? 'bg-red-500 text-white'
: 'bg-white/10 text-slate-400 hover:bg-white/20'
}`}
>
M
</button>
<button
onClick={() => toggleStemSolo(stem.name)}
title={soloedStems[stem.name] ? 'Unsolo' : 'Solo'}
className={`w-7 h-7 text-xs font-bold rounded transition-colors flex-shrink-0 ${
soloedStems[stem.name]
? 'bg-yellow-400 text-black'
: 'bg-white/10 text-slate-400 hover:bg-white/20'
}`}
>
S
</button>
<div className="flex-1 flex items-center min-w-0 py-2 -my-2">
<input
type="range"
min={0}
max={2}
step={0.1}
value={gains[stem.name] ?? 1.0}
onChange={e => updateStemGain(stem.name, parseFloat(e.target.value))}
className="w-full h-1.5 accent-brand-500 cursor-pointer"
/>
</div>
<span className="text-[10px] tabular-nums text-slate-500 w-8 text-right shrink-0">
{(gains[stem.name] ?? 1.0).toFixed(1)}x
</span>
</div>
))}
</div>
)
}
return (
<div className="rounded-xl overflow-hidden border border-slate-200/30 dark:border-gray-700/50">
<div
ref={fullscreenRef}
className={isFullscreen
? 'flex flex-col'
: 'rounded-xl overflow-hidden border border-slate-200/30 dark:border-gray-700/50'
}
style={isFullscreen ? { backgroundColor: effectiveBg } : undefined}
>
{/* Visual style tabs + toolbar — scrollable on narrow screens */}
<div className="flex items-center gap-1 px-2 py-1 bg-slate-100/80 dark:bg-gray-800/80 border-b border-slate-200/30 dark:border-gray-700/50 overflow-x-auto [&::-webkit-scrollbar]:hidden">
<div className={`flex items-center gap-1 px-2 py-1 border-b overflow-x-auto [&::-webkit-scrollbar]:hidden ${isFullscreen ? 'border-white/10' : 'bg-slate-100/80 dark:bg-gray-800/80 border-slate-200/30 dark:border-gray-700/50'}`}>
{VISUAL_STYLES.map(s => (
<button
key={s.id}
onClick={() => setVisualStyle(s.id)}
onClick={() => { setVisualStyle(s.id); localStorage.setItem('stemengine-lyric-visual-style', s.id) }}
title={s.desc}
className={`shrink-0 px-2.5 py-1.5 rounded-md text-xs font-medium transition-colors ${
visualStyle === s.id
@@ -1249,31 +1680,38 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
<button
onClick={handleRecenter}
title="Recenter active line"
className="p-1.5 rounded-md transition-colors text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700"
className={`p-1.5 rounded-md transition-colors ${isFullscreen ? 'text-slate-400 hover:bg-white/10' : 'text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700'}`}
>
<Crosshair className="w-3.5 h-3.5" />
</button>
<button
onClick={() => { setShowStylePicker(p => !p); setShowEditor(false) }}
title="Style picker"
className={`p-1.5 rounded-md transition-colors ${showStylePicker ? 'bg-brand-500 text-white' : 'text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700'}`}
className={`p-1.5 rounded-md transition-colors ${showStylePicker ? 'bg-brand-500 text-white' : isFullscreen ? 'text-slate-400 hover:bg-white/10' : 'text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700'}`}
>
<Palette className="w-3.5 h-3.5" />
</button>
<button
onClick={() => { setShowEditor(p => !p); setShowStylePicker(false) }}
title="Edit timing / lyrics"
className={`p-1.5 rounded-md transition-colors ${showEditor ? 'bg-brand-500 text-white' : 'text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700'}`}
className={`p-1.5 rounded-md transition-colors ${showEditor ? 'bg-brand-500 text-white' : isFullscreen ? 'text-slate-400 hover:bg-white/10' : 'text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700'}`}
>
<Pencil className="w-3.5 h-3.5" />
</button>
<button
onClick={toggleFullscreen}
title={isFullscreen ? 'Exit fullscreen' : 'Fullscreen'}
className={`p-1.5 rounded-md transition-colors ${isFullscreen ? 'text-slate-400 hover:bg-white/10' : 'text-slate-500 dark:text-gray-400 hover:bg-slate-200 dark:hover:bg-gray-700'}`}
>
{isFullscreen ? <Minimize2 className="w-3.5 h-3.5" /> : <Maximize2 className="w-3.5 h-3.5" />}
</button>
</div>
</div>
{/* Style picker panel */}
{showStylePicker && (
<div style={{ backgroundColor: bgColor }}>
<StylePicker style={style} onChange={patchStyle} />
<div style={{ backgroundColor: effectiveBg }}>
<StylePicker style={style} onChange={patchStyle} hasCoverArt={!!coverArtUrl && artLoaded} />
{/* Speaker colour swatches — always shown so user knows if diarization is missing */}
<div className="px-3 py-2 border-b border-white/10 flex items-center gap-3 flex-wrap">
<span className="text-[10px] text-slate-500 shrink-0">Speakers</span>
@@ -1338,7 +1776,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
{showEditor && (
<LrcEditor
lrcText={activeLrc}
bgColor={bgColor}
bgColor={effectiveBg}
onApply={(newLrc) => {
setEditedLrc(newLrc)
onLrcChange?.(newLrc)
@@ -1348,150 +1786,53 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
/>
)}
{/* Lyrics display */}
<div
ref={containerRef}
style={{
backgroundColor: bgColor,
fontFamily,
height: 'clamp(200px, 40vh, 340px)',
overflow: 'hidden',
position: 'relative', // offsetParent for lyric line offsetTop measurements
}}
>
{/* Inner wrapper: translateY centres the active line vertically.
No position:relative here — children's offsetParent stays as
containerRef so offsetTop is measured from the container top. */}
<div
style={{
transform: `translateY(${translateY}px)`,
transition: 'transform 0.35s cubic-bezier(0.33, 1, 0.68, 1)',
willChange: 'transform',
textAlign,
}}
>
{visualStyle === 'scroll' && renderScrollStyle()}
{visualStyle === 'fade' && renderFadeStyle()}
{visualStyle === 'karaoke' && renderKaraokeStyle()}
{visualStyle === 'glow' && renderGlowStyle()}
</div>
</div>
{/* Transport controls */}
<div
style={{ backgroundColor: bgColor }}
className="flex items-center gap-1.5 px-2.5 py-2 border-t border-white/10"
>
<button
onClick={handleReset}
className="p-1.5 rounded text-slate-400 hover:text-white transition-colors shrink-0"
title="Restart"
>
<SkipBack className="w-4 h-4" />
</button>
<button
onClick={togglePlay}
className="p-1.5 rounded text-white hover:text-brand-300 transition-colors shrink-0"
title={isPlaying ? 'Pause' : 'Play'}
>
{isPlaying
? <Pause className="w-5 h-5" />
: <Play className="w-5 h-5" />
}
</button>
{/* Seek bar — wrapped in a div so the invisible padding expands the touch area */}
<div className="flex-1 flex items-center min-w-0 py-2 -my-2">
<input
type="range"
min={0}
max={displayDuration}
step={0.1}
value={currentTime}
onChange={handleSeek}
className="w-full h-1.5 accent-brand-500 cursor-pointer"
/>
</div>
<span className="text-[11px] tabular-nums text-slate-400 whitespace-nowrap shrink-0 ml-0.5">
{formatTime(currentTime)}
</span>
{hasAudio && (
<div className="flex items-center gap-1 ml-1 shrink-0">
<button
onClick={toggleMute}
className="p-1.5 rounded text-slate-400 hover:text-white transition-colors"
title={isMuted ? 'Unmute' : 'Mute'}
>
{isMuted || volume === 0
? <VolumeX className="w-4 h-4" />
: <Volume2 className="w-4 h-4" />
}
</button>
{/* Volume slider — hidden on narrow screens via CSS container query */}
<input
type="range"
min={0}
max={1}
step={0.05}
value={isMuted ? 0 : volume}
onChange={handleVolumeChange}
className="w-14 h-1.5 accent-brand-500 cursor-pointer hidden sm:block"
{/* Content area: fullscreen with art → left panel + right column; otherwise stacked */}
{isFullscreen && coverArtUrl && artLoaded ? (
<div style={{ flex: '1 1 0', display: 'flex', flexDirection: 'row', overflow: 'hidden' }}>
{/* Left panel: large album art + artist + title */}
<div style={{
width: '38%', flexShrink: 0, backgroundColor: effectiveBg,
display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center',
padding: '2rem', gap: '1.25rem', borderRight: '1px solid rgba(255,255,255,0.07)',
}}>
<img
src={coverArtUrl}
alt="Cover"
style={{
width: '100%', maxWidth: 240, aspectRatio: '1/1',
objectFit: 'cover', borderRadius: 12,
boxShadow: '0 20px 60px rgba(0,0,0,0.6)',
}}
onError={e => { (e.currentTarget as HTMLImageElement).style.display = 'none' }}
/>
</div>
)}
</div>
{/* Per-stem mute / solo / gain */}
{isMultiStem && stems && (
<div
style={{ backgroundColor: bgColor }}
className="space-y-1.5 px-3 pb-2.5 pt-1.5 border-t border-white/10"
>
{stems.map(stem => (
<div key={stem.name} className="flex items-center gap-2 min-h-[36px]">
<span className="text-xs font-medium text-slate-400 w-16 truncate capitalize shrink-0">
{stem.label}
</span>
<button
onClick={() => toggleStemMute(stem.name)}
title={mutedStems[stem.name] ? 'Unmute' : 'Mute'}
className={`w-7 h-7 text-xs font-bold rounded transition-colors flex-shrink-0 ${
mutedStems[stem.name]
? 'bg-red-500 text-white'
: 'bg-white/10 text-slate-400 hover:bg-white/20'
}`}
>
M
</button>
<button
onClick={() => toggleStemSolo(stem.name)}
title={soloedStems[stem.name] ? 'Unsolo' : 'Solo'}
className={`w-7 h-7 text-xs font-bold rounded transition-colors flex-shrink-0 ${
soloedStems[stem.name]
? 'bg-yellow-400 text-black'
: 'bg-white/10 text-slate-400 hover:bg-white/20'
}`}
>
S
</button>
{/* Expanded touch area for the slider */}
<div className="flex-1 flex items-center min-w-0 py-2 -my-2">
<input
type="range"
min={0}
max={2}
step={0.1}
value={gains[stem.name] ?? 1.0}
onChange={e => updateStemGain(stem.name, parseFloat(e.target.value))}
className="w-full h-1.5 accent-brand-500 cursor-pointer"
/>
</div>
<span className="text-[10px] tabular-nums text-slate-500 w-8 text-right shrink-0">
{(gains[stem.name] ?? 1.0).toFixed(1)}x
</span>
<div style={{ textAlign: 'center', maxWidth: '100%' }}>
{artist && (
<p style={{ margin: '0 0 4px 0', fontSize: 16, color: '#e2e8f0',
fontWeight: 700, lineHeight: 1.3, wordBreak: 'break-word' }}>
{artist}
</p>
)}
{title && (
<p style={{ margin: 0, fontSize: 13, color: '#94a3b8',
lineHeight: 1.3, wordBreak: 'break-word' }}>
{title}
</p>
)}
</div>
))}
</div>
{/* Right column: lyrics + transport + stems */}
<div style={{ flex: 1, display: 'flex', flexDirection: 'column', overflow: 'hidden' }}>
{renderLyricsArea()}
{renderTransport()}
{renderStems()}
</div>
</div>
) : (
<>
{renderLyricsArea()}
{renderTransport()}
{renderStems()}
</>
)}
{/* Hidden single-track audio element */}
+211 -21
View File
@@ -4,6 +4,7 @@ import {
Music2, Mic2, FileCode, FileText, Download, Loader2, Trash2,
CheckCircle, XCircle, Clock, AlertCircle, ChevronDown, ChevronUp,
ListMusic, Edit2, Check, X as XIcon, Wand2, RotateCcw, Copy, Users,
Play, Pause, Square,
} from 'lucide-react'
import { getProjectDetail, transcribeProject, abcProject, mixProject, lrcProject, lyricTranscribeProject, downloadUrl, reprocessStem, renameStem, getPerInstrumentModels } from '../api/client'
import type { ProjectDetail as PD, Job, JobFile } from '../types'
@@ -77,12 +78,32 @@ function DownloadButton({ file, jobId }: { file: JobFile; jobId: string }) {
)
}
// ─── Piano roll ─────────────────────────────────────────────────────────────
// ─── Piano roll + MIDI player ────────────────────────────────────────────────
const TRACK_COLORS = ['#3b82f6', '#f97316', '#a855f7', '#10b981', '#ec4899', '#eab308']
const NOTE_PX = 4 // pixel height per semitone
const SVG_W = 800
function PianoRoll({ midi }: { midi: any }) {
// Salamander Grand Piano — standard Tone.js sampler sample set
const SALAMANDER_BASE = 'https://tonejs.github.io/audio/salamander/'
const SALAMANDER_URLS: Record<string, string> = {
A0:'A0.mp3', C1:'C1.mp3', 'D#1':'Ds1.mp3', 'F#1':'Fs1.mp3',
A1:'A1.mp3', C2:'C2.mp3', 'D#2':'Ds2.mp3', 'F#2':'Fs2.mp3',
A2:'A2.mp3', C3:'C3.mp3', 'D#3':'Ds3.mp3', 'F#3':'Fs3.mp3',
A3:'A3.mp3', C4:'C4.mp3', 'D#4':'Ds4.mp3', 'F#4':'Fs4.mp3',
A4:'A4.mp3', C5:'C5.mp3', 'D#5':'Ds5.mp3', 'F#5':'Fs5.mp3',
A5:'A5.mp3', C6:'C6.mp3', 'D#6':'Ds6.mp3', 'F#6':'Fs6.mp3',
A6:'A6.mp3', C7:'C7.mp3', 'D#7':'Ds7.mp3', 'F#7':'Fs7.mp3',
A7:'A7.mp3', C8:'C8.mp3',
}
function PianoRoll({
midi,
playheadRef,
}: {
midi: any
playheadRef: React.RefObject<SVGLineElement | null>
}) {
const allNotes = midi.tracks
.flatMap((t: any, ti: number) => t.notes.map((n: any) => ({ ...n, ti })))
.filter((n: any) => n.duration > 0)
@@ -95,40 +116,39 @@ function PianoRoll({ midi }: { midi: any }) {
const minP = Math.max(0, Math.min(...allNotes.map((n: any) => n.midi)) - 2)
const maxP = Math.min(127, Math.max(...allNotes.map((n: any) => n.midi)) + 2)
const range = maxP - minP + 1
const svgW = 800
const svgH = range * NOTE_PX
const svgH = range * NOTE_PX
const isBlack = (p: number) => [1, 3, 6, 8, 10].includes(p % 12)
return (
<div className="overflow-x-auto bg-gray-950">
<svg
viewBox={`0 0 ${svgW} ${svgH}`}
viewBox={`0 0 ${SVG_W} ${svgH}`}
width="100%"
preserveAspectRatio="none"
style={{ display: 'block', minHeight: Math.min(svgH, 180), maxHeight: 220 }}
>
<rect width={svgW} height={svgH} fill="#06060f" />
<rect width={SVG_W} height={svgH} fill="#06060f" />
{/* Black-key row highlights */}
{Array.from({ length: range }, (_, i) => {
const p = maxP - i
return isBlack(p) ? (
<rect key={i} x={0} y={i * NOTE_PX} width={svgW} height={NOTE_PX} fill="rgba(255,255,255,0.025)" />
<rect key={i} x={0} y={i * NOTE_PX} width={SVG_W} height={NOTE_PX} fill="rgba(255,255,255,0.025)" />
) : null
})}
{/* C-note guide lines */}
{Array.from({ length: range }, (_, i) => {
const p = maxP - i
return p % 12 === 0 ? (
<line key={i} x1={0} y1={i * NOTE_PX} x2={svgW} y2={i * NOTE_PX} stroke="rgba(255,255,255,0.2)" strokeWidth={0.5} />
<line key={i} x1={0} y1={i * NOTE_PX} x2={SVG_W} y2={i * NOTE_PX} stroke="rgba(255,255,255,0.2)" strokeWidth={0.5} />
) : null
})}
{/* Notes */}
{allNotes.map((n: any, i: number) => (
<rect
key={i}
x={(n.time / totalTime) * svgW}
x={(n.time / totalTime) * SVG_W}
y={(maxP - n.midi) * NOTE_PX + 0.4}
width={Math.max((n.duration / totalTime) * svgW, 1.5)}
width={Math.max((n.duration / totalTime) * SVG_W, 1.5)}
height={NOTE_PX - 0.8}
fill={TRACK_COLORS[n.ti % TRACK_COLORS.length]}
rx={0.5}
@@ -137,6 +157,14 @@ function PianoRoll({ midi }: { midi: any }) {
<title>{n.name} {n.time.toFixed(2)}s</title>
</rect>
))}
{/* Playhead — position updated directly via ref by the RAF loop */}
<line
ref={playheadRef}
x1={0} y1={0} x2={0} y2={svgH}
stroke="rgba(255,255,255,0.75)"
strokeWidth={1.5}
style={{ display: 'none' }}
/>
</svg>
</div>
)
@@ -147,6 +175,68 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
const [midi, setMidi] = useState<any>(null)
const [loading, setLoading] = useState(false)
const [err, setErr] = useState<string | null>(null)
const [playing, setPlaying] = useState(false)
// 'idle' → first play loads sampler; 'loading' → fetching samples; 'ready' → can play
const [sfState, setSfState] = useState<'idle' | 'loading' | 'ready' | 'error'>('idle')
const ToneRef = useRef<any>(null)
const samplerRef = useRef<any>(null)
const partRef = useRef<any>(null)
const rafRef = useRef<number | null>(null)
const playheadRef = useRef<SVGLineElement | null>(null)
// Clean up Tone.js on unmount
useEffect(() => () => { cleanupAudio() }, [])
function cancelRaf() {
if (rafRef.current !== null) { cancelAnimationFrame(rafRef.current); rafRef.current = null }
}
function resetPlayhead() {
if (!playheadRef.current) return
playheadRef.current.style.display = 'none'
playheadRef.current.setAttribute('x1', '0')
playheadRef.current.setAttribute('x2', '0')
}
function cleanupAudio() {
cancelRaf()
try {
ToneRef.current?.Transport.stop()
ToneRef.current?.Transport.cancel()
partRef.current?.dispose(); partRef.current = null
samplerRef.current?.dispose(); samplerRef.current = null
} catch { /* ignore */ }
resetPlayhead()
setPlaying(false)
}
function startRaf(totalDuration: number) {
cancelRaf()
function loop() {
const t = ToneRef.current?.Transport.seconds ?? 0
if (playheadRef.current) {
const pct = Math.min(1, t / (totalDuration || 1))
playheadRef.current.style.display = 'block'
playheadRef.current.setAttribute('x1', String(pct * SVG_W))
playheadRef.current.setAttribute('x2', String(pct * SVG_W))
}
if (t >= totalDuration) { handleStop(); return }
rafRef.current = requestAnimationFrame(loop)
}
rafRef.current = requestAnimationFrame(loop)
}
function buildPart(Tone: any, midiData: any) {
const allNotes = midiData.tracks.flatMap((t: any) =>
t.notes.map((n: any) => ({ time: n.time, name: n.name, duration: n.duration, velocity: n.velocity }))
)
const part = new Tone.Part((time: number, note: any) => {
samplerRef.current?.triggerAttackRelease(note.name, note.duration, time, note.velocity)
}, allNotes.map((n: any) => [n.time, n]))
part.start(0)
return part
}
async function toggle() {
if (!open && midi === null && !err) {
@@ -162,7 +252,68 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
setLoading(false)
}
}
setOpen((v) => !v)
if (open) cleanupAudio()
setOpen(v => !v)
}
async function handlePlay() {
if (!midi) return
// First play: lazy-load Tone.js and fetch Salamander samples
if (sfState === 'idle') {
setSfState('loading')
try {
const Tone = await import('tone')
ToneRef.current = Tone
await Tone.start() // unlock AudioContext — must happen inside user gesture
const sampler = new Tone.Sampler({
urls: SALAMANDER_URLS,
baseUrl: SALAMANDER_BASE,
release: 1,
onload: () => {
setSfState('ready')
// Auto-start once all samples are buffered
Tone.Transport.start()
setPlaying(true)
startRaf(midi.duration || 1)
},
}).toDestination()
samplerRef.current = sampler
partRef.current = buildPart(Tone, midi)
} catch {
setSfState('error')
}
return
}
// Sampler already ready — resume or restart
const Tone = ToneRef.current
await Tone.start()
Tone.Transport.start()
setPlaying(true)
startRaf(midi.duration || 1)
}
function handlePause() {
ToneRef.current?.Transport.pause()
cancelRaf()
setPlaying(false)
}
function handleStop() {
cancelRaf()
const Tone = ToneRef.current
if (Tone && midi) {
Tone.Transport.stop()
Tone.Transport.cancel()
Tone.Transport.position = 0
// Rebuild the Part so the next play starts from the beginning
partRef.current?.dispose()
partRef.current = buildPart(Tone, midi)
}
resetPlayhead()
setPlaying(false)
}
const noteCount = midi?.tracks.reduce((s: number, t: any) => s + t.notes.length, 0) ?? 0
@@ -172,6 +323,7 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
return (
<div className="rounded-lg border border-purple-200/60 dark:border-purple-900/40 overflow-hidden">
{/* Toggle header */}
<button
onClick={toggle}
className="w-full flex items-center justify-between px-3 py-2 bg-purple-50/60 dark:bg-purple-950/20 text-xs font-medium text-purple-700 dark:text-purple-300 hover:bg-purple-100/60 dark:hover:bg-purple-950/40 transition-colors"
@@ -191,7 +343,7 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
<a
href={downloadUrl(jobId, file.name)}
download={file.name}
onClick={(e) => e.stopPropagation()}
onClick={e => e.stopPropagation()}
title={`Download ${file.name}`}
className="text-purple-400 hover:text-purple-600 dark:hover:text-purple-300 transition-colors"
>
@@ -204,13 +356,41 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
{open && (
err
? <p className="px-3 py-4 text-xs text-red-400 italic text-center bg-gray-950">{err}</p>
: midi
? <PianoRoll midi={midi} />
: (
<div className="flex items-center justify-center py-8 bg-gray-950">
<Loader2 className="w-5 h-5 animate-spin text-purple-400" />
</div>
)
: !midi
? <div className="flex items-center justify-center py-8 bg-gray-950"><Loader2 className="w-5 h-5 animate-spin text-purple-400" /></div>
: <>
<PianoRoll midi={midi} playheadRef={playheadRef} />
{/* Playback transport bar */}
<div className="flex items-center gap-2 px-3 py-2 bg-gray-900 border-t border-gray-800">
<button
onClick={playing ? handlePause : handlePlay}
disabled={sfState === 'error'}
className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-purple-600 hover:bg-purple-500 disabled:bg-gray-700 disabled:text-gray-500 text-white text-xs font-medium transition-colors min-w-[72px] justify-center"
>
{sfState === 'loading'
? <><Loader2 className="w-3 h-3 animate-spin" /> Loading</>
: playing
? <><Pause className="w-3 h-3" /> Pause</>
: <><Play className="w-3 h-3" /> Play</>}
</button>
{sfState === 'ready' && (
<button
onClick={handleStop}
title="Stop and rewind"
className="p-1.5 rounded-lg text-gray-500 hover:text-white hover:bg-gray-700 transition-colors"
>
<Square className="w-3 h-3" />
</button>
)}
<span className="ml-auto text-[10px] text-gray-600 select-none">
{sfState === 'idle' && 'Grand Piano · click Play to load'}
{sfState === 'loading' && 'Fetching piano samples…'}
{sfState === 'ready' && 'Grand Piano'}
{sfState === 'error' && 'Soundfont load failed'}
</span>
</div>
</>
)}
</div>
)
@@ -1028,13 +1208,15 @@ const WHISPER_LANGS = [
{ id: 'hi', label: 'Hindi' },
]
function LrcPreview({ file, jobId, originalAudioUrl, stems, title, storageKey }: {
function LrcPreview({ file, jobId, originalAudioUrl, stems, title, storageKey, coverArtUrl, artist }: {
file: JobFile
jobId: string
originalAudioUrl?: string
stems?: { name: string; label: string; url: string }[]
title?: string
storageKey?: string
coverArtUrl?: string
artist?: string
}) {
const [open, setOpen] = useState(false)
const [text, setText] = useState<string | null>(null)
@@ -1093,6 +1275,8 @@ function LrcPreview({ file, jobId, originalAudioUrl, stems, title, storageKey }:
audioUrl={(!stems || stems.length === 0) ? originalAudioUrl : undefined}
title={title}
storageKey={storageKey}
coverArtUrl={coverArtUrl}
artist={artist}
/>
)}
</div>
@@ -1144,6 +1328,8 @@ function StemLyricsRow({
whisperJob,
originalAudioUrl,
stems,
coverArtUrl,
artist,
onAction,
}: {
projectId: string
@@ -1153,6 +1339,8 @@ function StemLyricsRow({
whisperJob: Job | null
originalAudioUrl?: string
stems?: { name: string; label: string; url: string }[]
coverArtUrl?: string
artist?: string
onAction: () => void
}) {
const [mode, setMode] = useState<'align' | 'transcribe'>('transcribe')
@@ -1460,7 +1648,7 @@ function StemLyricsRow({
{activeJob.files
.filter((f) => f.name.endsWith('.elrc') || f.name.endsWith('.lrc') || f.name.endsWith('.sylrc'))
.map((f) => (
<LrcPreview key={f.name} file={f} jobId={activeJob.id} originalAudioUrl={originalAudioUrl} stems={stems} title={projectName} storageKey={`${activeJob.id}-${f.name}`} />
<LrcPreview key={f.name} file={f} jobId={activeJob.id} originalAudioUrl={originalAudioUrl} stems={stems} title={projectName} storageKey={`${activeJob.id}-${f.name}`} coverArtUrl={coverArtUrl} artist={artist} />
))}
{/* Inline transcript display for lyric-transcribe jobs */}
{activeJob.job_type === 'lyric-transcribe' && (
@@ -1622,6 +1810,8 @@ function LyricsSection({
whisperJob={lrcJobs[`${stemName}_whisper`] ?? null}
originalAudioUrl={originalAudioUrl}
stems={stemEntries.length > 0 ? stemEntries : undefined}
coverArtUrl={`/api/projects/${project.id}/cover-art`}
artist={project.source_artist}
onAction={onAction}
/>
)
+1
View File
@@ -164,6 +164,7 @@ export interface ProjectDetail {
mix_job_id: string | null
sep_job: Job
mix_job: Job | null
source_artist?: string
}
// Lyrics Style options