New set of open source fonts for display. Added "pop" display option to live lyric preview as well as an experimental hip-hop detection option for callbacks, responses, and sample detection.
This commit is contained in:
1 parent
86d182af41
commit
80a226fc3d
16 files changed
+764
-61
No files matched your search
+207
@@ -0,0 +1,207 @@
|
||||
# StemEngine — Session Summaries
|
||||
|
||||
---
|
||||
|
||||
## 2026-03-06
|
||||
|
||||
### Diarization fix — `whisperx.DiarizationPipeline` not exposed
|
||||
|
||||
**Problem:** `whisperx.DiarizationPipeline` raised `AttributeError` because this version of
|
||||
WhisperX uses a lazy-loader `__init__.py` that only re-exports a handful of functions
|
||||
(`load_model`, `align`, `assign_word_speakers`, etc.) and never added `DiarizationPipeline`
|
||||
to the list.
|
||||
|
||||
**Fix:** Import directly from the submodule:
|
||||
```python
|
||||
from whisperx.diarize import DiarizationPipeline as _DiarizationPipeline
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Diarization fix — `use_auth_token` renamed to `token`
|
||||
|
||||
**Problem:** After fixing the import, `DiarizationPipeline.__init__()` raised
|
||||
`TypeError: unexpected keyword argument 'use_auth_token'`. The installed version renamed the
|
||||
parameter to match the huggingface_hub convention change.
|
||||
|
||||
**Fix:** Changed constructor call to `_DiarizationPipeline(token=hf_token, device=device)`.
|
||||
|
||||
Also noted: this version defaults to `pyannote/speaker-diarization-community-1`, a community
|
||||
model that does not require accepting HuggingFace terms of use.
|
||||
|
||||
---
|
||||
|
||||
### Diarization fix — speaker labels not propagated from words to segments
|
||||
|
||||
**Problem:** `whisperx.assign_word_speakers` in some versions assigns `"speaker"` only to
|
||||
individual word dicts, not to the parent segment dict. `seg.get("speaker")` returned `None`
|
||||
so `|sp:SPEAKER_XX|` tags were never emitted in LRC/ELRC output.
|
||||
|
||||
**Fix (already applied in prior session):** After `assign_word_speakers`, iterate segments
|
||||
and propagate the most-frequent word-level speaker to each segment that lacks one:
|
||||
```python
|
||||
for _seg in wx_result.get("segments", []):
|
||||
if _seg.get("speaker"):
|
||||
continue
|
||||
_spk_counts = {}
|
||||
for _w in _seg.get("words", []):
|
||||
_s = _w.get("speaker")
|
||||
if _s:
|
||||
_spk_counts[_s] = _spk_counts.get(_s, 0) + 1
|
||||
if _spk_counts:
|
||||
_seg["speaker"] = max(_spk_counts, key=_spk_counts.get)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Timing fix — unaligned words all sharing `seg["start"]`
|
||||
|
||||
**Problem:** When WhisperX cannot align individual words (returns `start=None`), all such
|
||||
words in a segment received `seg["start"]` as their timestamp. In the frontend
|
||||
`wordFillPct()` function, consecutive words with identical timestamps produce `dur = 0`
|
||||
→ `NaN` fill percentage → broken karaoke highlight animation.
|
||||
|
||||
**Fix:** Distribute unaligned words evenly across `[seg_start, seg_end]` by index:
|
||||
```python
|
||||
if _ws is None:
|
||||
_ws = _seg_start + (_wi / _n_words) * (_seg_end - _seg_start)
|
||||
_we = _seg_start + ((_wi + 1) / _n_words) * (_seg_end - _seg_start)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### CUDA OOM — retry with reduced settings
|
||||
|
||||
**Problem:** Loading `large`, `large-v2`, or `large-v3` models on cards with limited VRAM
|
||||
raised `RuntimeError: CUDA out of memory`, failing the job entirely.
|
||||
|
||||
**Fix:** Wrapped `load_model` + `transcribe` in a 2-attempt retry loop. On first OOM:
|
||||
- Clear CUDA cache (`gc.collect()` + `torch.cuda.empty_cache()`)
|
||||
- Reduce `batch_size` to 4
|
||||
- Switch `compute_type` to `"int8"`
|
||||
- Update job progress to inform the user
|
||||
- Retry
|
||||
|
||||
On second OOM or any non-OOM error, re-raise. Also added `_clear_cuda_cache()` at job start
|
||||
to flush leftover allocations from previous jobs.
|
||||
|
||||
Added user-friendly error message on OOM failure suggesting smaller model or Eco preset.
|
||||
|
||||
---
|
||||
|
||||
### WSL2 display stability — VRAM cap + CPU niceness + inter-phase yields
|
||||
|
||||
**Problem:** On WSL2, WhisperX saturates the shared GPU, starving the Windows Display
|
||||
Manager (DWM). This causes TDR timeouts, screen flicker, and display driver restarts.
|
||||
|
||||
**Fixes added to `lyric_transcriber_service.py`:**
|
||||
|
||||
1. **`_apply_cuda_limits()`** called at job start:
|
||||
- `os.nice(10)` — lowers the job thread's CPU scheduling priority
|
||||
- `torch.cuda.set_per_process_memory_fraction(fraction)` — hard-caps VRAM usage,
|
||||
reserving headroom for DWM. Default 80%, configurable via
|
||||
`STEMENGINE_GPU_MEMORY_FRACTION` env var.
|
||||
|
||||
2. **`time.sleep(0.15)`** inserted after each model-unload+cache-clear between the three
|
||||
heavy GPU phases (Whisper → alignment → diarization), giving the GPU scheduler a window
|
||||
to service DWM between compute bursts.
|
||||
|
||||
**System-level recommendation** (Windows host, not in code):
|
||||
- Create `%UserProfile%\.wslconfig`:
|
||||
```ini
|
||||
[wsl2]
|
||||
memory=8GB
|
||||
processors=6
|
||||
```
|
||||
- Set `STEMENGINE_GPU_MEMORY_FRACTION=0.75` in Docker compose or `.env` for tighter cards.
|
||||
|
||||
---
|
||||
|
||||
## 2026-03-06 (continued)
|
||||
|
||||
### Syllables checkbox + hide Phonemization in Auto Pipeline
|
||||
|
||||
Added `lsyncAlign` checkbox ("Syllables") to Auto Pipeline options (`Settings.tsx`, `AutoPipelineDialog.tsx`).
|
||||
Removed Phonemization from both `ProjectDetail.tsx` (Lyrics Sync tab) and `Settings.tsx` (Auto Pipeline step editor) — state kept for API compatibility.
|
||||
|
||||
---
|
||||
|
||||
### Explicit mode — Whisper profanity uncensor
|
||||
|
||||
Added `explicit: bool` to `LyricTranscribeRequest`, `run_lyric_transcription()`, and frontend (`ProjectDetail.tsx`, `AutoPipelineDialog.tsx`, `client.ts`).
|
||||
|
||||
`_uncensor_segments()` in `lyric_transcriber_service.py` applies regex post-processing after transcription to restore censored words (`f***` → `fuck`, etc.). Initial-prompt primer approach was discarded — it caused Whisper to reproduce the prompt text verbatim at the start of the transcription.
|
||||
|
||||
---
|
||||
|
||||
### Hip-hop vocal style
|
||||
|
||||
Added `vocal_style: str = "general"` parameter throughout the stack. When `"hiphop"`:
|
||||
- VAD thresholds: `vad_onset=0.25`, `vad_offset=0.15`, `no_speech_threshold=0.35`, `hallucination_silence_threshold=1.0`
|
||||
- Monkeypatches `Silero.__call__` in try/finally to inject `min_speech_duration_ms=100, min_silence_duration_ms=150` (these params are commented out in whisperx's wrapper, so they must be injected at call time).
|
||||
|
||||
---
|
||||
|
||||
### GlobalAudioBar fixes + X button
|
||||
|
||||
- **Bar not appearing**: `isActiveRef` is a ref so changing it doesn't trigger effects. Fixed by calling `publishMeta(...)` immediately after `isActiveRef.current = true` in both `DynamicMixPlayer` and `LyricsLivePreview` `togglePlay` branches.
|
||||
- **Audio continuing after navigation**: `unregisterPlayback` only removes the registry reference, not the audio element. Fixed by calling `stopPlayback()` before `unregisterPlayback()` in unmount cleanup of both players.
|
||||
- **X button**: Added `stopCurrent()` export to `audioRegistry.ts` (calls registered stop callback + clears `_current`). Added X (`lucide-react`) button to `GlobalAudioBar.tsx`.
|
||||
|
||||
---
|
||||
|
||||
### Karaoke fill/pop mode + Open Foundry fonts
|
||||
|
||||
**`LyricsLivePreview.tsx`**:
|
||||
- Added `karaokeMode: 'fill' | 'pop'` to `StyleState` (default `'fill'`).
|
||||
- `KaraokeWord` now accepts `isActive?: boolean` and `mode?: 'fill' | 'pop'`:
|
||||
- `fill`: existing left-to-right gradient sweep.
|
||||
- `pop`: active word bright + 15% larger; already-sung words stay bright; future words dim. Line accumulates highlighted words as sung — no gradient, no fill.
|
||||
- All three `KaraokeWord` call sites (karaoke, glow, blur styles) pass `isActive` and `mode`.
|
||||
- Added Fill/Pop toggle buttons to `StylePicker`.
|
||||
- Replaced `FONT_OPTIONS` with 10 Open Foundry fonts (Archivo, EB Garamond, Inter, Libre Baskerville, Oswald, Poppins, Roboto, Work Sans, Young Serif + System fallback).
|
||||
|
||||
**`package.json`**: Added `@fontsource/*` packages for all 9 named fonts — self-hosted, no Google CDN calls. Installed automatically by `start.sh` at container start.
|
||||
|
||||
**`main.tsx`**: Imports each `@fontsource/*` package so Vite bundles the font files.
|
||||
|
||||
---
|
||||
|
||||
### Font dropdown dark background + bold toggle
|
||||
|
||||
- **Dropdown white background fix**: Added `colorScheme: 'dark'` on the `<select>` and `background: '#1e293b', color: '#e2e8f0'` on each `<option>` so the native OS dropdown renders dark.
|
||||
- **Bold toggle**: Added `fontBold: boolean` to `StyleState` (default `false`). **B** button in `StylePicker` next to the font selector. Applied as `fontWeight: 700 / 400` on the lyrics container so it cascades to all rendered text.
|
||||
|
||||
---
|
||||
|
||||
### Pop mode rendering bug fix
|
||||
|
||||
**Problem:** In pop mode, `WebkitTextFillColor: 'transparent'` set by a previous fill-mode render persisted on the DOM element (React doesn't always clear vendor-prefixed style properties when they're removed from the style object). This made text invisible while the gradient background showed as a colored rectangle.
|
||||
|
||||
**Fix:** In the `KaraokeWord` pop-mode branch, explicitly reset all gradient/clip properties:
|
||||
```typescript
|
||||
backgroundImage: 'none',
|
||||
WebkitBackgroundClip: 'border-box',
|
||||
WebkitTextFillColor: 'currentColor', // key: restores text color rendering
|
||||
backgroundClip: 'border-box',
|
||||
```
|
||||
`WebkitTextFillColor: 'currentColor'` makes the text use the CSS `color` property again, which is what pop mode sets per-word.
|
||||
|
||||
---
|
||||
|
||||
### Pop mode — RAF loop still writing gradient to DOM
|
||||
|
||||
**Problem:** Even after the React CSS resets in `KaraokeWord`, a fill bar kept appearing in pop mode. The real cause: a `requestAnimationFrame` loop writes `backgroundImage` directly to `activeWordSpanRef.current.style` every frame (to avoid 60fps React re-renders). This direct DOM write completely bypassed React's style reconciliation and overwrote the pop-mode resets on every tick.
|
||||
|
||||
**Fix:**
|
||||
- Added `karaokeModeRef = useRef(karaokeMode)` alongside the other RAF refs.
|
||||
- Updated it inline each render: `karaokeModeRef.current = karaokeMode` (outside any effect, so it's always current without restarting the loop).
|
||||
- Gated the gradient write in the RAF loop: `if (activeWordSpanRef.current && lastLineIdx >= 0 && karaokeModeRef.current !== 'pop')`.
|
||||
|
||||
---
|
||||
|
||||
### Pop mode — done words should stay bright
|
||||
|
||||
Corrected pop mode behavior: sung (done) words stay bright, not dim. Only future words are dim. Active word is bright + 15% larger.
|
||||
|
||||
Changed `color: isActive ? bright : dim` → `color: (isActive || done) ? bright : dim` in `KaraokeWord`.
|
||||
@@ -80,6 +80,8 @@ class LyricTranscribeRequest(BaseModel):
|
||||
phoneme_align: bool = False # use phonemizer + phoneme model for syllable-level timestamps
|
||||
lsync_align: bool = False # use lsync wav2vec2 CTC + pyphen for syllable-level timestamps
|
||||
two_pass: bool = False # run a second VAD pass to fill gaps > 2 s
|
||||
vocal_style: str = "general" # 'general' | 'hiphop' — genre-specific VAD/ASR tuning
|
||||
explicit: bool = False # disable Whisper's profanity filter via initial_prompt primer
|
||||
lyrics_source: str = "transcribe" # 'transcribe' | 'auto' | 'tag' | 'file' — when not 'transcribe',
|
||||
# skip Whisper and force-align existing lyrics instead
|
||||
existing_lyrics: Optional[str] = None # lyrics text (plain or LRC) to align; overrides lyrics_source
|
||||
@@ -504,6 +506,8 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
|
||||
lsync_align=body.lsync_align,
|
||||
two_pass=body.two_pass,
|
||||
existing_lyrics=resolved_lyrics,
|
||||
vocal_style=body.vocal_style,
|
||||
explicit=body.explicit,
|
||||
)
|
||||
return {"job_id": job_id, "output_dir": str(output_dir)}
|
||||
|
||||
|
||||
@@ -856,6 +856,108 @@ _LSYNC_DIR = str(Path(__file__).parent.parent.parent / "lyrics-sync")
|
||||
_lsync_recognizer: dict = {} # device → cached PhonemeRecognizer
|
||||
|
||||
|
||||
def _clear_cuda_cache() -> None:
|
||||
"""Release all unreferenced CUDA tensors and empty the allocator cache."""
|
||||
try:
|
||||
import gc, torch
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Explicit-mode post-processing
|
||||
# ---------------------------------------------------------------------------
|
||||
# Whisper censors with asterisks when confidence is low or context is
|
||||
# ambiguous. Rather than poisoning initial_prompt (which bleeds into the
|
||||
# transcription output), we fix the artefacts after the fact.
|
||||
#
|
||||
# Patterns use a character class that matches *, •, ·, and common Unicode
|
||||
# replacements so we catch "f**k", "f••k", "f*ck", etc.
|
||||
import re as _re
|
||||
|
||||
_C = r'[*•·\u2022\u00b7]' # census-character class
|
||||
|
||||
_UNCENSOR_PATTERNS: list = [
|
||||
# fuck / fucking / fucked
|
||||
(_re.compile(rf'\bf{_C}{{2,3}}k\b', _re.I), 'fuck'),
|
||||
(_re.compile(rf'\bf{_C}{{3}}ing\b', _re.I), 'fucking'),
|
||||
(_re.compile(rf'\bf{_C}{{3}}ed\b', _re.I), 'fucked'),
|
||||
(_re.compile(rf'\bf{_C}{{3}}er\b', _re.I), 'fucker'),
|
||||
# shit / shitting
|
||||
(_re.compile(rf'\bs{_C}{{2}}t\b', _re.I), 'shit'),
|
||||
(_re.compile(rf'\bs{_C}{{6}}ing\b', _re.I), 'shitting'),
|
||||
# bitch / bitches
|
||||
(_re.compile(rf'\bb{_C}{{3}}h\b', _re.I), 'bitch'),
|
||||
(_re.compile(rf'\bb{_C}{{5}}s\b', _re.I), 'bitches'),
|
||||
# ass / asses
|
||||
(_re.compile(rf'\ba{_C}{{2}}\b', _re.I), 'ass'),
|
||||
(_re.compile(rf'\ba{_C}{{4}}s\b', _re.I), 'asses'),
|
||||
# damn / damned
|
||||
(_re.compile(rf'\bd{_C}{{2}}n\b', _re.I), 'damn'),
|
||||
(_re.compile(rf'\bd{_C}{{4}}d\b', _re.I), 'damned'),
|
||||
# hell (less common but Whisper censors it in some contexts)
|
||||
(_re.compile(rf'\bh{_C}{{2}}l\b', _re.I), 'hell'),
|
||||
# dick
|
||||
(_re.compile(rf'\bd{_C}{{2}}k\b', _re.I), 'dick'),
|
||||
# cock
|
||||
(_re.compile(rf'\bc{_C}{{2}}k\b', _re.I), 'cock'),
|
||||
# pussy
|
||||
(_re.compile(rf'\bp{_C}{{4}}y\b', _re.I), 'pussy'),
|
||||
# cunt
|
||||
(_re.compile(rf'\bc{_C}{{2}}t\b', _re.I), 'cunt'),
|
||||
# whore
|
||||
(_re.compile(rf'\bwh{_C}{{2}}e\b', _re.I), 'whore'),
|
||||
# bastard
|
||||
(_re.compile(rf'\bb{_C}{{6}}d\b', _re.I), 'bastard'),
|
||||
# motherfucker (various lengths)
|
||||
(_re.compile(rf'\bm{_C}{{11}}r\b', _re.I), 'motherfucker'),
|
||||
(_re.compile(rf'\bmother{_C}{{4}}er\b', _re.I), 'motherfucker'),
|
||||
]
|
||||
|
||||
|
||||
def _uncensor_segments(segments: list) -> None:
|
||||
"""Replace asterisk-censored words in segment text and word dicts in-place."""
|
||||
for seg in segments:
|
||||
for pat, rep in _UNCENSOR_PATTERNS:
|
||||
seg["text"] = pat.sub(rep, seg["text"])
|
||||
for w in seg.get("words", []):
|
||||
txt = w.get("word", "")
|
||||
for pat, rep in _UNCENSOR_PATTERNS:
|
||||
txt = pat.sub(rep, txt)
|
||||
w["word"] = txt
|
||||
|
||||
|
||||
# Fraction of total GPU VRAM this process is allowed to use.
|
||||
# Keeping this below 1.0 on WSL2 reserves headroom for the Windows display
|
||||
# compositor (DWM), preventing TDR timeouts that cause screen flicker.
|
||||
# Override with the STEMENGINE_GPU_MEMORY_FRACTION env var (0.0–1.0).
|
||||
_GPU_MEMORY_FRACTION: float = float(os.environ.get("STEMENGINE_GPU_MEMORY_FRACTION", "0.80"))
|
||||
|
||||
|
||||
def _apply_cuda_limits() -> None:
|
||||
"""Cap per-process VRAM and lower CPU niceness for display-friendly operation.
|
||||
|
||||
Safe to call multiple times — torch guards against re-setting the fraction
|
||||
on an already-initialised context, and os.nice is a no-op when already niced.
|
||||
"""
|
||||
# Lower CPU scheduling priority so Windows UI processes aren't starved.
|
||||
try:
|
||||
import os as _os
|
||||
_os.nice(10)
|
||||
except Exception:
|
||||
pass
|
||||
# Reserve a fraction of VRAM for the Windows display driver / other processes.
|
||||
try:
|
||||
import torch as _t
|
||||
if _t.cuda.is_available() and 0.0 < _GPU_MEMORY_FRACTION < 1.0:
|
||||
_t.cuda.set_per_process_memory_fraction(_GPU_MEMORY_FRACTION)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _lsync_syllabify(segments: list, audio_arr, device: str) -> bool:
|
||||
"""Add syllable timestamps to words using lsync CTC forced alignment.
|
||||
|
||||
@@ -1170,6 +1272,8 @@ def run_lyric_transcription(
|
||||
lsync_align: bool = False,
|
||||
two_pass: bool = False,
|
||||
existing_lyrics: Optional[str] = None,
|
||||
vocal_style: str = "general",
|
||||
explicit: bool = False,
|
||||
):
|
||||
"""
|
||||
Background task: transcribe vocals/lyrics from audio.
|
||||
@@ -1225,6 +1329,11 @@ def run_lyric_transcription(
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
log.debug("[%s] device=%s power_preset=%s", job_id, device, power_preset)
|
||||
# Flush VRAM and apply resource limits before loading any models.
|
||||
# On WSL2 this keeps the Windows display compositor (DWM) from
|
||||
# starving — see STEMENGINE_GPU_MEMORY_FRACTION env var (default 0.80).
|
||||
_apply_cuda_limits()
|
||||
_clear_cuda_cache()
|
||||
_wx_preset = _WHISPERX_POWER_PRESETS.get(power_preset, _WHISPERX_POWER_PRESETS['standard'])
|
||||
batch_size = _wx_preset['batch_size']
|
||||
compute_type = _wx_preset['compute_type_cuda'] if device == "cuda" else "int8"
|
||||
@@ -1255,6 +1364,47 @@ def run_lyric_transcription(
|
||||
"vad_offset": 0.200,
|
||||
}
|
||||
|
||||
# Hip-hop tuning: WhisperX's Silero VAD wrapper hard-codes
|
||||
# min_speech_duration_ms=250 (default), which silently drops
|
||||
# ad-libs, call phrases, and short responses. We patch Silero.__call__
|
||||
# in a try/finally to inject a 100 ms minimum and also tighten the
|
||||
# ASR thresholds for denser, faster vocal delivery.
|
||||
_silero_orig_call = None
|
||||
if vocal_style == "hiphop":
|
||||
_asr_opts["no_speech_threshold"] = 0.35
|
||||
_asr_opts["hallucination_silence_threshold"] = 1.0
|
||||
if not initial_prompt:
|
||||
_asr_opts["initial_prompt"] = (
|
||||
"Hip-hop rap lyrics with ad-libs, hooks, "
|
||||
"call and response, and slang."
|
||||
)
|
||||
_vad_opts["vad_onset"] = 0.25
|
||||
_vad_opts["vad_offset"] = 0.15
|
||||
# Patch Silero to accept min_speech_duration_ms=100 ms so
|
||||
# short utterances aren't filtered before Whisper sees them.
|
||||
try:
|
||||
import whisperx.vads.silero as _silero_mod
|
||||
from whisperx.diarize import Segment as _SileroSeg
|
||||
_silero_orig_call = _silero_mod.Silero.__call__
|
||||
def _hiphop_silero_call(_self, audio, **_kw):
|
||||
sr = audio["sample_rate"]
|
||||
if sr != 16000:
|
||||
raise ValueError("Only 16000Hz sample rate is allowed")
|
||||
ts = _self.get_speech_timestamps(
|
||||
audio["waveform"],
|
||||
model=_self.vad_pipeline,
|
||||
sampling_rate=sr,
|
||||
max_speech_duration_s=_self.chunk_size,
|
||||
threshold=_self.vad_onset,
|
||||
min_speech_duration_ms=100,
|
||||
min_silence_duration_ms=150,
|
||||
)
|
||||
return [_SileroSeg(i['start'] / sr, i['end'] / sr, "UNKNOWN") for i in ts]
|
||||
_silero_mod.Silero.__call__ = _hiphop_silero_call
|
||||
except Exception as _patch_err:
|
||||
log.warning("[%s] Silero patch failed (%s) — using default VAD", job_id, _patch_err)
|
||||
_silero_orig_call = None
|
||||
|
||||
import gc
|
||||
|
||||
audio_arr = whisperx.load_audio(audio_path)
|
||||
@@ -1279,20 +1429,50 @@ def run_lyric_transcription(
|
||||
_result_model_tag = "whisperx-align"
|
||||
else:
|
||||
# ── Normal transcription path ──────────────────────────────────
|
||||
wx_model = whisperx.load_model(
|
||||
model, device,
|
||||
compute_type=compute_type,
|
||||
language=language or None,
|
||||
asr_options=_asr_opts,
|
||||
vad_options=_vad_opts,
|
||||
)
|
||||
# Attempt transcription; on CUDA OOM retry once with a smaller
|
||||
# batch and int8 quantisation to fit tighter VRAM budgets.
|
||||
for _tx_attempt in range(2):
|
||||
try:
|
||||
wx_model = whisperx.load_model(
|
||||
model, device,
|
||||
compute_type=compute_type,
|
||||
language=language or None,
|
||||
asr_options=_asr_opts,
|
||||
vad_options=_vad_opts,
|
||||
)
|
||||
|
||||
jm.update_job(job_id, progress=f"Transcribing with WhisperX ({model}, {power_preset})...")
|
||||
wx_result = wx_model.transcribe(audio_arr, batch_size=batch_size)
|
||||
break # success — exit retry loop
|
||||
except RuntimeError as _oom_err:
|
||||
_is_oom = "out of memory" in str(_oom_err).lower()
|
||||
if _tx_attempt == 0 and _is_oom and device == "cuda":
|
||||
log.warning(
|
||||
"[%s] CUDA OOM on attempt 1 (batch=%d compute=%s) — "
|
||||
"clearing cache and retrying with batch=4 int8",
|
||||
job_id, batch_size, compute_type,
|
||||
)
|
||||
try:
|
||||
del wx_model
|
||||
except NameError:
|
||||
pass
|
||||
_clear_cuda_cache()
|
||||
batch_size = 4
|
||||
compute_type = "int8"
|
||||
jm.update_job(
|
||||
job_id,
|
||||
progress=f"CUDA out of memory — retrying with reduced settings (batch=4, int8)...",
|
||||
)
|
||||
else:
|
||||
raise
|
||||
|
||||
jm.update_job(job_id, progress=f"Transcribing with WhisperX ({model}, {power_preset})...")
|
||||
wx_result = wx_model.transcribe(audio_arr, batch_size=batch_size)
|
||||
detected_lang = wx_result.get("language") or language or "unknown"
|
||||
log.debug("[%s] transcription done — lang=%s segments=%d",
|
||||
job_id, detected_lang, len(wx_result.get("segments", [])))
|
||||
|
||||
if explicit:
|
||||
_uncensor_segments(wx_result.get("segments", []))
|
||||
|
||||
if two_pass:
|
||||
# Re-transcribe any gaps > 2 s that VAD may have missed
|
||||
wx_result["segments"] = _fill_transcript_gaps(
|
||||
@@ -1307,8 +1487,20 @@ def run_lyric_transcription(
|
||||
gc.collect()
|
||||
if device == "cuda":
|
||||
torch.cuda.empty_cache()
|
||||
# Brief pause between heavy GPU phases so the Windows display
|
||||
# compositor gets a scheduling slot (WSL2 WDDM shared GPU).
|
||||
import time as _time; _time.sleep(0.15)
|
||||
_result_model_tag = f"whisperx-{model}"
|
||||
|
||||
# Restore Silero if it was patched for hip-hop mode.
|
||||
if _silero_orig_call is not None:
|
||||
try:
|
||||
import whisperx.vads.silero as _silero_mod
|
||||
_silero_mod.Silero.__call__ = _silero_orig_call
|
||||
except Exception:
|
||||
pass
|
||||
_silero_orig_call = None
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
@@ -1381,6 +1573,7 @@ def run_lyric_transcription(
|
||||
gc.collect()
|
||||
if device == "cuda":
|
||||
torch.cuda.empty_cache()
|
||||
import time as _time; _time.sleep(0.15)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
@@ -1403,10 +1596,26 @@ def run_lyric_transcription(
|
||||
_hf_login(token=hf_token, add_to_git_credential=False)
|
||||
except Exception:
|
||||
pass
|
||||
diarize_model = whisperx.DiarizationPipeline(use_auth_token=hf_token, device=device)
|
||||
from whisperx.diarize import DiarizationPipeline as _DiarizationPipeline
|
||||
diarize_model = _DiarizationPipeline(token=hf_token, device=device)
|
||||
diarize_segments = diarize_model(audio_arr)
|
||||
wx_result = whisperx.assign_word_speakers(diarize_segments, wx_result)
|
||||
# Some WhisperX versions assign speaker only to words, not segments.
|
||||
# Propagate: pick the most frequent word-level speaker for each segment.
|
||||
for _seg in wx_result.get("segments", []):
|
||||
if _seg.get("speaker"):
|
||||
continue # already set by this whisperx version
|
||||
_spk_counts: dict = {}
|
||||
for _w in _seg.get("words", []):
|
||||
_s = _w.get("speaker")
|
||||
if _s:
|
||||
_spk_counts[_s] = _spk_counts.get(_s, 0) + 1
|
||||
if _spk_counts:
|
||||
_seg["speaker"] = max(_spk_counts, key=_spk_counts.get)
|
||||
except Exception as _diar_err:
|
||||
import traceback as _tb
|
||||
log.warning("[%s] diarization failed: %s\n%s",
|
||||
job_id, _diar_err, _tb.format_exc())
|
||||
jm.update_job(job_id, progress=f"Diarization failed ({_diar_err}), continuing without speaker labels...")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
@@ -1421,13 +1630,25 @@ def run_lyric_transcription(
|
||||
segments: List[Segment] = []
|
||||
for seg in raw_segments:
|
||||
words = []
|
||||
for w in seg.get("words", []):
|
||||
_seg_start = float(seg["start"])
|
||||
_seg_end = float(seg["end"])
|
||||
_seg_words = seg.get("words", [])
|
||||
_n_words = len(_seg_words)
|
||||
for _wi, w in enumerate(_seg_words):
|
||||
_ws = w.get("start")
|
||||
_we = w.get("end")
|
||||
if _ws is None:
|
||||
# Distribute unaligned words evenly across the segment
|
||||
# so consecutive words have distinct timestamps and
|
||||
# wordFillPct never computes dur=0 → NaN.
|
||||
_ws = _seg_start + (_wi / _n_words) * (_seg_end - _seg_start)
|
||||
_we = _seg_start + ((_wi + 1) / _n_words) * (_seg_end - _seg_start)
|
||||
elif _we is None:
|
||||
_we = _seg_end
|
||||
words.append(WordTimestamp(
|
||||
word=w.get("word", ""),
|
||||
start=float(_ws if _ws is not None else seg["start"]),
|
||||
end=float(_we if _we is not None else seg["end"]),
|
||||
start=float(_ws),
|
||||
end=float(_we),
|
||||
confidence=w.get("score"),
|
||||
phonemes=w.get("phonemes"),
|
||||
syllables=w.get("syllables"),
|
||||
@@ -1597,17 +1818,28 @@ def run_lyric_transcription(
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
log.debug("[%s] failed — %s: %s", job_id, type(exc).__name__, exc)
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
# Best-effort GPU flush so a failed job doesn't strand VRAM for the next
|
||||
# queued job. gc.collect() releases live tensor refs before empty_cache()
|
||||
# can reclaim the CUDA memory they hold.
|
||||
_is_oom = "out of memory" in str(exc).lower()
|
||||
if _is_oom:
|
||||
log.warning("[%s] CUDA out of memory: %s", job_id, exc)
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error=(
|
||||
"CUDA out of memory. Try a smaller model (medium / small), "
|
||||
"switch to the Eco power preset, or disable Phonemes/Syllables."
|
||||
),
|
||||
progress="Error.",
|
||||
)
|
||||
else:
|
||||
log.debug("[%s] failed — %s: %s", job_id, type(exc).__name__, exc)
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
# Always flush VRAM after a failure so the next queued job starts clean.
|
||||
_clear_cuda_cache()
|
||||
# Restore Silero if a hip-hop patch was applied before the failure.
|
||||
try:
|
||||
import gc
|
||||
gc.collect()
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
if _silero_orig_call is not None:
|
||||
import whisperx.vads.silero as _silero_mod
|
||||
_silero_mod.Silero.__call__ = _silero_orig_call
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="StemEngine" />
|
||||
<link rel="apple-touch-icon" href="/icon.svg" />
|
||||
<script type="module" crossorigin src="/assets/index-CkVcWz3N.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DVL4P0ML.css">
|
||||
<script type="module" crossorigin src="/assets/index-BRW-p_r8.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DBHjVSaY.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
Generated
+90
@@ -8,6 +8,15 @@
|
||||
"name": "stemengine-frontend",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@fontsource/archivo": "^5.0.0",
|
||||
"@fontsource/eb-garamond": "^5.0.0",
|
||||
"@fontsource/inter": "^5.0.0",
|
||||
"@fontsource/libre-baskerville": "^5.0.0",
|
||||
"@fontsource/oswald": "^5.0.0",
|
||||
"@fontsource/poppins": "^5.0.0",
|
||||
"@fontsource/roboto": "^5.0.0",
|
||||
"@fontsource/work-sans": "^5.0.0",
|
||||
"@fontsource/young-serif": "^5.0.0",
|
||||
"@tanstack/react-query": "^5.0.0",
|
||||
"@tonejs/midi": "^2.0.28",
|
||||
"abcjs": "^6.6.2",
|
||||
@@ -679,6 +688,87 @@
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/archivo": {
|
||||
"version": "5.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/archivo/-/archivo-5.2.8.tgz",
|
||||
"integrity": "sha512-Fc4yuMt85MmcYooUVpYHAJRJTymPv2+J9HsMP2f5mxR/0pIWgE2V5XfgNTavPpklX2rJFprdT36iE6Rr8ds9MA==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/eb-garamond": {
|
||||
"version": "5.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/eb-garamond/-/eb-garamond-5.2.7.tgz",
|
||||
"integrity": "sha512-V42tTlHDbnDo0+lENnXKWMx63Llq6Gfl2l7ozkoeRQN60S0jm6hEgCLlOT/5YyKAN9QZ0e2ofpy0rGKlz9jBrw==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/inter": {
|
||||
"version": "5.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/inter/-/inter-5.2.8.tgz",
|
||||
"integrity": "sha512-P6r5WnJoKiNVV+zvW2xM13gNdFhAEpQ9dQJHt3naLvfg+LkF2ldgSLiF4T41lf1SQCM9QmkqPTn4TH568IRagg==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/libre-baskerville": {
|
||||
"version": "5.2.10",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/libre-baskerville/-/libre-baskerville-5.2.10.tgz",
|
||||
"integrity": "sha512-Y/bGj4cOdWhAxU4+sBPen3MB5Og3t/WvezVL+05CJ7cvhDQUKpQFUuw/hm1IQKRc9u0lWs8VP2MKDOK2uQtloQ==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/oswald": {
|
||||
"version": "5.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/oswald/-/oswald-5.2.8.tgz",
|
||||
"integrity": "sha512-DFoPK1BqsFIQhVjs2xgO/oL9veF9oUJgFojEWIU+nazUrfqsBDtAMI4n9L9CZFX3kQNDLcCyzKx1xoEDvQT4WQ==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/poppins": {
|
||||
"version": "5.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/poppins/-/poppins-5.2.7.tgz",
|
||||
"integrity": "sha512-6uQyPmseo4FgI97WIhA4yWRlNaoLk4vSDK/PyRwdqqZb5zAEuc+Kunt8JTMcsHYUEGYBtN15SNkMajMdqUSUmg==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/roboto": {
|
||||
"version": "5.2.10",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/roboto/-/roboto-5.2.10.tgz",
|
||||
"integrity": "sha512-8HlA5FtSfz//oFSr2eL7GFXAiE7eIkcGOtx7tjsLKq+as702x9+GU7K95iDeWFapHC4M2hv9RrpXKRTGGBI8Zg==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/work-sans": {
|
||||
"version": "5.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/work-sans/-/work-sans-5.2.8.tgz",
|
||||
"integrity": "sha512-6LaHjVVgts+rnrcqvEkP2+iUB/jw1oDSYsGO0+TltAhnWki9Hnf/UGpgMQh2jcm0GEH8VqCPnq4PpmHLFzxXtQ==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@fontsource/young-serif": {
|
||||
"version": "5.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@fontsource/young-serif/-/young-serif-5.2.7.tgz",
|
||||
"integrity": "sha512-BltTmAlXLWQpu9XCQa+dfDJHQXt0EHUA4h9Q53VToDW3Onq0dkWzCPM1EyhinqfuwD9tb/5cEcGoRcRwiyiUUA==",
|
||||
"license": "OFL-1.1",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ayuhito"
|
||||
}
|
||||
},
|
||||
"node_modules/@jridgewell/gen-mapping": {
|
||||
"version": "0.3.13",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
|
||||
|
||||
@@ -9,6 +9,15 @@
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fontsource/archivo": "^5.0.0",
|
||||
"@fontsource/eb-garamond": "^5.0.0",
|
||||
"@fontsource/inter": "^5.0.0",
|
||||
"@fontsource/libre-baskerville": "^5.0.0",
|
||||
"@fontsource/oswald": "^5.0.0",
|
||||
"@fontsource/poppins": "^5.0.0",
|
||||
"@fontsource/roboto": "^5.0.0",
|
||||
"@fontsource/work-sans": "^5.0.0",
|
||||
"@fontsource/young-serif": "^5.0.0",
|
||||
"@tanstack/react-query": "^5.0.0",
|
||||
"@tonejs/midi": "^2.0.28",
|
||||
"abcjs": "^6.6.2",
|
||||
|
||||
@@ -391,6 +391,8 @@ export const lyricTranscribeProject = (
|
||||
phonemeAlign?: boolean
|
||||
lsyncAlign?: boolean
|
||||
twoPass?: boolean
|
||||
vocalStyle?: string
|
||||
explicit?: boolean
|
||||
}
|
||||
) => {
|
||||
const wordTs = opts?.wordTimestamps ?? true
|
||||
@@ -418,6 +420,8 @@ export const lyricTranscribeProject = (
|
||||
phoneme_align: phonemeAlign,
|
||||
lsync_align: opts?.lsyncAlign ?? false,
|
||||
two_pass: opts?.twoPass ?? false,
|
||||
vocal_style: opts?.vocalStyle ?? 'general',
|
||||
explicit: opts?.explicit ?? false,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -184,8 +184,10 @@ export default function AutoPipelineDialog({ target, onClose, onComplete, onOpen
|
||||
model: step.whisperModel,
|
||||
wordTimestamps: wordTs,
|
||||
diarize: step.diarize,
|
||||
phonemeAlign: step.phonemeAlign,
|
||||
lsyncAlign: step.lsyncAlign ?? false,
|
||||
twoPass: step.twoPass ?? false,
|
||||
vocalStyle: step.vocalStyle ?? 'general',
|
||||
explicit: step.explicit ?? false,
|
||||
})
|
||||
await pollUntilDone(transcribeJobId, ctrl.signal)
|
||||
}, ctrl.signal, retry)
|
||||
|
||||
@@ -44,10 +44,12 @@ export default function DynamicMixPlayer({ stems, title, onLevelChange }: Props)
|
||||
clearMeta()
|
||||
}, [])
|
||||
|
||||
// Unregister + clear bar when unmounted
|
||||
// Stop audio + unregister when unmounted.
|
||||
// unregisterPlayback alone only removes the _current reference — it does NOT
|
||||
// pause the audio elements, so they keep playing after navigation.
|
||||
useEffect(() => () => {
|
||||
stopPlayback()
|
||||
unregisterPlayback(stopPlayback)
|
||||
clearMeta()
|
||||
}, [stopPlayback])
|
||||
|
||||
// Lazily create the shared AudioContext on first interaction (browser autoplay policy)
|
||||
@@ -111,6 +113,13 @@ export default function DynamicMixPlayer({ stems, title, onLevelChange }: Props)
|
||||
} else {
|
||||
registerPlayback(stopPlayback)
|
||||
isActiveRef.current = true
|
||||
// Publish immediately so the bar appears at once — don't wait for the
|
||||
// next RAF tick or effect cycle (there is a window between clearMeta()
|
||||
// from the old player and the effect re-running for this player).
|
||||
publishMeta(
|
||||
{ title: title ?? 'Dynamic Mix', isPlaying: true, currentTime, duration, volume: isMuted ? 0 : masterVolume },
|
||||
{ toggle: () => togglePlayRef.current(), seek: handleSeek, setVolume: (v) => updateMasterVolumeRef.current(v) }
|
||||
)
|
||||
// Suspend the audio clock, seek & start all elements, then resume —
|
||||
// guarantees all stems begin from the same audio frame simultaneously.
|
||||
await ctx.suspend()
|
||||
@@ -121,7 +130,7 @@ export default function DynamicMixPlayer({ stems, title, onLevelChange }: Props)
|
||||
await ctx.resume()
|
||||
setIsPlaying(true)
|
||||
}
|
||||
}, [isPlaying, currentTime, stopPlayback])
|
||||
}, [isPlaying, currentTime, duration, title, masterVolume, isMuted, stopPlayback])
|
||||
|
||||
function handleSeek(t: number) {
|
||||
setCurrentTime(t)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useState, useEffect, useRef } from 'react'
|
||||
import { Volume2, VolumeX } from 'lucide-react'
|
||||
import { getMeta, getActions, subscribePlayback } from '../utils/audioRegistry'
|
||||
import { Volume2, VolumeX, X } from 'lucide-react'
|
||||
import { getMeta, getActions, subscribePlayback, stopCurrent } from '../utils/audioRegistry'
|
||||
import type { PlaybackMeta } from '../utils/audioRegistry'
|
||||
|
||||
function fmtTime(s: number): string {
|
||||
@@ -99,6 +99,15 @@ export default function GlobalAudioBar() {
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Dismiss */}
|
||||
<button
|
||||
onClick={stopCurrent}
|
||||
className="p-1 rounded text-slate-400 dark:text-gray-500 hover:text-slate-600 dark:hover:text-gray-300 transition-colors shrink-0 ml-1"
|
||||
title="Stop and close"
|
||||
>
|
||||
<X className="w-3.5 h-3.5" />
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
@@ -55,11 +55,16 @@ const VISUAL_STYLES: { id: LyricsVisualStyle; label: string; desc: string }[] =
|
||||
]
|
||||
|
||||
const FONT_OPTIONS = [
|
||||
{ value: 'system-ui, sans-serif', label: 'System' },
|
||||
{ value: '"Inter", system-ui, sans-serif', label: 'Inter' },
|
||||
{ value: 'Georgia, "Times New Roman", serif', label: 'Serif' },
|
||||
{ value: '"Courier New", Courier, monospace', label: 'Mono' },
|
||||
{ value: '"Trebuchet MS", Arial, sans-serif', label: 'Trebuchet' },
|
||||
{ value: 'system-ui, sans-serif', label: 'System' },
|
||||
{ value: '"Inter", sans-serif', label: 'Inter' },
|
||||
{ value: '"Work Sans", sans-serif', label: 'Work Sans' },
|
||||
{ value: '"Archivo", sans-serif', label: 'Archivo' },
|
||||
{ value: '"Poppins", sans-serif', label: 'Poppins' },
|
||||
{ value: '"Roboto", sans-serif', label: 'Roboto' },
|
||||
{ value: '"Oswald", sans-serif', label: 'Oswald' },
|
||||
{ value: '"EB Garamond", serif', label: 'EB Garamond' },
|
||||
{ value: '"Libre Baskerville", serif', label: 'Libre Baskerville' },
|
||||
{ value: '"Young Serif", serif', label: 'Young Serif' },
|
||||
]
|
||||
|
||||
// ─── Speaker colours ──────────────────────────────────────────────────────────
|
||||
@@ -225,7 +230,7 @@ function formatTime(s: number): string {
|
||||
|
||||
// ─── Word rendering helpers ───────────────────────────────────────────────────
|
||||
|
||||
/** Standard karaoke fill: single-render gradient clip — no positional drift. */
|
||||
/** Karaoke word renderer — supports 'fill' (gradient sweep) and 'pop' (active word lit) modes. */
|
||||
function KaraokeWord({
|
||||
word,
|
||||
fill,
|
||||
@@ -235,6 +240,8 @@ function KaraokeWord({
|
||||
fontSize,
|
||||
isWordEnd,
|
||||
elRef,
|
||||
isActive,
|
||||
mode,
|
||||
}: {
|
||||
word: string
|
||||
fill: number // 0–1
|
||||
@@ -244,16 +251,41 @@ function KaraokeWord({
|
||||
fontSize: number
|
||||
isWordEnd: boolean
|
||||
elRef?: { current: HTMLSpanElement | null }
|
||||
isActive?: boolean
|
||||
mode?: 'fill' | 'pop'
|
||||
}) {
|
||||
const base: React.CSSProperties = {
|
||||
display: 'inline-block',
|
||||
verticalAlign: 'top',
|
||||
marginRight: isWordEnd ? '0.25em' : '0',
|
||||
fontSize,
|
||||
}
|
||||
if (mode === 'pop') {
|
||||
return (
|
||||
<span
|
||||
ref={el => { if (elRef) elRef.current = el }}
|
||||
style={{
|
||||
...base,
|
||||
color: (isActive || done) ? bright : dim,
|
||||
fontSize: isActive ? fontSize * 1.15 : fontSize,
|
||||
transition: 'font-size 0.08s ease, color 0.08s ease',
|
||||
// Explicitly clear gradient-clip props so they don't bleed in from a prior fill-mode render
|
||||
backgroundImage: 'none',
|
||||
WebkitBackgroundClip: 'border-box',
|
||||
WebkitTextFillColor: 'currentColor',
|
||||
backgroundClip: 'border-box',
|
||||
} as React.CSSProperties}
|
||||
>
|
||||
{word}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
const pct = done ? 100 : Math.round(fill * 100)
|
||||
return (
|
||||
<span
|
||||
ref={el => { if (elRef) elRef.current = el }}
|
||||
style={{
|
||||
display: 'inline-block',
|
||||
verticalAlign: 'top',
|
||||
marginRight: isWordEnd ? '0.25em' : '0',
|
||||
fontSize,
|
||||
...base,
|
||||
backgroundImage: `linear-gradient(to right, ${bright} ${pct}%, ${dim} ${pct}%)`,
|
||||
WebkitBackgroundClip: 'text',
|
||||
WebkitTextFillColor: 'transparent',
|
||||
@@ -277,6 +309,8 @@ interface StyleState {
|
||||
textAlign: 'left' | 'center' | 'right'
|
||||
lineSpacing: number
|
||||
dynBg: boolean
|
||||
karaokeMode: 'fill' | 'pop'
|
||||
fontBold: boolean
|
||||
}
|
||||
|
||||
function StylePicker({
|
||||
@@ -298,11 +332,17 @@ function StylePicker({
|
||||
value={style.fontFamily}
|
||||
onChange={e => onChange({ fontFamily: e.target.value })}
|
||||
className="text-xs bg-white/10 text-slate-200 rounded px-1.5 py-0.5 border border-white/15 focus:outline-none"
|
||||
style={{ colorScheme: 'dark' }}
|
||||
>
|
||||
{FONT_OPTIONS.map(f => (
|
||||
<option key={f.value} value={f.value} style={{ fontFamily: f.value }}>{f.label}</option>
|
||||
<option key={f.value} value={f.value} style={{ background: '#1e293b', color: '#e2e8f0', fontFamily: f.value }}>{f.label}</option>
|
||||
))}
|
||||
</select>
|
||||
<button
|
||||
onClick={() => onChange({ fontBold: !style.fontBold })}
|
||||
title="Bold"
|
||||
className={`px-1.5 py-0.5 rounded text-xs font-bold transition-colors ${style.fontBold ? 'bg-white/20 text-white' : 'text-slate-400 hover:text-slate-200'}`}
|
||||
>B</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
<span className="text-[10px] text-slate-500">Size</span>
|
||||
@@ -337,6 +377,16 @@ function StylePicker({
|
||||
<ColorSwatch label="Text" value={style.textColor} onChange={v => onChange({ textColor: v })} />
|
||||
<ColorSwatch label="Line" value={style.highlightColor} onChange={v => onChange({ highlightColor: v })} />
|
||||
<ColorSwatch label="Word" value={style.wordHighlight} onChange={v => onChange({ wordHighlight: v })} />
|
||||
<div className="flex items-center gap-1 ml-1 border-l border-white/10 pl-2">
|
||||
<span className="text-[10px] text-slate-500">Fill</span>
|
||||
{(['fill', 'pop'] as const).map(m => (
|
||||
<button
|
||||
key={m}
|
||||
onClick={() => onChange({ karaokeMode: m })}
|
||||
className={`text-[10px] px-1.5 py-0.5 rounded capitalize transition-colors ${style.karaokeMode === m ? 'bg-white/20 text-white' : 'text-slate-400 hover:text-slate-200'}`}
|
||||
>{m}</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex items-center gap-1 ml-auto">
|
||||
<span className="text-[10px] text-slate-500">Spacing</span>
|
||||
<input
|
||||
@@ -519,6 +569,8 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
textAlign: (styleOptions?.textAlign ?? 'center') as 'left' | 'center' | 'right',
|
||||
lineSpacing: styleOptions?.lineSpacing ?? 1.6,
|
||||
dynBg: false,
|
||||
karaokeMode: 'fill',
|
||||
fontBold: false,
|
||||
}
|
||||
try {
|
||||
const saved = localStorage.getItem('stemengine-lyric-style')
|
||||
@@ -535,7 +587,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
}, [])
|
||||
|
||||
// Destructure for convenience
|
||||
const { fontFamily, fontSize: baseFontSize, bgColor, textColor, highlightColor, wordHighlight, textAlign, lineSpacing, dynBg } = style
|
||||
const { fontFamily, fontSize: baseFontSize, bgColor, textColor, highlightColor, wordHighlight, textAlign, lineSpacing, dynBg, karaokeMode, fontBold } = style
|
||||
// fontSize is finalised after isFullscreen is available (declared below)
|
||||
|
||||
// Track whether cover art image loaded successfully
|
||||
@@ -643,10 +695,12 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
clearMeta()
|
||||
}, []) // intentionally stable — reads refs, not state
|
||||
|
||||
// Unregister + clear bar when the component unmounts
|
||||
// Stop audio + unregister when the component unmounts.
|
||||
// unregisterPlayback alone only removes the _current reference — it does NOT
|
||||
// pause the audio elements, so they keep playing after navigation.
|
||||
useEffect(() => () => {
|
||||
stopPlayback()
|
||||
unregisterPlayback(stopPlayback)
|
||||
clearMeta()
|
||||
}, [stopPlayback])
|
||||
|
||||
// Refs for direct DOM gradient updates — avoids 60fps React re-renders
|
||||
@@ -655,6 +709,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
const linesRef = useRef<LrcLine[]>(lines)
|
||||
const brightRef = useRef(wordHighlight)
|
||||
const dimRef = useRef(textColor + '60')
|
||||
const karaokeModeRef = useRef(karaokeMode)
|
||||
|
||||
// Keep linesRef in sync (done directly in render so RAF always sees latest parse)
|
||||
linesRef.current = lines
|
||||
@@ -684,6 +739,9 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [activeLine, wordHighlight, textColor, speakerColors])
|
||||
|
||||
// Keep karaokeModeRef in sync so the RAF loop sees the latest value without restarting
|
||||
karaokeModeRef.current = karaokeMode
|
||||
|
||||
// ── Auto-detect default style for new content (only if user has no saved preference) ──
|
||||
useEffect(() => {
|
||||
if (!defaultStyle && !localStorage.getItem('stemengine-lyric-visual-style')) {
|
||||
@@ -825,7 +883,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
}
|
||||
|
||||
// ── Direct DOM: update active karaoke-word gradient (no React re-render) ──
|
||||
if (activeWordSpanRef.current && lastLineIdx >= 0) {
|
||||
if (activeWordSpanRef.current && lastLineIdx >= 0 && karaokeModeRef.current !== 'pop') {
|
||||
const line = curLines[lastLineIdx]
|
||||
if (line?.words.length > 0) {
|
||||
const wi = findActiveWord(line.words, t)
|
||||
@@ -909,6 +967,11 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
} else {
|
||||
registerPlayback(stopPlayback)
|
||||
isActiveRef.current = true
|
||||
// Publish immediately so the bar appears at once.
|
||||
publishMeta(
|
||||
{ title: title ?? 'Live Preview', isPlaying: true, currentTime, duration, volume: isMuted ? 0 : volume },
|
||||
{ toggle: () => togglePlayRef.current(), seek: seekTime, setVolume: (v) => setVolumeFromBar.current(v) }
|
||||
)
|
||||
await ctx.suspend()
|
||||
audios.forEach(a => { a.currentTime = currentTime; a.play().catch(() => {}) })
|
||||
await ctx.resume()
|
||||
@@ -922,6 +985,12 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
registerPlayback(stopPlayback)
|
||||
isActiveRef.current = true
|
||||
audioRef.current.play().catch(() => {})
|
||||
// Publish immediately so the bar appears at once rather than waiting
|
||||
// for the next effect cycle (avoids the clearMeta→publishMeta gap).
|
||||
publishMeta(
|
||||
{ title: title ?? 'Live Preview', isPlaying: true, currentTime, duration, volume: isMuted ? 0 : volume },
|
||||
{ toggle: () => togglePlayRef.current(), seek: seekTime, setVolume: (v) => setVolumeFromBar.current(v) }
|
||||
)
|
||||
}
|
||||
setIsPlaying(!isPlaying)
|
||||
} else {
|
||||
@@ -929,6 +998,10 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
if (!isPlaying) {
|
||||
registerPlayback(stopPlayback)
|
||||
isActiveRef.current = true
|
||||
publishMeta(
|
||||
{ title: title ?? 'Live Preview', isPlaying: true, currentTime, duration, volume: isMuted ? 0 : volume },
|
||||
{ toggle: () => togglePlayRef.current(), seek: seekTime, setVolume: (v) => setVolumeFromBar.current(v) }
|
||||
)
|
||||
}
|
||||
// Don't unregister on pause — keep bar visible
|
||||
lastRafTime.current = null
|
||||
@@ -1220,6 +1293,8 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
fontSize={fontSize * 1.05}
|
||||
isWordEnd={w.isWordEnd}
|
||||
elRef={wi === activeWordIdx ? activeWordSpanRef : undefined}
|
||||
isActive={wi === activeWordIdx}
|
||||
mode={karaokeMode}
|
||||
/>
|
||||
)
|
||||
})
|
||||
@@ -1329,6 +1404,8 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
fontSize={fontSize * 1.05}
|
||||
isWordEnd={w.isWordEnd}
|
||||
elRef={wi === activeWordIdx ? activeWordSpanRef : undefined}
|
||||
isActive={wi === activeWordIdx}
|
||||
mode={karaokeMode}
|
||||
/>
|
||||
)
|
||||
})
|
||||
@@ -1432,6 +1509,8 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
fontSize={fontSize * 1.1}
|
||||
isWordEnd={w.isWordEnd}
|
||||
elRef={wi === activeWordIdx ? activeWordSpanRef : undefined}
|
||||
isActive={wi === activeWordIdx}
|
||||
mode={karaokeMode}
|
||||
/>
|
||||
)
|
||||
})
|
||||
@@ -1479,6 +1558,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
|
||||
style={{
|
||||
backgroundColor: effectiveBg,
|
||||
fontFamily,
|
||||
fontWeight: fontBold ? 700 : 400,
|
||||
height: isFullscreen ? undefined : 'clamp(200px, 40vh, 340px)',
|
||||
flex: isFullscreen ? '1 1 0' : undefined,
|
||||
overflow: 'hidden',
|
||||
|
||||
@@ -3,6 +3,15 @@ import ReactDOM from 'react-dom/client'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import App from './App'
|
||||
import './index.css'
|
||||
import '@fontsource/archivo'
|
||||
import '@fontsource/eb-garamond'
|
||||
import '@fontsource/inter'
|
||||
import '@fontsource/libre-baskerville'
|
||||
import '@fontsource/oswald'
|
||||
import '@fontsource/poppins'
|
||||
import '@fontsource/roboto'
|
||||
import '@fontsource/work-sans'
|
||||
import '@fontsource/young-serif'
|
||||
|
||||
if ('serviceWorker' in navigator) {
|
||||
window.addEventListener('load', () => {
|
||||
|
||||
@@ -1690,6 +1690,8 @@ function StemLyricsRow({
|
||||
const [diarize, setDiarize] = useState(false)
|
||||
const [phonemeAlign, setPhonemeAlign] = useState(false)
|
||||
const [lsyncAlign, setLsyncAlign] = useState(false)
|
||||
const [vocalStyle, setVocalStyle] = useState<'general' | 'hiphop'>('general')
|
||||
const [explicit, setExplicit] = useState(false)
|
||||
const [whisperModel, setWhisperModel] = useState(
|
||||
() => localStorage.getItem('stemengine-whisper-model') ?? 'large-v2'
|
||||
)
|
||||
@@ -1825,6 +1827,8 @@ function StemLyricsRow({
|
||||
phonemeAlign,
|
||||
lsyncAlign,
|
||||
twoPass,
|
||||
vocalStyle,
|
||||
explicit,
|
||||
})
|
||||
sessionJobIds.current.add(res.job_id)
|
||||
onAction()
|
||||
@@ -1941,6 +1945,15 @@ function StemLyricsRow({
|
||||
<option value="transcribe">Transcribe</option>
|
||||
<option value="translate">Translate → English</option>
|
||||
</select>
|
||||
<select
|
||||
value={vocalStyle}
|
||||
onChange={(e) => setVocalStyle(e.target.value as 'general' | 'hiphop')}
|
||||
title="Vocal style — tunes VAD thresholds and ASR priors for the genre"
|
||||
className="rounded-lg bg-white dark:bg-gray-900 border border-slate-300 dark:border-gray-700 px-2.5 py-1.5 text-xs text-slate-900 dark:text-gray-100 focus:outline-none focus:border-violet-400"
|
||||
>
|
||||
<option value="general">General</option>
|
||||
<option value="hiphop">Hip-hop / Rap</option>
|
||||
</select>
|
||||
<label className="flex items-center gap-1.5 cursor-pointer select-none">
|
||||
<input
|
||||
type="checkbox"
|
||||
@@ -1950,15 +1963,6 @@ function StemLyricsRow({
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Diarize</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Use phonemizer + facebook/wav2vec2-xlsr-53-espeak-cv-ft for syllable-level timestamps">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={phonemeAlign}
|
||||
onChange={(e) => setPhonemeAlign(e.target.checked)}
|
||||
className="w-3.5 h-3.5 rounded accent-violet-500"
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Phonemes</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Use lsync wav2vec2 CTC forced alignment + pyphen for syllable-level timestamps (more accurate than Phonemes mode)">
|
||||
<input
|
||||
type="checkbox"
|
||||
@@ -1977,6 +1981,15 @@ function StemLyricsRow({
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Two-pass</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Add an explicit-language primer to the transcription prompt so Whisper stops censoring swear words">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={explicit}
|
||||
onChange={(e) => setExplicit(e.target.checked)}
|
||||
className="w-3.5 h-3.5 rounded accent-violet-500"
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Explicit</span>
|
||||
</label>
|
||||
<button
|
||||
onClick={handleTranscribe}
|
||||
disabled={busy || !!isActive(whisperJob)}
|
||||
|
||||
@@ -1145,7 +1145,7 @@ function AutoPipelineSettings() {
|
||||
setAddOpen(false)
|
||||
if (type === 'reprocess') saveSteps([...steps, { type: 'reprocess', stem: 'other', model: sepModel }])
|
||||
if (type === 'midi') saveSteps([...steps, { type: 'midi', mode: 'polyphonic' }])
|
||||
if (type === 'lyrics') saveSteps([...steps, { type: 'lyrics', whisperModel: 'large-v2', diarize: false, phonemeAlign: false, twoPass: false }])
|
||||
if (type === 'lyrics') saveSteps([...steps, { type: 'lyrics', whisperModel: 'large-v2', diarize: false, lsyncAlign: false, twoPass: false, vocalStyle: 'general', explicit: false }])
|
||||
if (type === 'lyricsAlign') saveSteps([...steps, { type: 'lyricsAlign', timingPrecision: 'word' }])
|
||||
}
|
||||
|
||||
@@ -1405,6 +1405,18 @@ function AutoPipelineSettings() {
|
||||
{WHISPER_MODEL_LIST.map(m => <option key={m.id} value={m.id}>{m.label}</option>)}
|
||||
</select>
|
||||
</label>
|
||||
<label className="space-y-1 block pt-1">
|
||||
<span className="text-xs text-slate-500 dark:text-gray-400 uppercase tracking-wider">Vocal Style</span>
|
||||
<select
|
||||
value={step.vocalStyle ?? 'general'}
|
||||
onChange={e => patchStep(idx, { vocalStyle: e.target.value } as Partial<AutoStep>)}
|
||||
className={AP_SELECT}
|
||||
title="Tunes VAD thresholds and ASR priors for the genre"
|
||||
>
|
||||
<option value="general">General</option>
|
||||
<option value="hiphop">Hip-hop / Rap</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="flex items-center gap-2 cursor-pointer select-none pt-1">
|
||||
<input
|
||||
type="checkbox"
|
||||
@@ -1414,14 +1426,14 @@ function AutoPipelineSettings() {
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Diarize (speaker separation)</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-2 cursor-pointer select-none" title="Use phonemizer + facebook/wav2vec2-xlsr-53-espeak-cv-ft for syllable-level timestamps">
|
||||
<label className="flex items-center gap-2 cursor-pointer select-none" title="Use lsync wav2vec2 CTC forced alignment + pyphen for syllable-level timestamps">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={step.phonemeAlign}
|
||||
onChange={e => patchStep(idx, { phonemeAlign: e.target.checked } as Partial<AutoStep>)}
|
||||
checked={step.lsyncAlign ?? false}
|
||||
onChange={e => patchStep(idx, { lsyncAlign: e.target.checked } as Partial<AutoStep>)}
|
||||
className="w-3.5 h-3.5 rounded accent-violet-500"
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Phoneme alignment</span>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Syllables</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-2 cursor-pointer select-none" title="Run a second VAD pass to fill transcription gaps larger than 2 seconds">
|
||||
<input
|
||||
@@ -1432,6 +1444,15 @@ function AutoPipelineSettings() {
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Two-pass (fill gaps)</span>
|
||||
</label>
|
||||
<label className="flex items-center gap-2 cursor-pointer select-none" title="Add an explicit-language primer so Whisper stops censoring swear words">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={step.explicit ?? false}
|
||||
onChange={e => patchStep(idx, { explicit: e.target.checked } as Partial<AutoStep>)}
|
||||
className="w-3.5 h-3.5 rounded accent-violet-500"
|
||||
/>
|
||||
<span className="text-xs text-slate-600 dark:text-gray-400">Explicit</span>
|
||||
</label>
|
||||
</>
|
||||
)}
|
||||
|
||||
|
||||
@@ -58,6 +58,18 @@ export function unregisterPlayback(stop: StopFn): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop and dismiss the active player from anywhere (e.g. the global bar's ✕).
|
||||
* Calls the registered stop callback (which pauses audio and clears meta),
|
||||
* then removes the reference so the next player starts fresh.
|
||||
*/
|
||||
export function stopCurrent(): void {
|
||||
if (_current) {
|
||||
_current()
|
||||
_current = null
|
||||
}
|
||||
}
|
||||
|
||||
// ── Bar state ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,8 +22,10 @@ export interface AutoStepLyrics {
|
||||
type: 'lyrics'
|
||||
whisperModel: string // e.g. 'large-v2'
|
||||
diarize: boolean
|
||||
phonemeAlign: boolean // use phonemizer + wav2vec2-xlsr for syllable-level timestamps
|
||||
lsyncAlign: boolean // use lsync wav2vec2 CTC + pyphen for syllable-level timestamps
|
||||
twoPass: boolean // run a second VAD pass to fill transcription gaps > 2 s
|
||||
vocalStyle: string // 'general' | 'hiphop'
|
||||
explicit: boolean // disable Whisper's profanity filter via initial_prompt primer
|
||||
}
|
||||
|
||||
export interface AutoStepLyricsAlign {
|
||||
|
||||
Reference in new issue
Block a user