From 4982cf8427340daf336d91cce21a63ef6b3d65a7 Mon Sep 17 00:00:00 2001 From: translator Date: Tue, 17 Mar 2026 11:39:17 +0900 Subject: [PATCH] Myriad improvements. --- 20260313_Fixes (Copilot).md | 197 ++ 20260317_Components_Review_and_Fix_Plan.md | 138 + 20260317_GPU_Limiter_MIDI_Integration.md | 36 + 20260317_LyricsLivePreview_Review.md | 98 + 20260317_lrc_and_lyric_transcriber_review.md | 152 ++ 20260317_lrc_lyric_transcriber_remediation.md | 51 + README.md | 7 +- backend/services/lrc_service.py | 138 +- backend/services/lyric_transcriber_service.py | 168 +- backend/services/separator_service.py | 6 +- backend/services/transcriber_service.py | 9 + .../2020310_LyricsLivePreview_Claude.tsx | 2318 ----------------- .../src/components/AutoPipelineDialog.tsx | 27 +- frontend/src/components/DynamicMixPlayer.tsx | 93 +- frontend/src/components/FileUpload.tsx | 5 +- frontend/src/components/KaraokeEditor.tsx | 45 +- frontend/src/components/LyricsCanvas.tsx | 1452 ----------- frontend/src/components/LyricsLivePreview.tsx | 1299 ++++++++- frontend/src/components/SettingsPanel.tsx | 2 +- frontend/src/components/Sidebar.tsx | 14 +- omnizart-sidecar/app.py | 83 +- stemengine.yml | 3 + 22 files changed, 2351 insertions(+), 3990 deletions(-) create mode 100644 20260317_Components_Review_and_Fix_Plan.md create mode 100644 20260317_GPU_Limiter_MIDI_Integration.md create mode 100644 20260317_LyricsLivePreview_Review.md create mode 100644 20260317_lrc_and_lyric_transcriber_review.md create mode 100644 20260317_lrc_lyric_transcriber_remediation.md delete mode 100755 frontend/src/components/2020310_LyricsLivePreview_Claude.tsx delete mode 100644 frontend/src/components/LyricsCanvas.tsx diff --git a/20260313_Fixes (Copilot).md b/20260313_Fixes (Copilot).md index 3d2d034..ba328c3 100644 --- a/20260313_Fixes (Copilot).md +++ b/20260313_Fixes (Copilot).md @@ -2262,3 +2262,200 @@ Status impact: - Runtime blocker for Parakeet backend execution is cleared. - Remaining work is manual UI sign-off coverage and rollout approval documentation. +### 2026-03-17 continuation (LyricsLivePreview follow-up hardening) + +Reviewed and continued hardening `frontend/src/components/LyricsLivePreview.tsx` after the +canvas-based renderer migration. + +Artifacts added: +- `20260317_LyricsLivePreview_Review.md` + +Review findings logged: +- style reset dropped caller-provided defaults +- speaker localStorage state could bleed across `storageKey` changes +- RAF loop continued running while idle +- multi-stem audio resources lacked teardown +- legacy recenter path was still present even though the active renderer is `LyricsCanvas` + +Fixes implemented: +- reset now restores merged defaults derived from incoming `styleOptions` +- speaker preferences are rehydrated on `storageKey` change with a hydration guard to avoid stale writes +- paused preview now stops the RAF loop instead of scheduling idle animation frames continuously +- multi-stem preview now disposes removed/unmounted audio elements, metadata listeners, + source nodes, gain nodes, and closes the `AudioContext` on final teardown +- removed the dead recenter button/effect/refs from the canvas-based preview path + +Validation: +- file diagnostics clean for `frontend/src/components/LyricsLivePreview.tsx` +- frontend typecheck passes (`npm run typecheck`) + +### 2026-03-17 continuation (LyricsLivePreview no longer depends on LyricsCanvas.tsx) + +Refactored `frontend/src/components/LyricsLivePreview.tsx` so it no longer imports or depends on +`frontend/src/components/LyricsCanvas.tsx`. + +What changed: +- inlined the Two.js/SVG lyric renderer and its scene/update helpers directly into + `LyricsLivePreview.tsx` +- removed the `LyricsCanvas` import from the preview component +- reconciled the inlined renderer with the preview component's existing `StyleState` + shape (`wordHighlight` / `dynBg`) + +Validation: +- no remaining `LyricsCanvas` import in `LyricsLivePreview.tsx` +- file diagnostics clean for `frontend/src/components/LyricsLivePreview.tsx` +- frontend typecheck passes (`npm run typecheck`) + +### 2026-03-17 continuation (final cleanup: remove obsolete LyricsCanvas.tsx) + +Completed the follow-up cleanup after inlining the renderer into +`frontend/src/components/LyricsLivePreview.tsx`: + +- removed obsolete file: `frontend/src/components/LyricsCanvas.tsx` +- confirmed no remaining frontend references to `LyricsCanvas` import paths +- re-ran frontend typecheck successfully (`npm run typecheck`) + +### 2026-03-17 continuation (reviewed remaining frontend components + fix plan) + +Completed a targeted review of frontend components under `frontend/src/components/` +excluding `LyricsLivePreview.tsx`, and documented findings plus a phased remediation plan. + +Review artifact created: +- `20260317_Components_Review_and_Fix_Plan.md` + +Scope reviewed: +- `AutoPipelineDialog.tsx` +- `DynamicMixPlayer.tsx` +- `FileUpload.tsx` +- `GlobalAudioBar.tsx` +- `JobStatus.tsx` +- `KaraokeEditor.tsx` +- `ResultFiles.tsx` +- `SettingsPanel.tsx` +- `Sidebar.tsx` + +Prioritized issues captured in the plan: +- high: DynamicMixPlayer resource lifecycle/disposal gaps +- high: Sidebar project delete flow may update UI as if success on failed delete +- medium: KaraokeEditor word timing lower-bound clamp should enforce previous-word ordering +- medium: AutoPipelineDialog abort-listener cleanup hygiene in retry/poll delays +- medium: DynamicMixPlayer state reconciliation when stem set changes +- low/polish: invalid Tailwind class in SettingsPanel; FileUpload multi-select UX mismatch; Subsonic mutation error feedback + +Next-step plan included in artifact: +- Phase 1 stability hotfixes +- Phase 2 reliability cleanup +- Phase 3 UX polish +- explicit validation checklist (`npm run typecheck` + focused manual checks) + +### 2026-03-17 continuation (implemented components hardening plan) + +Implemented the component fixes from `20260317_Components_Review_and_Fix_Plan.md`. + +Updated files: +- `frontend/src/components/DynamicMixPlayer.tsx` +- `frontend/src/components/Sidebar.tsx` +- `frontend/src/components/KaraokeEditor.tsx` +- `frontend/src/components/AutoPipelineDialog.tsx` +- `frontend/src/components/SettingsPanel.tsx` +- `frontend/src/components/FileUpload.tsx` + +What was implemented: +- DynamicMixPlayer + - added per-stem disposal helper for `HTMLAudioElement` + listeners + WebAudio nodes + - dispose removed stems when stem-set changes + - full unmount teardown (pause/dispose all stems, close `AudioContext`, unregister) + - reconciled per-stem state maps (`gains`, `loaded`, `mutedStems`, `soloedStems`) when stem-set changes + - applied gain matrix reactively when relevant state changes + - tightened readiness check to require loaded metadata for all stems +- Sidebar + - project deletion now updates UI only after successful backend delete + - added visible failure feedback for project delete and import-delete failures +- KaraokeEditor + - enforced monotonic word timing bounds (respect previous word lower bound + next word/line upper bound) + - applied same constraint path for drag edits and typed edits + - made waveform fetch/decode pipeline abortable on cleanup +- AutoPipelineDialog + - replaced ad hoc abortable timeout blocks with a shared `sleepWithAbort()` helper + - ensured abort listeners are removed on both resolve and abort paths +- SettingsPanel + - fixed invalid Tailwind token (`dark:hover:bg-gray-750` → `dark:hover:bg-gray-700`) +- FileUpload + - added `multiple` prop (default `true`) and wired it to the hidden file input + +Validation: +- frontend typecheck passes (`npm run typecheck`) + +### 2026-03-17 continuation (backend review: lrc_service + lyric_transcriber_service) + +Completed issue review and documented findings for: +- `backend/services/lrc_service.py` +- `backend/services/lyric_transcriber_service.py` + +Review artifact created: +- `20260317_lrc_and_lyric_transcriber_review.md` + +Top issues flagged in report: +- high: Whisper timeout thread can continue while GPU lock is released (race/resource contention risk) +- high: global `torch.load` monkey patch forcing `weights_only=False` +- high: secondary diarization path diverges from primary GPU lock/cleanup orchestration +- medium: HTML preview duration reads wrong data attribute (`data-start` vs `data-time`) +- medium: HTML generation interpolates unescaped dynamic content +- medium: bare cleanup `except: pass` hides temp-file IO errors + +Report includes a phased fix plan and validation checklist. + +### 2026-03-17 continuation (implemented remediation: lrc_service + lyric_transcriber_service) + +Implemented remediation items from `20260317_lrc_and_lyric_transcriber_review.md`. + +Updated files: +- `backend/services/lrc_service.py` +- `backend/services/lyric_transcriber_service.py` + +What was fixed: +- `lrc_service.py` + - fixed styled preview duration parser mismatch (`[data-start]` → derived from `data-time`) + - escaped dynamic lyric/title HTML content (`html.escape`) + - added style-value sanitization for font family, color, numeric ranges, and text alignment + - replaced bare cleanup `except: pass` with `except OSError` + debug logging +- `lyric_transcriber_service.py` + - added env-gated guardrail for torch-load compatibility patch (`STEMENGINE_PATCH_TORCH_LOAD`) + - added startup warning when compatibility patch is enabled + - removed detached timeout-thread pattern in Whisper path and run transcription inline under GPU lock + to eliminate lock-release/background-worker race + - added shared GPU lock + cleanup symmetry to `run_lyric_transcription_with_diarization()` + +Remediation report artifact: +- `20260317_lrc_lyric_transcriber_remediation.md` + +Validation: +- `python3 -m py_compile backend/services/lrc_service.py backend/services/lyric_transcriber_service.py` + completed successfully + + +### 2026-03-17 continuation (silence-aware lyric fill timing) + +**Problem:** Word/syllable highlights in the lyrics preview extended their fill animation across long +gaps and silences before the next token, producing slow-creeping or jarring overly-long transitions. + +**Root cause:** Both the DOM RAF path (`wordFillPct`) and the canvas/Two.js path (per-`CanvasWordUnit` +`duration`) used the raw distance to the next word start (or next line start) as the full fill window, +with no awareness of whether that distance represented active speech or silent dead air. + +**Fix — `frontend/src/components/LyricsLivePreview.tsx`:** +- Added two module-level constants: + - `FILL_GAP_THRESHOLD_S = 0.6` — gaps wider than 600 ms are treated as silences + - `FILL_MAX_DURATION_S = 0.5` — fill window is capped to 500 ms for silence gaps +- Added `silenceAwareEnd(wordStart, rawEnd)` helper that applies the cap when needed. + For tight inter-word gaps (≤ 600 ms) the raw next-token start is used unchanged; + for silences the effective end is `wordStart + 500 ms`, so the highlight sweeps + quickly and then freezes cleanly until the next word is reached. +- **DOM path** — `wordFillPct()` now computes `rawEnd` first and routes it through + `silenceAwareEnd()` before deriving `dur`. +- **Canvas path** — the `CanvasWordUnit.duration` assignment now computes `rawEnd` + (next word time or `line.endTime` fallback) and routes it through `silenceAwareEnd()` + before subtracting `word.time`, matching the DOM path exactly. + +Validation: +- `npm run typecheck` completed with no errors diff --git a/20260317_Components_Review_and_Fix_Plan.md b/20260317_Components_Review_and_Fix_Plan.md new file mode 100644 index 0000000..205d09b --- /dev/null +++ b/20260317_Components_Review_and_Fix_Plan.md @@ -0,0 +1,138 @@ +# 2026-03-17 — Frontend Components Review (excluding `LyricsLivePreview.tsx`) + +## Scope reviewed +- `frontend/src/components/AutoPipelineDialog.tsx` +- `frontend/src/components/DynamicMixPlayer.tsx` +- `frontend/src/components/FileUpload.tsx` +- `frontend/src/components/GlobalAudioBar.tsx` +- `frontend/src/components/JobStatus.tsx` +- `frontend/src/components/KaraokeEditor.tsx` +- `frontend/src/components/ResultFiles.tsx` +- `frontend/src/components/SettingsPanel.tsx` +- `frontend/src/components/Sidebar.tsx` + +--- + +## Executive summary +- **High priority fixes:** 2 +- **Medium priority fixes:** 4 +- **Low priority fixes / polish:** 3 +- **No immediate issues found:** `GlobalAudioBar.tsx`, `ResultFiles.tsx`, `JobStatus.tsx` (functional review only) + +--- + +## Findings + +### High + +1) **DynamicMixPlayer does not dispose stem audio graph/resources on unmount or stem-set changes** +- **File:** `frontend/src/components/DynamicMixPlayer.tsx` +- **Evidence:** lines ~50-54, 63-91 +- **Why this matters:** `HTMLAudioElement`, `MediaElementAudioSourceNode`, and `GainNode` are created but never disconnected/removed per stem. This can leak nodes/listeners and cause stale playback state after navigation or stem list mutation. +- **Recommended fix:** + - Add a `disposeStemAudio(name)` helper to pause audio, remove listeners, clear `src`, disconnect source/gain nodes, delete refs. + - On `stems` changes, remove refs for missing stems. + - On unmount, dispose all stems and close `AudioContext`. + +2) **Project delete path can update UI as if delete succeeded even when API delete fails** +- **File:** `frontend/src/components/Sidebar.tsx` +- **Evidence:** `ProjectItem.handleDelete()` around lines ~123-131 +- **Why this matters:** Errors are swallowed, then `onDeleted()` is still called. UI can desync from backend state and wrongly close tabs/remove project views. +- **Recommended fix:** + - Only call `onDeleted()` after successful `deleteProject`. + - Surface error toast or inline message when deletion fails. + +### Medium + +3) **Word timing edits can violate monotonic order against previous word** +- **File:** `frontend/src/components/KaraokeEditor.tsx` +- **Evidence:** clamping uses `line.time` and next-word cap only (lines ~531, ~547, plus drag path ~780+) +- **Why this matters:** A word can be set earlier than the previous word, creating invalid ELRC timing order. +- **Recommended fix:** + - Clamp lower bound to `max(line.time, prevWord.time + 0.01)`. + - Apply same rule for drag edits and typed edits. + +4) **Abort listeners are attached repeatedly without cleanup in retry/poll delay helpers** +- **File:** `frontend/src/components/AutoPipelineDialog.tsx` +- **Evidence:** lines ~69 and ~84 +- **Why this matters:** Repeated runs/retries can accumulate listeners unnecessarily. +- **Recommended fix:** + - Use `{ once: true }` for abort listeners and/or store a handler and remove it after timeout resolves. + +5) **`DynamicMixPlayer` local state not reinitialized for changed stem sets** +- **File:** `frontend/src/components/DynamicMixPlayer.tsx` +- **Evidence:** `gains` initialized once from initial props (line ~27), no reconciliation effect. +- **Why this matters:** New stems may lack initialized gain/mute/solo state; removed stems may leave stale entries. +- **Recommended fix:** + - Add reconciliation when `stems` changes to initialize missing keys and prune removed keys in `gains`, `loaded`, `mutedStems`, `soloedStems`. + +6) **Waveform fetch/decode has no request cancellation** +- **File:** `frontend/src/components/KaraokeEditor.tsx` +- **Evidence:** audio decode effect around lines ~230-248 +- **Why this matters:** On rapid source switching/unmount, unnecessary network/decode work continues. +- **Recommended fix:** + - Add `AbortController` to `fetch(audioUrl)`. + - Guard decode pipeline and skip `decodeAudioData` work after abort. + +### Low / polish + +7) **Invalid Tailwind utility class likely never applies** +- **File:** `frontend/src/components/SettingsPanel.tsx` +- **Evidence:** `dark:hover:bg-gray-750` at line ~82 +- **Why this matters:** Style inconsistency and dead class. +- **Recommended fix:** replace with valid token (`dark:hover:bg-gray-700` or configured custom scale). + +8) **`FileUpload` supports multi-file callback but input does not expose multi-select** +- **File:** `frontend/src/components/FileUpload.tsx` +- **Evidence:** file input at line ~37 (no `multiple`) +- **Why this matters:** UX mismatch: drop supports many files; picker appears single-file. +- **Recommended fix:** add `multiple` prop (default `true`) and pass to input. + +9) **Import/delete actions in Subsonic panel swallow API errors** +- **File:** `frontend/src/components/Sidebar.tsx` +- **Evidence:** `doDeleteImport` and some mutation handlers +- **Why this matters:** silent failure makes troubleshooting difficult. +- **Recommended fix:** show lightweight inline error state/toast on failures. + +--- + +## Component-by-component status snapshot + +- **AutoPipelineDialog.tsx** — solid orchestration flow; minor abort-listener hygiene issue. +- **DynamicMixPlayer.tsx** — good UX, but needs lifecycle/resource hardening. +- **FileUpload.tsx** — clean/small; minor UX enhancement (`multiple`). +- **GlobalAudioBar.tsx** — no blocking issues found. +- **JobStatus.tsx** — polling logic acceptable; no blocking issues found. +- **KaraokeEditor.tsx** — strong feature set; needs timing-order clamp + fetch cancellation. +- **ResultFiles.tsx** — no blocking issues found. +- **SettingsPanel.tsx** — one styling token fix. +- **Sidebar.tsx** — robust overall; deletion/error-flow handling needs tightening. + +--- + +## Proposed fix plan + +### Phase 1 — Stability hotfixes (same day) +1. `DynamicMixPlayer`: add full stem/audio graph disposal + stem-set reconciliation. +2. `Sidebar` project delete flow: only update UI on confirmed success; display failure feedback. +3. `KaraokeEditor`: enforce previous-word lower-bound clamp for all word timing edits. + +### Phase 2 — Reliability/cleanup (next pass) +4. `AutoPipelineDialog`: cleanup abort listener usage (`once` or removeEventListener). +5. `KaraokeEditor`: add abortable waveform fetch/decode pipeline. +6. `Sidebar` import/delete operations: standardized mutation error feedback. + +### Phase 3 — UX polish +7. `SettingsPanel`: replace invalid Tailwind class. +8. `FileUpload`: expose configurable multi-select support. + +--- + +## Validation plan after fixes +- Run: `npm run typecheck` +- Run focused manual checks: + - DynamicMixPlayer: mount/unmount cycles, stem swaps, no lingering audio playback. + - Sidebar: failed delete path leaves UI unchanged and surfaces error. + - KaraokeEditor: cannot set word start before previous word. + - SettingsPanel hover styles in dark mode. + - FileUpload click-select supports intended file count behavior. diff --git a/20260317_GPU_Limiter_MIDI_Integration.md b/20260317_GPU_Limiter_MIDI_Integration.md new file mode 100644 index 0000000..c566dc5 --- /dev/null +++ b/20260317_GPU_Limiter_MIDI_Integration.md @@ -0,0 +1,36 @@ +# 2026-03-17 — GPU limiter integration for MIDI transcription + +## Objective +Ensure MIDI transcription engines are governed by the same GPU safety policy used by separation/lyric flows, while preserving existing queue behavior and compatibility. + +## Recommendation +1. Keep queue-level serialization as-is (`jm.submit_gpu`) for MIDI jobs. +2. Explicitly apply backend GPU safety limits before `basic-pitch` inference. +3. Apply a TensorFlow GPU memory cap in `omnizart-sidecar` using the same reserve-MB policy. +4. Expose one optional sidecar override variable, defaulting to the global reserve. +5. Document the behavior in project docs. + +## Implementation plan (completed) +- [x] Wire `basic-pitch` path to existing `_apply_cuda_limits()` before model load. +- [x] Add TensorFlow memory-limit setup in `omnizart-sidecar/app.py`. +- [x] Add `OMNIZART_GPU_VRAM_RESERVE_MB` env in compose files with fallback to `STEMENGINE_GPU_VRAM_RESERVE_MB`. +- [x] Update README configuration/GPU notes. + +## Files changed +- `backend/services/transcriber_service.py` +- `omnizart-sidecar/app.py` +- `stemengine.yml` (workspace-level) +- `appdata/stemengine/stemengine.yml` (repo-level) +- `README.md` + +## Behavioral outcome +- `basic-pitch` now executes after backend GPU limit setup. +- `omnizart-sidecar` now configures TensorFlow GPU usage to preserve VRAM headroom: + - hard cap when total VRAM is detectable via `nvidia-smi` + - memory-growth fallback when total VRAM is not detectable +- Existing single-worker GPU queue semantics remain unchanged. + +## Notes +- TensorFlow logical-device configuration must occur before TF runtime initialization. +- Sidecar applies limiter at process startup (module import path) before loading Omnizart. +- Set reserve to `0` to disable cap behavior. diff --git a/20260317_LyricsLivePreview_Review.md b/20260317_LyricsLivePreview_Review.md new file mode 100644 index 0000000..8e58452 --- /dev/null +++ b/20260317_LyricsLivePreview_Review.md @@ -0,0 +1,98 @@ +# LyricsLivePreview Review + +Date: 2026-03-17 +Target: `frontend/src/components/LyricsLivePreview.tsx` +Supporting context: `frontend/src/components/2020310_LyricsLivePreview_Claude.tsx` + +## Summary + +The file has no current TypeScript diagnostics, but it contains a few real behavioral and maintainability issues that are worth addressing before further feature work. + +## Findings + +### 1. Reset style drops caller-provided defaults +Severity: High + +`style` is initialized by merging `DEFAULT_STYLE` with incoming `styleOptions`, but `resetStyle()` restores only `DEFAULT_STYLE`. + +Impact: +- project- or page-specific default styling is lost after a reset +- reset behavior is inconsistent with initial load behavior + +Relevant area: +- `frontend/src/components/LyricsLivePreview.tsx` + +### 2. Speaker settings can bleed across `storageKey` changes +Severity: High + +`speakerColors` and `speakerNames` are initialized from `storageKey` only once. If the component later receives a different `storageKey`, the old in-memory speaker state remains active and is then persisted under the new key. + +Impact: +- speaker names/colors from one song/project can appear on another +- the new key can be overwritten before its stored values are loaded + +Relevant area: +- `frontend/src/components/LyricsLivePreview.tsx` + +### 3. RAF loop runs continuously while idle +Severity: Medium + +The animation loop keeps scheduling `requestAnimationFrame()` even when playback is stopped and there is no active audio progression. + +Impact: +- unnecessary CPU usage for a static preview +- avoidable background work while component is mounted + +Relevant area: +- `frontend/src/components/LyricsLivePreview.tsx` + +### 4. Multi-stem audio setup lacks teardown +Severity: Medium + +The multi-stem initialization effect creates `Audio` elements, event listeners, and Web Audio graph nodes, but does not clean them up when stems change or the component unmounts. + +Impact: +- stale audio nodes can accumulate +- event listeners remain attached longer than needed +- future stem changes are riskier than they need to be + +Relevant area: +- `frontend/src/components/LyricsLivePreview.tsx` + +### 5. Recenter path appears dead in the canvas-based version +Severity: Medium + +The current production component renders `LyricsCanvas`, while legacy `scrollDivRef` / `activeLineRef`-based recenter logic remains in the file. Those refs no longer appear to be wired into the active render path. + +Impact: +- recenter button likely no-ops +- dead code increases confusion during future changes + +Relevant area: +- `frontend/src/components/LyricsLivePreview.tsx` + +## Validation performed + +- File diagnostics check: no errors reported +- Frontend typecheck: passed via `npm run typecheck` + +## Recommended next steps + +1. Fix `resetStyle()` to restore merged defaults rather than raw `DEFAULT_STYLE` +2. Rehydrate speaker state when `storageKey` changes +3. Stop or gate the RAF loop while idle +4. Add cleanup for multi-stem audio resources +5. Remove or replace dead recenter logic if `LyricsCanvas` fully owns layout now + +## Follow-up implemented + +Implemented after this review was written: + +- `resetStyle()` now restores merged defaults derived from incoming `styleOptions` +- speaker preferences are reloaded when `storageKey` changes, with a hydration guard to avoid stale writes +- the RAF loop now stops while playback is paused instead of running continuously while idle + +Still remaining from the review: + +- add teardown for multi-stem audio resources +- remove or replace dead recenter logic if the canvas path fully owns layout diff --git a/20260317_lrc_and_lyric_transcriber_review.md b/20260317_lrc_and_lyric_transcriber_review.md new file mode 100644 index 0000000..bc9b62d --- /dev/null +++ b/20260317_lrc_and_lyric_transcriber_review.md @@ -0,0 +1,152 @@ +# 2026-03-17 Review: `lrc_service.py` and `lyric_transcriber_service.py` + +## Scope +- `backend/services/lrc_service.py` +- `backend/services/lyric_transcriber_service.py` + +## Executive summary +- **High-risk issues:** 3 +- **Medium-risk issues:** 3 +- **Low-risk issues:** 2 + +The most urgent problem is the timeout behavior in the Whisper fallback path: the worker thread may continue running after timeout while GPU lock/state is released, enabling overlapping GPU-heavy jobs. + +--- + +## Findings + +## High + +### 1) Timeout path leaves transcription thread running while shared GPU lock is released +**File:** `backend/services/lyric_transcriber_service.py` +**Evidence:** around thread creation/join timeout (`_t.join(timeout=...)`, `_t.is_alive()`) near lines 1987-1999, with lock release in `finally` near lines 2108-2113. + +**Why this is risky:** +- On timeout, function returns error but daemon thread continues using Whisper/GPU in background. +- `finally` still runs and releases `jm.gpu_vram_lock`, so another job can start and contend for VRAM with the still-running timed-out thread. +- Inconsistent state: job marked failed, but expensive compute may still be active. + +**Recommended fix:** +- Replace ad-hoc thread timeout with process-level timeout/cancellation boundary, or keep lock held until worker exits. +- If thread timeout is retained, do **not** release shared GPU lock until worker confirms exit. +- Add a watchdog flag and explicit “detached background cleanup in progress” state if unavoidable. + +--- + +### 2) Global monkey patch overrides all `torch.load()` calls to `weights_only=False` +**File:** `backend/services/lyric_transcriber_service.py` +**Evidence:** startup patch around lines 30-49. + +**Why this is risky:** +- Affects global process behavior, not just WhisperX code paths. +- Disables safer default for all downstream libraries in this process. +- Makes future security posture and debugging harder. + +**Recommended fix:** +- Scope the compatibility workaround to specific loader call sites instead of monkey-patching globally. +- If patch must remain, gate behind explicit env flag and emit startup warning. + +--- + +### 3) Secondary diarization entrypoint bypasses GPU coordination/cleanup standards +**File:** `backend/services/lyric_transcriber_service.py` +**Evidence:** `run_lyric_transcription_with_diarization()` near lines 2115+. + +**Why this is risky:** +- Does not use the shared `jm.gpu_vram_lock` path used in `run_lyric_transcription()`. +- Has no explicit `_clear_cuda_cache()`/resource-finalization symmetry. +- Can diverge behaviorally and regress memory stability under load. + +**Recommended fix:** +- Route diarization workflow through the primary orchestration path or extract shared resource-guard wrappers. +- Ensure lock acquisition/release + cache cleanup is consistent in all GPU-heavy entrypoints. + +--- + +## Medium + +### 4) HTML preview duration logic reads wrong data attribute +**File:** `backend/services/lrc_service.py` +**Evidence:** elements are rendered with `data-time` (lines ~550, ~562), but duration scanner queries `[data-start]` (line ~701). + +**Impact:** +- `allTimestamps` resolves empty. +- Preview duration falls back to hardcoded default (300s), causing inaccurate seek/play behavior. + +**Recommended fix:** +- Query `data-time` consistently, or write explicit numeric seconds attributes for robust parsing. + +--- + +### 5) Styled HTML generation inserts unescaped dynamic content +**File:** `backend/services/lrc_service.py` +**Evidence:** string interpolation of `title`, `lyrics_html`, and style values into full HTML template. + +**Impact:** +- If source text/style contains HTML/JS payloads, generated preview can execute script when opened. +- Risk profile depends on trust model, but this is still unsafe-by-default. + +**Recommended fix:** +- Escape HTML entities for text payloads. +- Strictly validate/allowlist style values (font, colors, alignment) before interpolation. +- Prefer templating with escaping helpers. + +--- + +### 6) Broad exception swallowing in temp-file cleanup hides real IO issues +**File:** `backend/services/lrc_service.py` +**Evidence:** bare `except: pass` cleanup blocks around lines ~333-334 and ~420-421. + +**Impact:** +- Silent failures for filesystem/permission issues. +- Harder incident diagnosis when temp files accumulate. + +**Recommended fix:** +- Catch `OSError` explicitly and log at debug/warn with path context. + +--- + +## Low + +### 7) `run_lyric_transcription_with_diarization()` appears functionally redundant +**File:** `backend/services/lyric_transcriber_service.py` +**Impact:** +- Duplicate orchestration path increases maintenance burden and drift risk. + +**Recommended fix:** +- Deprecate legacy path and consolidate on `run_lyric_transcription()`. + +### 8) `_ensure_vocals()` extraction is not cancellation-aware +**File:** `backend/services/lyric_transcriber_service.py` +**Impact:** +- Long Demucs extraction can continue despite job cancellation intent. + +**Recommended fix:** +- Poll cancellation before/after extraction and abort subprocess when cancellation is detected. + +--- + +## Prioritized fix plan + +## Phase 1 (Immediate) +1. Fix timeout/lock race in Whisper fallback thread path. +2. Add guardrails around global `torch.load` patch (feature flag + warnings) or remove global patch. +3. Make secondary diarization path use shared GPU lock + cleanup (or route through primary flow). + +## Phase 2 +4. Fix HTML `data-time`/duration attribute mismatch. +5. Replace bare cleanup `except` blocks with typed exception handling + logs. + +## Phase 3 +6. Harden HTML preview generation with escaping/allowlisting. +7. Consolidate/deprecate redundant diarization entrypoint. +8. Add cancellation-aware subprocess handling in `_ensure_vocals()`. + +--- + +## Suggested validation checklist after fixes +- Timeout test: force Whisper timeout and verify no lingering worker consumes GPU after job failure. +- Concurrency test: start separation + lyric job; confirm lock behavior prevents overlap. +- Preview test: generated HTML duration matches last lyric timestamp. +- Security test: inject HTML-like lyrics/title and confirm preview displays escaped text. +- Cleanup test: simulate temp-file deletion failure and verify diagnostic log appears. diff --git a/20260317_lrc_lyric_transcriber_remediation.md b/20260317_lrc_lyric_transcriber_remediation.md new file mode 100644 index 0000000..d6b3e5f --- /dev/null +++ b/20260317_lrc_lyric_transcriber_remediation.md @@ -0,0 +1,51 @@ +# 2026-03-17 Remediation Implemented: `lrc_service.py` + `lyric_transcriber_service.py` + +## Scope +- `backend/services/lrc_service.py` +- `backend/services/lyric_transcriber_service.py` + +## Implemented changes + +### 1) `lrc_service.py` + +- Fixed HTML preview duration parsing bug: + - JS now derives duration from `.lyric-line` and `.word` elements using `data-time`. + - Removed incorrect `[data-start]` query path. + +- Hardened HTML generation: + - Escaped dynamic lyric/title text via `html.escape()` before interpolation. + - Added style sanitizers for: + - colors (`hex/rgb/rgba/hsl/hsla` pattern check), + - numeric ranges (`fontSize`, `lineSpacing`), + - text alignment (`left|center|right`), + - font family conservative allow-list. + +- Improved temp-file cleanup diagnostics: + - Replaced bare `except: pass` cleanup blocks with `except OSError as exc` and debug logging. + +### 2) `lyric_transcriber_service.py` + +- Added guardrail for global `torch.load` compatibility patch: + - Patch is now controlled by env var `STEMENGINE_PATCH_TORCH_LOAD` (default enabled). + - Emits startup warning when active. + +- Removed timeout-thread race in Whisper path: + - Replaced detached timeout thread pattern with inline `transcribe_lyrics(...)` execution under shared GPU lock. + - Eliminates case where timed-out background thread keeps GPU active after lock release. + +- Aligned legacy diarization entrypoint with GPU lifecycle safeguards: + - `run_lyric_transcription_with_diarization()` now acquires/releases `jm.gpu_vram_lock`. + - Applies CUDA limits/cache cleanup symmetry (`_apply_cuda_limits()`, `_clear_cuda_cache()`). + +## Validation + +- Syntax validation executed: + - `python3 -m py_compile backend/services/lrc_service.py backend/services/lyric_transcriber_service.py` + - Result: success (no syntax errors). + +## Remaining follow-up items + +- Optional hardening: + - Make `_ensure_vocals()` subprocess fully cancellation-aware during long Demucs runs. + - Consider replacing global `torch.load` patch with narrower call-site overrides once upstream deps are stable. + - Consolidate/deprecate legacy `run_lyric_transcription_with_diarization()` if primary path covers all needs. diff --git a/README.md b/README.md index f60680f..31b45ad 100755 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ - **Stem Separation** — 40+ models across 10 model families - **MIDI Transcription** — basic-pitch with polyphonic and melody modes +- **GPU Safety Limits** — shared VRAM reserve for separation, lyric transcription, basic-pitch, and Omnizart sidecar - **ABC Notation** — convert MIDI to ABC notation for sheet music - **Lyrics Sync** — forced-alignment to produce ELRC files with word-level timestamps and a live animated preview - **Project Management** — persistent project tabs, per-stem reprocessing, stem renaming @@ -79,6 +80,8 @@ All configuration is via environment variables. The `docker-compose.yml` accepts |---|---|---| | `CPU_SEPARATION` | `false` | Set to `true` to force CPU inference | | `MAX_UPLOAD_MB` | `500` | Maximum upload file size in MB | +| `STEMENGINE_GPU_VRAM_RESERVE_MB` | `2048` | Keeps this many MB free on the GPU (WSL2-safe headroom) | +| `OMNIZART_GPU_VRAM_RESERVE_MB` | `STEMENGINE_GPU_VRAM_RESERVE_MB` | Optional sidecar override for Omnizart TensorFlow GPU cap | | `TZ` | `UTC` | Timezone | | `NAVSERVER` | — | Subsonic server URL (can also be set in the UI) | | `NAVUSER` | — | Subsonic username | @@ -166,4 +169,6 @@ docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi PyTorch wheels are self-contained (cu124) and do not require a matching CUDA toolkit on the host — only the driver. -`onnxruntime-gpu` is used for MDX-Net and basic-pitch ONNX inference; it auto-selects `CUDAExecutionProvider` when a GPU is available. +`STEMENGINE_GPU_VRAM_RESERVE_MB` is applied in backend GPU code paths (separation, lyric transcription, and basic-pitch startup) to preserve display headroom on WSL2. + +Omnizart sidecar now applies a TensorFlow memory cap using `OMNIZART_GPU_VRAM_RESERVE_MB` (or `STEMENGINE_GPU_VRAM_RESERVE_MB` fallback), so sidecar transcription follows the same reserve policy. diff --git a/backend/services/lrc_service.py b/backend/services/lrc_service.py index e6921c9..ec25264 100755 --- a/backend/services/lrc_service.py +++ b/backend/services/lrc_service.py @@ -3,6 +3,7 @@ LRC alignment service for creating enhanced ELRC files with word-level timestamp Uses the lyrics-sync module for vocal extraction and forced alignment. """ import logging +import html import os import re import sys @@ -215,30 +216,44 @@ def run_lrc_alignment( # Strip section markers ([Chorus], [Verse 1], etc.) and blank lines if not lyrics_synced: lyrics_text = clean_lyrics_text(lyrics_text) - - # Choose sync class: skip Demucs when audio is already vocals. - # In eco mode, force CPU for the phoneme recogniser to avoid GPU contention - # that causes display glitches on shared-GPU (WSL2) setups. + + # In eco mode, force CPU for the phoneme recogniser to avoid GPU + # contention that causes display glitches on shared-GPU (WSL2) setups. _align_device = "cpu" if lrc_power_preset == "eco" else None SyncClass = _LyricsSyncNoExtract if is_vocals else LyricsSync - # If lyrics are already synced (LRC format), convert to ELRC with word-level timestamps - if lyrics_synced: - jm.update_job(job_id, progress="Lyrics already synced. Enhancing to word-level...") - _enhance_existing_lrc( - job_id, audio_path, lyrics_text, out_path, - output_filename, SyncClass, style_options, timing_precision, - align_device=_align_device, - ) - else: - # Run full alignment pipeline - jm.update_job(job_id, progress="Starting lyrics alignment...") - _run_full_alignment( - job_id, audio_path, lyrics_text, out_path, - output_filename, SyncClass, style_options, timing_precision, - align_device=_align_device, - ) - + # Flush VRAM and apply the per-process cap before loading any model. + # Acquire the shared GPU lock so this job cannot overlap with a + # simultaneous WhisperX/parakeet transcription job in the other executor. + from .separation.execution import flush_vram_cache + from .lyric_transcriber_service import _apply_cuda_limits + flush_vram_cache() + _apply_cuda_limits() + jm.gpu_vram_lock.acquire() + try: + # If lyrics are already synced (LRC format), convert to ELRC with word-level timestamps + if lyrics_synced: + jm.update_job(job_id, progress="Lyrics already synced. Enhancing to word-level...") + _enhance_existing_lrc( + job_id, audio_path, lyrics_text, out_path, + output_filename, SyncClass, style_options, timing_precision, + align_device=_align_device, + ) + else: + # Run full alignment pipeline + jm.update_job(job_id, progress="Starting lyrics alignment...") + _run_full_alignment( + job_id, audio_path, lyrics_text, out_path, + output_filename, SyncClass, style_options, timing_precision, + align_device=_align_device, + ) + finally: + flush_vram_cache() + try: + jm.gpu_vram_lock.release() + except RuntimeError: + pass # already released (should not happen) + except Exception as exc: log.debug("[%s] lrc_alignment failed — %s: %s", job_id, type(exc).__name__, exc) jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.") @@ -330,8 +345,8 @@ def _run_full_alignment( # Cleanup temp file try: os.unlink(lyrics_temp_path) - except: - pass + except OSError as exc: + log.debug("[%s] temp-file cleanup failed (%s): %s", job_id, lyrics_temp_path, exc) def _enhance_existing_lrc( @@ -417,8 +432,8 @@ def _enhance_existing_lrc( finally: try: os.unlink(lyrics_temp_path) - except: - pass + except OSError as exc: + log.debug("[%s] temp-file cleanup failed (%s): %s", job_id, lyrics_temp_path, exc) def _fmt_ts(seconds: float) -> str: @@ -522,15 +537,44 @@ def _generate_styled_html(elrc_content: str, style_options: dict, title: str) -> """ from .metadata_util import parse_elrc - # Extract style options with defaults - font_family = style_options.get('fontFamily', 'system-ui') - font_size = style_options.get('fontSize', 24) - text_color = style_options.get('textColor', '#e2e8f0') - highlight_color = style_options.get('highlightColor', '#3b82f6') - background_color = style_options.get('backgroundColor', '#0f172a') - word_highlight_color = style_options.get('wordHighlightColor', '#60a5fa') - line_spacing = style_options.get('lineSpacing', 1.5) - text_align = style_options.get('textAlign', 'center') + def _safe_float(value, default: float, min_v: float, max_v: float) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + return default + return max(min_v, min(max_v, parsed)) + + def _safe_text_align(value: str) -> str: + return value if value in {'left', 'center', 'right'} else 'center' + + def _safe_color(value: str, default: str) -> str: + if not isinstance(value, str): + return default + candidate = value.strip() + color_re = re.compile( + r'^(#[0-9a-fA-F]{3,8}|rgb\([^\)]{1,64}\)|rgba\([^\)]{1,64}\)|' + r'hsl\([^\)]{1,64}\)|hsla\([^\)]{1,64}\))$' + ) + return candidate if color_re.match(candidate) else default + + def _safe_font_family(value: str) -> str: + if not isinstance(value, str): + return 'system-ui, sans-serif' + candidate = value.strip() + # Conservative allow-list for CSS font-family token(s). + if not re.fullmatch(r'[A-Za-z0-9\s,\-"\']{1,120}', candidate): + return 'system-ui, sans-serif' + return candidate + + # Extract and sanitize style options with defaults + font_family = _safe_font_family(style_options.get('fontFamily', 'system-ui, sans-serif')) + font_size = _safe_float(style_options.get('fontSize', 24), 24.0, 10.0, 96.0) + text_color = _safe_color(style_options.get('textColor', '#e2e8f0'), '#e2e8f0') + highlight_color = _safe_color(style_options.get('highlightColor', '#3b82f6'), '#3b82f6') + background_color = _safe_color(style_options.get('backgroundColor', '#0f172a'), '#0f172a') + word_highlight_color = _safe_color(style_options.get('wordHighlightColor', '#60a5fa'), '#60a5fa') + line_spacing = _safe_float(style_options.get('lineSpacing', 1.5), 1.5, 1.0, 3.5) + text_align = _safe_text_align(style_options.get('textAlign', 'center')) # Parse ELRC content lines = parse_elrc(elrc_content) @@ -547,11 +591,11 @@ def _generate_styled_html(elrc_content: str, style_options: dict, title: str) -> millis = int((word_ts % 1) * 1000) time_attr = f"{minutes:02d}:{seconds:02d}.{millis:03d}" word_spans.append( - f'{word}' + f'{html.escape(word)}' ) line_html = ' '.join(word_spans) else: - line_html = line_text + line_html = html.escape(line_text) minutes = int(line_ts // 60) seconds = int(line_ts % 60) @@ -564,12 +608,14 @@ def _generate_styled_html(elrc_content: str, style_options: dict, title: str) -> lyrics_html = '\n'.join(html_lines) + safe_title = html.escape(title) + html_template = f''' - {title} - Lyrics + {safe_title} - Lyrics