diff --git a/2026-04-13_Git_Managed_Auto_Pipeline_Presets_Plan.md b/2026-04-13_Git_Managed_Auto_Pipeline_Presets_Plan.md new file mode 100644 index 0000000..f3070a5 --- /dev/null +++ b/2026-04-13_Git_Managed_Auto_Pipeline_Presets_Plan.md @@ -0,0 +1,198 @@ +# 2026-04-13 — Git-Managed Auto Pipeline Presets Plan + +## Goal +Make selected Auto Pipeline presets ship with StemEngine from Git so they are available by default on first run, while still allowing local custom presets. + +## Implementation status (2026-04-13) + +### Completed + +- ✅ Added Git-managed built-in preset catalog file: + - `backend/presets/auto_pipeline/default_presets.json` +- ✅ Added backend catalog loader + normalization/merge helpers: + - `backend/services/auto_presets_catalog.py` +- ✅ Extended settings response to expose preset layers: + - `auto_presets_user` + - `auto_presets_builtin` + - `auto_presets_effective` +- ✅ Kept patch writes for `auto_presets` as user-editable list. +- ✅ Frontend Settings hydration now prefers backend effective presets while preserving local fallback. +- ✅ One-time local→backend backfill implemented when backend user preset list is empty. +- ✅ Preset execution path (`useConvertStatus`) now prefers `auto_presets_effective` from settings API. +- ✅ Validation run: + - `frontend`: `npm run -s typecheck` (pass) + - `frontend`: `npm run -s build` (pass) + +### Remaining (optional) + +- ⏳ UI source badges (`Built-in` / `User` / `Override`) and explicit read-only treatment for built-ins. +- ⏳ CI schema validation for preset catalog JSON. + +--- + +## Recommended storage model + +Use a **two-tier preset model**: + +1. **Built-in presets (Git-managed, read-only at runtime)** + - Stored in repo, versioned with code, reviewed via PR. + - Always available after `git pull` + restart. +2. **User presets (runtime editable)** + - Stored in existing settings (`settings.json` / `auto_presets`) for per-instance customization. + +This avoids losing local changes on upgrade while guaranteeing baseline defaults from Git. + +--- + +## Where to store built-ins + +Create a versioned preset catalog file in repo, e.g.: + +- `backend/presets/auto_pipeline/default_presets.json` + +Optional split by profile: + +- `backend/presets/auto_pipeline/default_presets.json` +- `backend/presets/auto_pipeline/karaoke_defaults.json` +- `backend/presets/auto_pipeline/lightweight_defaults.json` + +JSON schema mirrors current `AutoPreset` frontend shape: + +- `name` +- `model` +- `format` +- `instrumentModels` +- `steps` + +--- + +## Loading precedence (server-side) + +At runtime, backend resolves effective presets with this order: + +1. Load built-ins from Git file(s). +2. Load user presets from settings (`auto_presets`). +3. Merge by preset `name`: + - user preset with same `name` overrides built-in. + - unique names from both sets are kept. + +Expose both views in API response: + +- `auto_presets_builtin` (read-only) +- `auto_presets_user` (editable) +- `auto_presets_effective` (merged view for execution/UI) + +If keeping API minimal, keep only `auto_presets` as effective merged list + add metadata flags per preset (`source: builtin|user`). + +--- + +## Backend plan + +### B1) Add preset catalog loader + +- New helper module, e.g. `backend/services/auto_presets_catalog.py`. +- Responsibilities: + - read JSON file(s) + - validate schema + - normalize values + - return list of built-in presets + +### B2) Add startup validation + +- On startup, validate catalog and log actionable errors. +- If invalid: + - log with file path + preset name + field error + - continue boot with empty built-in list (non-fatal) + +### B3) Extend settings API + +- Keep patch writes only for user presets (`auto_presets`). +- Return merged/effective presets to frontend. +- Optionally add endpoint: + - `GET /api/settings/auto-presets` (builtin + user + effective) + +### B4) Preserve backward compatibility + +- Existing `auto_presets` in settings remain valid. +- Migration path: none required for existing installs. + +--- + +## Frontend plan + +### F1) Show source badges + +In Settings → Auto Pipeline preset list: + +- Tag presets with source: + - `Built-in` + - `User` + - `Override` (user replacing built-in name) + +### F2) Edit behavior + +- Built-in presets are not edited in place. +- “Save” with same name creates/updates user override. +- Delete on override removes user override, revealing built-in again. +- Delete on pure built-in is disabled (or “hide locally” if later desired). + +### F3) Selection behavior + +- SingUI `preset` mode selector uses effective merged list. +- If selected preset vanishes, fallback to first effective preset with warning toast. + +--- + +## Git + deployment workflow + +1. Add/update preset JSON in repo. +2. Commit + PR review. +3. Deploy updated StemEngine image/code. +4. On restart, built-ins appear automatically. +5. Existing user presets remain intact. + +This provides “installed/pulled with Git” defaults without clobbering local runtime state. + +--- + +## Optional enhancements + +- Support env override path: + - `AUTO_PRESETS_CATALOG_PATH=/app/config/auto_presets.json` +- Support multiple catalogs and deterministic order. +- Add `version` field per preset for future migration tooling. +- Add CI check that validates preset JSON schema. + +--- + +## Acceptance criteria + +- Fresh clone/run shows built-in presets without manual setup. +- `git pull` with new presets makes them available after restart. +- User-created presets persist separately and survive upgrades. +- Built-ins are clearly identifiable in UI. +- Execution paths (Auto Pipeline dialog + SingUI preset mode) consume merged effective list. + +--- + +## Rollout sequence + +### Phase 1 (backend foundation) +- Add catalog file + loader + merge + settings response updates. + +### Phase 2 (frontend UX) +- Add source badges and edit/delete semantics for built-in vs user presets. + +### Phase 3 (hardening) +- Add schema validation tooling + CI gate. + +--- + +## Risks and mitigations + +- **Risk:** malformed preset file breaks startup behavior. + - **Mitigation:** non-fatal loader + strong validation logs. +- **Risk:** name collisions confuse users. + - **Mitigation:** explicit `source` labels and override semantics. +- **Risk:** accidental overwrite of built-ins. + - **Mitigation:** treat Git presets as read-only in API/UI. diff --git a/2026-04-13_Leading_Backing_Buttons_Sidebar_Refactor.md b/2026-04-13_Leading_Backing_Buttons_Sidebar_Refactor.md new file mode 100644 index 0000000..e6717f7 --- /dev/null +++ b/2026-04-13_Leading_Backing_Buttons_Sidebar_Refactor.md @@ -0,0 +1,242 @@ +# Leading & Backing Buttons Sidebar Refactor — 2026-04-13 + +## Overview + +**Objective**: Consolidate karaoke display controls into the performance sidebar by: +1. Moving Leading/Backing lyrics track selection buttons from overlay +2. Adding line display mode options (Scroll, Line, Word, Syllable) to the sidebar + +**Status**: ✅ Completed and validated + +--- + +## Problem Statements + +### Issue 1: Lyrics Track Button Placement +Previously, the lyrics track selection buttons (Leading, Backing, Main) were rendered in two locations: +1. **Performance mode**: Fixed position overlay at top-right of the video display +2. **Editor mode**: Inline buttons in the universal controls toolbar + +This created visual clutter and inconsistency in the UI, especially in fullscreen mode. + +### Issue 2: Display Mode Options Hidden +The line display mode selector (Scroll, Line, Word, Syllable) was only available in the style picker panel, requiring users to open a separate UI layer to access frequently-used options. + +--- + +## Solution Implemented + +### Changes Made + +#### 1. **PerformanceSidebar Component** (`frontend/src/components/PerformanceSidebar.tsx`) + +**Added Props**: +- `lyricsTrackKeys: string[]` — Available lyrics track names (e.g., `['leading', 'default', 'backing']`) +- `selectedLyricsTrack: string` — Currently selected track name +- `onSelectLyricsTrack: (key: string) => void` — Callback when user selects a track + +**New UI Section**: +- Added "LYRICS" section at the top of the sidebar (when multiple tracks exist) +- Displays track buttons with the same styling as BACKGROUND section buttons +- Responsive: Grid layout on normal widths, flex wrap on narrow viewports +- Visual feedback: Selected track button highlighted in violet + +```tsx +{/* ── LYRICS TRACKS ── */} +{lyricsTrackKeys.length > 1 && ( +
+
LYRICS
+
+ {lyricsTrackKeys.map((key) => ( + + ))} +
+
+)} +``` + +#### 2. **Display Mode Selector** (NEW) + +**Added Props to PerformanceSidebar**: +- `hasElrc: boolean` — Whether word-timing data is available +- Uses existing `style` and `patchStyle` props to manage `videoDisplayMode` + +**New UI Section**: +- Added "DISPLAY" section in the sidebar +- Shows four mode buttons: Scroll, Line, Word, Syllable +- Word and Syllable modes are disabled when ELRC (word-timing) data is unavailable +- Disabled states show visual feedback (reduced opacity, no-cursor) +- Same grid/responsive layout as other sections + +```tsx +{/* ── LINE DISPLAY ── */} +
+
DISPLAY
+
+ {(['scroll', 'line', 'word', 'syllable'] as const).map((mode) => ( + + ))} +
+
+``` + +#### 3. **LyricsLivePreview Component** (`frontend/src/components/LyricsLivePreview.tsx`) + +**Removed UI Elements**: +- Deleted the fixed-position overlay buttons from performance mode (was at top-right at 84px from top, 10px from right) +- Deleted the inline buttons from the editor mode toolbar (was after "ELRC" indicator) +- Removed unused `labelLyricsTrackKey()` function (was only used by the removed buttons) + +**Updated Sidebar Integration**: +- Added four new props passed to `PerformanceSidebar`: + - `lyricsTrackKeys={lyricsTrackKeys}` + - `selectedLyricsTrack={selectedLyricsTrack}` + - `onSelectLyricsTrack={setSelectedLyricsTrack}` + - `hasElrc={hasElrc}` (NEW - for display mode button state) + +#### 3. **Button Styling & UX** + +The lyrics track and display mode buttons now: +- Match the visual design of other sidebar sections (Background, Stems, Lyrics) +- Use consistent border and background colors +- Provide hover states for interactive feedback +- Adapt to narrow viewports by wrapping in flex layout +- Include descriptive tooltips +- For display modes: Show disabled state for unavailable options (Word/Syllable without ELRC data) + +--- + +## Files Modified + +| File | Changes | +|------|---------| +| [`frontend/src/components/PerformanceSidebar.tsx`](frontend/src/components/PerformanceSidebar.tsx#L200-L280) | Added lyrics track + display mode sections; added `hasElrc` prop | +| [`frontend/src/components/LyricsLivePreview.tsx`](frontend/src/components/LyricsLivePreview.tsx#L3190-L3230) | Removed overlay + toolbar buttons; removed `labelLyricsTrackKey()`; added sidebar props including `hasElrc` | + +--- + +## Visual Impact + +### Before +- Lyrics track buttons floating in top-right corner during karaoke (visual clutter) +- Same buttons inline in editor toolbar (competing for space) +- Display mode options only accessible through style picker panel +- Different styling in each location + +### After +- Unified "LYRICS" section in the performance sidebar for track selection +- New "DISPLAY" section in sidebar for line display modes (Scroll, Line, Word, Syllable) +- Clean toolbar without overlapping buttons in editor mode +- Single, centralized control panel for all display options +- Consistent styling across all sections +- Buttons only visible when relevant (display modes always, lyrics tracks only when multiple exist) +- Better use of screen space, especially in fullscreen +- Word/Syllable modes clearly disabled when timing data unavailable + +### Sidebar Layout (Performance Mode) +``` +┌─────────────────────┐ +│ Controls ✕ │ +├─────────────────────┤ +│ LYRICS │ +│ [Leading] [Backing] │ +├─────────────────────┤ +│ DISPLAY │ +│ [Scroll] [Line] │ +│ [Word] [Syllable] │ (Word/Syllable disabled if no ELRC) +├─────────────────────┤ +│ BACKGROUND │ +│ [Static] [Muon] │ +│ [Fluid] [Video] │ +├─────────────────────┤ +│ STEMS │ +│ ... (volume controls) +``` + +--- + +## Validation + +### TypeScript Checking +``` +✅ No errors in PerformanceSidebar.tsx +✅ No errors in LyricsLivePreview.tsx +``` + +### Production Build +``` +✅ Build succeeded in 5.75s +✅ All 2860 modules transformed successfully +✅ Bundle sizes nominal (523.33 kB gzip) +``` + +--- + +## User Experience + +**When Performance Sidebar is open**: +1. LYRICS section shows track buttons (leading, backing, main) + - Currently selected track highlighted in violet + - Clicking instantly switches displayed lyrics +2. DISPLAY section shows line display mode options + - Scroll: Horizontal scrolling of lyrics + - Line: Full-screen line display + - Word: Word-by-word sync (requires ELRC data) + - Syllable: Syllable-by-syllable sync (requires ELRC data) + - Disabled options show grayed appearance + +**When Performance Sidebar is closed**: +- No display controls visible (sidebar can be toggled with button) + +**In editor mode**: +- Lyrics tracks and display modes no longer shown in toolbar +- Cleaner interface focused on editing controls (style, timing, effects) +- Display options still accessible via style picker panel + +--- + +## Backward Compatibility + +✅ **No breaking changes**: +- PerformanceSidebar adds optional props (with sensible defaults) +- Display mode logic remains unchanged +- User preferences persist in localStorage as before +- Graceful degradation for Word/Syllable modes when timing data unavailable + +--- + +## Performance Notes + +- No additional render loops or re-renders introduced +- Button clicks directly update state via `patchStyle()` +- Conditional rendering based on `hasElrc` avoids rendering disabled buttons +- Same responsive breakpoint logic as existing sidebar sections + +--- + +## Summary + +This refactor successfully consolidates all karaoke display controls into a unified performance sidebar: +- **LYRICS section** for track selection (analogous to BACKGROUND section) +- **DISPLAY section** for line display modes +- Eliminates visual clutter from overlays and toolbars +- Improves UX by centralizing related controls +- Maintains full functionality while improving discoverability + +The sidebar now serves as the comprehensive control center for performance mode. diff --git a/2026-04-13_Muon_VisualRefinement_Plan.md b/2026-04-13_Muon_VisualRefinement_Plan.md new file mode 100644 index 0000000..73200a4 --- /dev/null +++ b/2026-04-13_Muon_VisualRefinement_Plan.md @@ -0,0 +1,1191 @@ +# Muon Visualizer — Visual Refinement (Reference Alignment) + +Date: 2026-04-13 +Scope: Bring `MuonVisualizerCanvas.tsx` and its supporting modules closer to the look of the +reference implementation at https://muon-vis.netlify.app/ (source: +https://github.com/najafmohammed/muon-music-visualizer). + +The reference uses Three.js + WebGL + UnrealBloomPass + AfterimagePass. The StemEngine version +uses Canvas 2D, which is appropriate for the use case. The changes below close the most +impactful visual gaps without changing the rendering backend. + +--- + +## Summary of changes + +| # | File | Change | +|---|---|---| +| 1 | `muon/noise.ts` | Replace hash noise with smooth value noise + add `curlNoise2D` export | +| 2 | `MuonVisualizerCanvas.tsx` | Use `curlNoise2D` in `emitParticle` | +| 3 | `MuonVisualizerCanvas.tsx` | Replace `shadowBlur` glow with per-particle radial gradient | +| 4 | `MuonVisualizerCanvas.tsx` | Beat-driven `patternShift` cycling with cooldown | +| 5 | `MuonVisualizerCanvas.tsx` | Treble → saturation modulation | +| 6 | `MuonVisualizerCanvas.tsx` | Circumference-distributed emission loop — concentric rings always fully populated | + +--- + +## Change 1 — Smooth value noise + curl field (`muon/noise.ts`) + +### Problem + +`centeredNoise2D` is a sin-hash function: + +```ts +const s = Math.sin(x * 127.1 + y * 311.7 + seed * 74.7) * 43758.5453123 +return s - Math.floor(s) +``` + +This is **not continuous** — it evaluates to a different pseudo-random value for every integer +lattice point with no interpolation between them. As a particle displacement field this produces +salt-and-pepper jitter rather than the smooth organic flows the reference exhibits. + +The reference (`src/Utils/curl/`) uses Perlin+Simplex noise from a 13 KB library fed into a +curl gradient computation (`getCurl()`), producing a **divergence-free vector field**. Particles +follow continuous, swirling streamlines rather than random walks. + +### Fix — full replacement of `noise.ts` + +```ts +// frontend/src/components/visualizers/muon/noise.ts + +function fade(t: number): number { + return t * t * t * (t * (t * 6 - 15) + 10) +} + +function lerpf(a: number, b: number, t: number): number { + return a + (b - a) * t +} + +// Deterministic float hash for a 2D integer lattice point. +function hashf(ix: number, iy: number): number { + const n = ix * 127.1 + iy * 311.7 + const s = Math.sin(n) * 43758.5453123 + return s - Math.floor(s) +} + +/** + * 2D value noise — bicubic (Hermite) interpolation between hashed lattice points. + * Returns values in [0, 1). + * `seed` shifts the sample domain so independent fields can share the same function. + */ +export function smoothNoise2D(x: number, y: number, seed = 0): number { + const ox = x + seed * 17.3 + const oy = y + seed * 31.7 + const ix = Math.floor(ox) + const iy = Math.floor(oy) + const fx = ox - ix + const fy = oy - iy + const ux = fade(fx) + const uy = fade(fy) + + const a = hashf(ix, iy ) + const b = hashf(ix + 1, iy ) + const c = hashf(ix, iy + 1) + const d = hashf(ix + 1, iy + 1) + + return lerpf(lerpf(a, b, ux), lerpf(c, d, ux), uy) +} + +/** + * 2D curl noise — divergence-free vector field derived from the gradient of smoothNoise2D. + * Returns { nx, ny } both in roughly (-1, 1). + * Because the field is curl-derived, particles following it form closed loops and spirals + * rather than random walks — visually similar to the reference implementation's behaviour. + */ +export function curlNoise2D(x: number, y: number, seed = 0): { nx: number; ny: number } { + const eps = 0.1 + const n1 = smoothNoise2D(x, y + eps, seed) - smoothNoise2D(x, y - eps, seed) + const n2 = smoothNoise2D(x + eps, y, seed) - smoothNoise2D(x - eps, y, seed) + // curl of scalar potential f: (∂f/∂y, -∂f/∂x) + // eps cancels — raw finite differences give components already in (-1, 1). + return { nx: n1, ny: -n2 } +} + +// Keep for any callers that still need a centred scalar noise value. +export function centeredNoise2D(x: number, y: number, seed = 0): number { + return smoothNoise2D(x, y, seed) * 2 - 1 +} +``` + +### Why eps = 0.1 (not 0.01) + +With eps = 0.01, the raw finite differences are proportional to the noise gradient divided +by `2 * eps = 0.02` — values up to ±50 — requiring an artificial normalisation constant that +is noise-field-dependent and hard to choose. With eps = 0.1, the differences are just the +delta between two nearby samples (max range ≈ ±1) and can be used directly. The resulting +field is coarser but has the correct divergence-free topology. + +--- + +## Change 2 — Use `curlNoise2D` in `emitParticle` (`MuonVisualizerCanvas.tsx`) + +### Current code (lines ~6, ~122–123) + +```ts +import { centeredNoise2D } from './muon/noise' +// ... +const nx = centeredNoise2D(sample.x * noiseScale + ringNorm * 1.3, sample.y * noiseScale, ring * 17.17) +const ny = centeredNoise2D(sample.y * noiseScale, sample.x * noiseScale + ringNorm * 1.9, ring * 29.37) +``` + +### Replacement + +```ts +// Import line — replace centeredNoise2D with curlNoise2D: +import { curlNoise2D } from './muon/noise' + +// In emitParticle, replace the two centeredNoise2D calls: +const { nx, ny } = curlNoise2D( + sample.x * noiseScale + ringNorm * 1.3, + sample.y * noiseScale + ringNorm * 0.9, + ring * 17.17, +) +``` + +All downstream usage (`noiseAmp`, `sx`, `sy`, `tvx`, `tvy`) is unchanged — `nx`/`ny` are still +scalar displacements in roughly (-1, 1), same contract as before. + +--- + +## Change 3 — Per-particle radial gradient replaces `shadowBlur` (`MuonVisualizerCanvas.tsx`) + +### Problem + +The reference's UnrealBloomPass produces a physically-based bloom where bright pixels bleed +into neighbours. The current implementation approximates this with `ctx.shadowBlur` on every +particle, which has two problems: + +1. **Performance**: Every `ctx.shadowBlur > 0` draw call triggers a full canvas rasterisation + of the shadow, even under `'lighter'` compositing. For 1600 particles this is 1600 + blur passes per frame. +2. **Visual**: `shadowBlur` adds a hard Gaussian halo at a fixed radius. It does not make + particles look like glowing points with a soft falloff from bright centre to transparent + edge — which is what point-sprite textures in the reference produce. + +### Fix — radial gradient per particle, remove all `shadowBlur` + +Replace the particle draw block inside the render loop: + +**Before:** +```ts +ctx.beginPath() +ctx.fillStyle = `hsla(${laneHue}, 92%, ${58 + bright * 24}%, ${alpha})` +ctx.shadowColor = p.highlightColor +ctx.shadowBlur = glow +ctx.arc(x, y, size, 0, Math.PI * 2) +ctx.fill() +``` + +**After:** +```ts +const gr = Math.max(0.5, size * 2.4) +const grad = ctx.createRadialGradient(x, y, 0, x, y, gr) +grad.addColorStop(0, `hsla(${laneHue}, ${85 + treble * 15}%, ${65 + bright * 20}%, ${alpha})`) +grad.addColorStop(0.40, `hsla(${laneHue}, 92%, ${58 + bright * 24}%, ${alpha * 0.55})`) +grad.addColorStop(1, `hsla(${laneHue}, 88%, 52%, 0)`) +ctx.fillStyle = grad +ctx.beginPath() +ctx.arc(x, y, gr, 0, Math.PI * 2) +ctx.fill() +``` + +Also remove these lines (no longer needed): + +```ts +// DELETE — remove from inside the particle loop: +ctx.shadowColor = p.highlightColor +ctx.shadowBlur = glow + +// DELETE — remove after the particle loop: +ctx.shadowBlur = 0 +``` + +The `glow` variable computation and the `glowIntensity` setting can also be removed since the +gradient naturally produces soft glow proportional to particle size. If desired, `glowIntensity` +can instead scale the outer `gr` radius: `const gr = size * (2.0 + glowIntensity * 0.8)`. + +### Why `size * 2.4` + +The reference's point sprites have a soft Gaussian falloff that extends visually well beyond the +"core" radius. Using `size * 2.4` as the gradient radius means the bright core occupies ~40% +of the drawn circle with transparent edges, approximating this look. The `particleSize` slider +continues to work correctly — it still drives `pt.size`. + +### Treble → saturation (included here) + +The innermost colour stop uses `${85 + treble * 15}%` for saturation (range 85–100%) rather +than the hardcoded 92%. This makes high-treble content appear more vivid, matching the +reference's tendency for bright/saturated hues on high-energy transients. `treble` is already +in scope in the render loop. + +--- + +## Change 4 — Beat-driven `patternShift` cycling (`MuonVisualizerCanvas.tsx`) + +### Problem + +`patternShift` currently increments only on `isBeatDrop`. The reference cycles through 16 +radius presets on every beat, giving the characteristic visual where the ring structure +noticeably rearranges on each bar. `patternShift` drives which pattern family each ring uses +— cycling it on beats makes the visualisation more dynamic between drops. + +### Current state (inside render loop) + +```ts +const dropTriggered = f.patternInfo.isBeatDrop && now - lastDropAt > 0.5 +if (dropTriggered) { + dropBoost = 1 + lastDropAt = now + patternShift += 1 +} +``` + +### Fix — add beat-driven cycle with cooldown + +Add a new closure variable alongside the other state variables: + +```ts +let lastPatternCycleAt = 0 +``` + +Replace the `dropTriggered` block with: + +```ts +const dropTriggered = f.patternInfo.isBeatDrop && now - lastDropAt > 0.5 +if (dropTriggered) { + dropBoost = 1 + lastDropAt = now + patternShift += 2 // larger jump on drop for dramatic effect + lastPatternCycleAt = now +} + +// Beat-driven cycle: every strong beat, rotate the pattern assignment across rings. +const beatCooldownSec = Math.max(0.6, 1.2 / Math.max(0.1, beatSensitivity)) +if ((f.beatDetected || phaseWrapped) && drum > 0.3 && now - lastPatternCycleAt > beatCooldownSec) { + patternShift += 1 + lastPatternCycleAt = now +} +``` + +### Why `beatCooldownSec = max(0.6, 1.2 / beatSensitivity)` + +At default `beatSensitivity = 1`, the cooldown is 1.2 s, limiting shifts to ~1 per bar at +120 BPM. Higher `beatSensitivity` lowers the cooldown (more frequent shifts). The hard floor +of 0.6 s prevents runaway cycling even at maximum sensitivity. + +--- + +## Change 6 — Concentric ring structure: circumference-distributed emission (`MuonVisualizerCanvas.tsx` + `muon/patterns.ts`) + +### Problem + +The reference renders all rings as fully-populated arcs at every frame. Particles are +positioned from a parametric ring equation each frame (via vertex shader) — they never leave +their ring path. The StemEngine implementation uses a birth/death particle pool: particles are +emitted at the ring's current angular phase and then fly away with velocity. At any given +moment, each ring has particles only at the narrow arc traversed in the last `lifespan` frames. +The other ~300° of each ring is empty. + +Two compounding issues: + +1. **Emission loop** iterates `i % divisions`. With `emitCount = 10` and `divisions = 16`, + only rings 0–9 receive emissions each frame; rings 10–15 are skipped entirely. +2. **All emissions for a ring share the same `t`** (the current global phase). Even if a ring + gets 3 particles in one frame, all 3 are at the same angular position, not spread around + the arc. + +### Fix — ring-first emission loop with circumference spread + +#### Step 1 — Make `t` an explicit parameter to `emitParticle` + +Currently `t` is computed inside the inner closure `emitOne`: + +```ts +const t = ((performance.now() * 0.0004) + ring / Math.max(1, divisions) + phaseOffset) * Math.PI * 2 +``` + +Remove this line and add `t` as a parameter of `emitParticle`: + +```ts +const emitParticle = ( + cx: number, + cy: number, + ring: number, + divisions: number, + t: number, // ← new: caller provides the angular position + beatPhase: number, + hueBase: number, + aperture: number, + mirror: boolean, + patternPreset: string, + ringRadius: number, + lifeFrames: number, + noiseScale: number, + noiseForce: number, + alphaSeed: number, + sizeSeed: number, + dynamicRadius: number, +) => { + const emitOne = (mirrorOffset: number) => { + const tFinal = t + mirrorOffset * Math.PI * 2 // mirror adds π + const pattern = patternFromPreset(patternPreset, ring + patternShift) + const ringNorm = ring / Math.max(1, divisions - 1) + const sample = samplePattern(pattern, tFinal, ringNorm, aperture, beatPhase) + // ... rest unchanged ... + } + emitOne(0) + if (mirror) emitOne(0.5) +} +``` + +#### Step 2 — Replace the emit loop + +**Before:** +```ts +for (let i = 0; i < emitCount; i += 1) { + const ring = i % divisions + const ringRadius = base + ring * spacing + emitParticle( + cx, cy, ring, divisions, beatPhase, hue + bright * 18, + aperture, mirror, preset, ringRadius, lifeFrames, + noiseScale, noiseForce, + 0.34 + drum * 0.45 + phasePulse * 0.1 + dropBoost * 0.15, + 0.45 + bright * 0.35, dynamicRadius, + ) +} +``` + +**After:** +```ts +// globalPhase advances smoothly; ring offset staggers each ring around the circle. +const globalPhase = now * 0.4 * rotationSpeed + +// spotsPerRing: how many evenly-spaced angular positions to populate on each ring this frame. +// emitDensity already accounts for particle_count slider. +const spotsPerRing = Math.max(1, Math.round((emitDensity * (0.8 + drum * 2.4 + phasePulse * 0.8 + dropBoost * 2.2)) / Math.max(1, divisions))) + +for (let ring = 0; ring < divisions; ring += 1) { + const ringRadius = base + ring * spacing + for (let j = 0; j < spotsPerRing; j += 1) { + // Spread j evenly across [0, 2π) on this ring so the arc is always populated. + const circumOffset = j / Math.max(1, spotsPerRing) + const t = (globalPhase + ring / divisions + circumOffset) * Math.PI * 2 + emitParticle( + cx, cy, ring, divisions, t, beatPhase, hue + bright * 18, + aperture, mirror, preset, ringRadius, lifeFrames, + noiseScale, noiseForce, + 0.34 + drum * 0.45 + phasePulse * 0.1 + dropBoost * 0.15, + 0.45 + bright * 0.35, dynamicRadius, + ) + } +} +``` + +#### Step 3 — Reduce particle launch velocity + +Currently particles are given significant velocity at emission (`speed ≈ 0.12–0.42`), causing +them to drift away from the ring path. The ring structure comes from accumulating the birth +positions, not from where particles are at draw time. Reduce the launch speed so particles +stay close to their birth position and the ring arc is visible rather than a dispersing cloud: + +In `emitParticle`, change: +```ts +// Before: +const speed = 0.12 + ringNorm * 0.22 + sizeSeed * 0.08 + +// After: noise provides wobble; velocity is a gentle orbital nudge only +const speed = (0.02 + ringNorm * 0.06 + sizeSeed * 0.02) * (1 + drum * 0.4) +``` + +The drag coefficient already bleeds velocity to near-zero within a few dozen frames, so older +particles settle close to their birth arc even at the old speed. Lowering the initial speed +makes the ring arc visible immediately on emission rather than only after drag has settled. + +#### Step 4 — Reduce default lifespan to shorten trail persistence + +With full-circumference emission every frame, long lifespan means every old position overlaps +newer ones — rings look muddy. A shorter lifespan (40–80 frames) keeps each ring as a clean +arc of recent positions. The `lifespan` setting (currently default 180) should default to 60: + +In the settings read block: +```ts +const lifespan = Math.max(20, Math.min(420, Math.round(s?.lifespan ?? 60))) +``` + +Users can still push `lifespan` up for a smoky trail effect. The change only shifts the +default; the slider range is unchanged. + +### Why not re-architect to persistent orbital particles + +The reference keeps particles permanently on their rings by recomputing position from +parametric equations each frame. That approach is natural in a vertex shader but requires +removing the pool-based lifetime system in Canvas 2D and rebuilding as a stateful orbital +system. The emit-loop fix above achieves the same visible result — fully populated rings at +all times — while preserving the particle lifecycle that drives the organic floating motion +between drops. The trade-off is that `spotsPerRing` particles per ring are created and +destroyed every frame, but the pool handles this correctly without allocation. + +--- + +## Edge cases + +### E0 — Pool exhaustion under new emit rate + +The old emit loop fired at most `emitCount ≈ 30` particles per frame. The new loop fires +`divisions * spotsPerRing` per frame. At `divisions = 16` and `spotsPerRing = 3` that is +48 per frame — about the same. But at high energy (`drum ≈ 1`, `dropBoost > 0`), +`spotsPerRing` can reach ~8, giving `divisions * 8 * 2 (mirror) = 256` per frame. With +`poolCap = 1600` and `lifespan = 60`, the pool holds `1600 / 60 ≈ 26.7` frames of +at-max emission — just enough before old slots start recycling. At `lifespan = 180`, the +pool runs dry in `1600 / 256 = 6.25` frames, causing visible particle reuse artefacts. + +**Guard**: cap `spotsPerRing` relative to pool capacity and lifespan: + +```ts +const maxTotalPerFrame = Math.max(1, Math.floor(poolCap / Math.max(1, lifeFrames) * 0.7)) +const spotsPerRing = Math.max( + 1, + Math.min( + Math.floor(maxTotalPerFrame / Math.max(1, divisions) / (mirror ? 2 : 1)), + Math.round((emitDensity * (0.8 + drum * 2.4 + phasePulse * 0.8 + dropBoost * 2.2)) / Math.max(1, divisions)), + ), +) +``` + +### E0b — `emitParticle` signature change requires updating all call sites + +`emitParticle` is a closure-local function called in one place (the emit loop). There is +no other call site. Verify with a TypeScript compile — the changed signature will produce a +compile error at any missed call site. No runtime risk. + +### E1 — `createRadialGradient` GC pressure + +At 1600 active particles × 60 fps = 96,000 gradient objects created per second. In V8, +short-lived objects of identical shape are typically stack-allocated or pooled; sustained +bench tests show Canvas 2D gradient creation at <1 µs/call. This is preferable to +`shadowBlur` which blits the full canvas per particle. Monitor for GC pauses in profiler if +performance degrades on low-end devices; if needed, bucket particles by size and cache one +gradient per bucket per frame. + +### E2 — Gradient `x, y` must not be NaN + +`createRadialGradient` throws if any argument is NaN. Both `x` and `y` come from: + +```ts +const wobble = Math.sin((now * 1.2 + pt.ring * 0.37 + pt.age * 0.03) * aperture) * (...) +const x = pt.x + wobble +const y = pt.y + wobble * ellipseRatio +``` + +These can only be NaN if `pt.x` / `pt.y` are NaN, which can only happen if `emitParticle` +produced NaN coordinates. The emission path has no explicit NaN guard. The minimum risk +path is `ringRadius = 0`, but `ringRadius = base + ring * spacing` where +`base = Math.min(w, h) * 0.17 * radiusScale ≥ 1 * 0.17 * 0.2 = 0.034` and `ring * spacing ≥ 0`, +so `ringRadius` is always positive. `spacing` cannot go negative since `spacingScale ≥ 0.2`. +In practice NaN should not occur, but add a guard at the top of the draw block as a safety net: + +```ts +if (!Number.isFinite(x) || !Number.isFinite(y)) { pt.active = false; continue } +``` + +### E3 — Gradient radius `gr` must be > 0 + +`createRadialGradient(x, y, 0, x, y, gr)` throws `IndexSizeError` if `gr ≤ 0`. `gr = size * 2.4` +where `size = Math.max(0.3, pt.size * particleSize * (0.4 + life * 0.9))`. With `particleSize ≥ 0.1` +and `life > 0`, the minimum `size ≥ 0.3 * 0.1 * 0.4 = 0.012`, giving `gr ≥ 0.029`. Safe. +Still, add a guard for clarity: + +```ts +const gr = Math.max(0.5, size * 2.4) +``` + +### E4 — `curlNoise2D` evaluates `smoothNoise2D` 4× per call + +`emitParticle` is called `emitCount` times per frame (max ~30). Each call invokes +`curlNoise2D` once, which calls `smoothNoise2D` 4 times, each doing 4 `Math.sin` hashes. +Total: max 30 × 4 × 4 = 480 `Math.sin` calls/frame. Previous: 30 × 2 = 60 calls. +This 8× increase is ~400 µs of additional work at worst — negligible for a 16 ms frame. + +### E5 — `centeredNoise2D` export still needed + +`noise.ts` re-exports `centeredNoise2D` (now as `smoothNoise2D * 2 - 1`). If anything else +imports `centeredNoise2D`, it continues to work without changes. The `MuonVisualizerCanvas.tsx` +import line changes from `centeredNoise2D` to `curlNoise2D` — verify no other file imports +`centeredNoise2D` before removing it entirely. + +```bash +grep -r "centeredNoise2D" frontend/src +``` + +### E6 — `patternShift += 2` on drop may skip the same family repeatedly + +`patternFromPreset` uses `ring + patternShift` modulo the family list length. Most presets have +lists of length 2–5. With `patternShift += 2` on drop and `+= 1` on beat, the increment +mod a length-2 list oscillates between two values; mod length-5 it cycles normally. Consider +`patternShift += 1` on drop and `+= 1` on beat if oscillation looks wrong during testing. + +### E7 — `lastPatternCycleAt` not initialised in the same block as other state + +`lastPatternCycleAt` must be declared with `let` at the top of the `useEffect` closure +alongside `prevBeatPhase`, `dropBoost`, etc. If declared inside the `render` function it +resets every frame. Declare at closure scope: + +```ts +let lastPatternCycleAt = 0 +``` + +### E8 — Beat cooldown denominator: `beatSensitivity` near 0 + +`beatCooldownSec = Math.max(0.6, 1.2 / Math.max(0.1, beatSensitivity))`. The `Math.max(0.1, ...)` +guard on `beatSensitivity` prevents division by zero. Setting already has `Math.max(0.1, s?.beat_sensitivity ?? 1)` +in the settings read block — double-guarded. + +### E9 — Gradient colour stops use frame-level `treble`, not per-particle + +`treble` is calculated once per frame before the particle draw loop. All particles drawn in +that frame share the same `treble` value for their saturation stop. This is intentional and +correct — `treble` reflects the current audio frame, which is consistent for all particles. +Per-particle treble would require storing it at emission time, which is not worth the overhead. + +### E10 — Drop sets `lastPatternCycleAt`, suppressing beat cycling for one full cooldown + +Inside the `dropTriggered` block, `lastPatternCycleAt = now` is set alongside the `patternShift += 2` +increment. This is **intentional**: after a dramatic drop shift, the beat-driven cycle is +suppressed for one full `beatCooldownSec` period (≥ 0.6 s), so the drop's new pattern arrangement +has time to be perceived before it starts cycling again. Do not remove or move this line. + +### E11 — `glow` variable and `glowIntensity` slot + +After removing `shadowBlur`, the `glow` variable becomes unused unless re-purposed to scale +the gradient radius (see Change 3 note). If `glowIntensity` is kept as a slider, wire it: + +```ts +const gr = Math.max(0.5, size * (2.0 + glowIntensity * 0.8)) +``` + +If not wired, remove the `glow` computation and the `glowIntensity` read to avoid lint warnings: + +```ts +// DELETE both of these lines: +const glowIntensity = Math.max(0, s?.glow_intensity ?? 1) +const glow = (10 + drum * 22 + phasePulse * 7 + dropBoost * 18) * glowIntensity +``` + +--- + +--- + +# Fluid Visualizer — Reference Parity (`FluidVisualizerCanvas.tsx` + `fluid/fluidSim.ts` + `fluid/webgl-fluid.mjs`) + +Reference: https://github.com/PavelDoGreat/Fluid-Simulation/blob/main/src/ +Vendored sim: `frontend/src/components/visualizers/fluid/webgl-fluid.mjs` (1811 lines, bundled) + +The vendored sim implements the full Navier-Stokes fluid pipeline from the reference +(advection → curl/vorticity → divergence → pressure solve → gradient subtract → dye advection) +and both post-processing effects (bloom with soft-knee prefilter, sunrays with 16-iteration +radial march). The simulation internals are correct. The gaps are all at the **JavaScript +injection layer** between `FluidVisualizerCanvas.tsx` / `fluidSim.ts` and the sim. + +--- + +## F1 (CRITICAL) — Color injection is non-functional + +### Problem + +`splatFluid` in `fluidSim.ts` ignores the `color` parameter (note the `_color` prefix): + +```ts +export function splatFluid(canvas, x, y, dx, dy, _color: FluidColor): void { + // ... + dispatchMouse(canvas, 'mousedown', px, py) + dispatchMouse(canvas, 'mousemove', mx, my) + dispatchMouse(canvas, 'mousemove', tx, ty) + window.dispatchEvent(new MouseEvent('mouseup', ...)) +} +``` + +The vendored sim's mousedown handler calls `generateColor()` to assign a random color to the +pointer. With `COLORFUL: false`, that color is then **never updated** — all splats use the +same pointer color for the lifetime of the page. The `nextPaletteColor()` calls in +`FluidVisualizerCanvas.tsx` and the entire palette / `colorSpeed` system produce values that +are passed to `splatFluid` but never reach the GPU. + +### Root cause + +The internal splat function in `webgl-fluid.mjs` is `fe(x, y, dx, dy, color)`: + +```js +function fe(o, s, a, f, p) { + // Pass 1: inject velocity (dx=a, dy=f) at (o, s) + n.uniform3f(Q.uniforms.color, a, f, 0) + // Pass 2: inject dye color p at (o, s) + n.uniform3f(Q.uniforms.color, p.r, p.g, p.b) +} +``` + +`fe` is a closure inside the module and is not exported. The only export is the initializer +function `ft` (exported as `default`). + +### Fix — Part A: expose `fe` on the canvas element (`webgl-fluid.mjs`) + +At the end of the main initializer function `ft`, before the closing brace, add: + +```js +// Expose direct programmatic splat for external callers. +// Called as: canvas._fluidSplat(x, y, dx, dy, { r, g, b }) +D._fluidSplat = fe; +``` + +`D` is the canvas element inside the closure. This attaches `fe` directly to the canvas DOM +node so `fluidSim.ts` can call it without importing internals. + +### Fix — Part B: use direct call in `splatFluid` (`fluidSim.ts`) + +Replace the mouse-event dispatch block in `splatFluid`: + +```ts +export function splatFluid( + canvas: HTMLCanvasElement, + x: number, + y: number, + dx: number, + dy: number, + color: FluidColor, // ← remove _ prefix +): void { + const inst = instances.get(canvas) + if (!inst) return + + const directSplat = (canvas as any)._fluidSplat as + | ((x: number, y: number, dx: number, dy: number, color: FluidColor) => void) + | undefined + + if (directSplat) { + // Direct path: exact color and velocity, no mouse-event overhead. + directSplat(clamp01(x), clamp01(y), dx, dy, color) + return + } + + // Fallback: sim not yet initialised (async load); drop the splat silently. +} +``` + +Remove the `dispatchMouse` calls and the `speedNorm / dragScale / tx / ty` block entirely — +they are replaced by the direct call. + +--- + +## F2 — Velocity magnitude restored by F1 + +The current mouse-event path converts `(dx, dy)` to a pixel drag distance: + +```ts +const dragScale = 0.02 + pressure * 0.09 // ≈ 0.02–0.11 +const tx = px + dx * dragScale // ~3% of input +``` + +The sim then multiplies by `SPLAT_FORCE = 6000`, giving a net velocity of ~60–660 for typical +inputs. The reference's `multipleSplats()` uses 1000 directly. Ambient splats in +`FluidVisualizerCanvas.tsx` have `force ≈ 80–520` — correct for direct injection. After +the F1 fix, `directSplat(x, y, dx, dy, color)` passes `(dx, dy)` directly as GPU velocity +uniforms with no further scaling. **No additional change needed** — the force values in +`FluidVisualizerCanvas.tsx` are already sized for direct injection. + +--- + +## F3 — SUNRAYS disabled; `SUNRAYS_WEIGHT` not mapped + +### Problem + +`SUNRAYS: false` is hardcoded in the `initFluidSim` call. The sunray effect (16-iteration +radial blur from canvas centre, with `Density=0.3, Decay=0.95, Exposure=0.7`) is one of the +most visually distinctive features of the reference, creating light-bleed streaks radiating +from bright fluid regions. With `SUNRAYS: false` it is never computed. + +> **Important**: `SUNRAYS` is compile-time in the display shader — the sim calls +> `t()` at init which calls `j.setKeywords(...)` based on the flags. Toggling `SUNRAYS` +> post-init via `updateFluidConfig` will **not** recompile the shader; it only affects +> whether the sunray framebuffers are used in the render pass. If the framebuffers were +> not created at init time, the sunrays uniform will be bound to an undefined texture. +> **SUNRAYS must be set to `true` at `initFluidSim` time.** + +### Fix — `FluidVisualizerCanvas.tsx` + +Change in `initFluidSim` call: + +```ts +SUNRAYS: true, +SUNRAYS_RESOLUTION: q === 'low' ? 96 : 196, +SUNRAYS_WEIGHT: 0.6, // initial value; updated per-frame below +``` + +Add to `updateFluidConfig` call (per-frame): + +```ts +SUNRAYS_WEIGHT: clamp(0.3 + bass * 1.4 + dropBoost * 0.8, 0.1, 2.0), +``` + +This makes sunrays pulse with bass — faint during quiet passages, intense on kicks and drops. + +--- + +## F4 — `BLOOM_SOFT_KNEE` not configured; not audio-reactive + +### Problem + +`BLOOM_SOFT_KNEE` controls the gradient width of the bloom threshold transition. The reference +default is `0.7`. At `knee = 0`, the bloom cutoff is a hard step (very high-contrast). At +`knee = 1`, the transition is maximally smooth. The vendored sim defaults this correctly +but it is never set by StemEngine. Audio-reactive knee creates a characteristic "bloom flare +on transients" effect: kick drums briefly sharpen the bloom cutoff, making bright dye regions +bloom more aggressively. + +### Fix — `FluidVisualizerCanvas.tsx` + +Add to `initFluidSim` call: + +```ts +BLOOM_SOFT_KNEE: 0.7, +``` + +Add to `updateFluidConfig` call (per-frame): + +```ts +BLOOM_SOFT_KNEE: clamp(0.65 - kick * 0.35, 0.20, 0.72), +``` + +At rest: `0.65`. On kick: drops to `0.30` — sharper bloom. Recovers as kick decays. + +--- + +## F5 — `PRESSURE` hardcoded; not audio-reactive + +### Problem + +`PRESSURE: 0.8` is hardcoded. In the reference sim, the pressure solve step uses this as the +decay multiplier when initialising the pressure buffer each frame: + +```js +clearProgram: gl.uniform1f(value, config.PRESSURE) +``` + +Higher pressure → more of the previous frame's pressure is retained → velocity field +maintains coherent eddies longer. Lower pressure → flow dissipates faster. Audio-reactive +pressure gives energetic sections more persistent turbulence. + +### Fix — add to `updateFluidConfig` call (per-frame) + +```ts +PRESSURE: clamp(0.6 + rms * 0.32, 0.40, 0.92), +``` + +Quiet: `0.60` (fast dissipation, clean transitions). Loud: `0.92` (long-lived turbulence). + +--- + +## F6 — Quality-aware bloom and sunrays resolution not configured + +### Problem + +`BLOOM_ITERATIONS` (default 8) and `BLOOM_RESOLUTION` (default 256) are not set in +`initFluidSim`. The vendored sim uses its hardcoded defaults for all quality levels. On +low-end devices this is the most expensive part of the render. + +### Fix — `FluidVisualizerCanvas.tsx`, in `initFluidSim` call + +```ts +BLOOM_ITERATIONS: q === 'low' ? 4 : 8, +BLOOM_RESOLUTION: q === 'low' ? 128 : 256, +PRESSURE_ITERATIONS: q === 'low' ? 10 : q === 'high' ? 25 : 20, +``` + +`BLOOM_ITERATIONS` and `BLOOM_RESOLUTION` are used only to build framebuffers at init time +(`initBloomFramebuffers`) and cannot be changed mid-session. `PRESSURE_ITERATIONS` is read +each frame from config and can be changed via `updateFluidConfig` — adjust it for quality +changes if the visualiser supports live quality switching. + +--- + +## F7 — `SHADING` not quality-aware + +`SHADING: true` is hardcoded. The shading pass computes a surface normal from the dye +gradient and applies diffuse lighting. It adds visual depth but has a per-pixel cost. +Disable for low quality: + +```ts +SHADING: q !== 'low', +``` + +`SHADING` is a compile-time shader keyword (same as SUNRAYS/BLOOM) — set at init only. + +--- + +## F9 — Preset system for the fluid sim + +### Background — safe vs. unsafe parameters + +The sim's `updateFluidConfig` mutates the live config object read each frame; all parameters +below are safe to change mid-session with no framebuffer rebuild. + +`BLOOM`/`SUNRAYS`/`SHADING` are compile-time `#define` flags baked into the display shader at +init (`t()` → `j.setKeywords(...)`). They **cannot** be toggled via `updateFluidConfig` — +doing so changes the JS config but the shader remains unchanged. Any preset variant that +needs different flag combinations requires a full `destroyFluidSim` + `initFluidSim` cycle. +Avoid this for in-session preset switching; design presets around the safe set only. + +### Preset type + +Add to `FluidVisualizerCanvas.tsx`: + +```ts +type FluidPreset = { + label: string + // Sim config base values (audio modulation applies on top) + densityBase: number // DENSITY_DISSIPATION when kick = 0 + velocityBase: number // VELOCITY_DISSIPATION when rms = 0 + curlBase: number // CURL when treble = 0, dropBoost = 0 + pressureBase: number // PRESSURE when rms = 0 + splatRadiusBase: number // SPLAT_RADIUS when kick = 0 + bloomIntensityBase: number // BLOOM_INTENSITY when audioIntensity = 0 + sunraysWeightBase: number // SUNRAYS_WEIGHT when bass = 0 + dropBoostDecay: number // dropBoost *= decay each frame (0.92–0.98) + // JS injection force multipliers (scale the 400/800/500 base constants) + ambientForce: number // scales ambient splat force (default 1.0) + burstForce: number // scales beat burst force (default 1.0) + sweepForce: number // scales phase sweep force (default 1.0) +} +``` + +### Five presets + +```ts +const FLUID_PRESETS: Record = { + ink: { + label: 'Ink', + densityBase: 0.996, velocityBase: 0.975, curlBase: 8, + pressureBase: 0.70, splatRadiusBase: 0.30, + bloomIntensityBase: 0.55, sunraysWeightBase: 0.25, + dropBoostDecay: 0.97, + ambientForce: 0.60, burstForce: 1.10, sweepForce: 0.80, + }, + lava: { + label: 'Lava', + densityBase: 0.999, velocityBase: 0.994, curlBase: 18, + pressureBase: 0.85, splatRadiusBase: 0.22, + bloomIntensityBase: 1.30, sunraysWeightBase: 0.55, + dropBoostDecay: 0.96, + ambientForce: 0.80, burstForce: 0.90, sweepForce: 1.20, + }, + storm: { + label: 'Storm', + densityBase: 0.960, velocityBase: 0.950, curlBase: 50, + pressureBase: 0.90, splatRadiusBase: 0.12, + bloomIntensityBase: 0.70, sunraysWeightBase: 1.40, + dropBoostDecay: 0.94, + ambientForce: 1.40, burstForce: 1.80, sweepForce: 1.50, + }, + mist: { + label: 'Mist', + densityBase: 0.993, velocityBase: 0.988, curlBase: 5, + pressureBase: 0.60, splatRadiusBase: 0.40, + bloomIntensityBase: 2.00, sunraysWeightBase: 0.15, + dropBoostDecay: 0.97, + ambientForce: 0.50, burstForce: 0.65, sweepForce: 0.55, + }, + plasma: { + label: 'Plasma', + densityBase: 0.972, velocityBase: 0.962, curlBase: 38, + pressureBase: 0.88, splatRadiusBase: 0.14, + bloomIntensityBase: 2.20, sunraysWeightBase: 1.60, + dropBoostDecay: 0.95, + ambientForce: 1.20, burstForce: 1.50, sweepForce: 1.80, + }, +} +``` + +### Integration into `FluidVisualizerCanvas.tsx` + +Read the preset at the top of the render loop: + +```ts +const presetKey = s?.visualization_preset ?? 'lava' +const preset = FLUID_PRESETS[presetKey] ?? FLUID_PRESETS.lava +``` + +Replace the hardcoded `dropBoost *= 0.96` with: +```ts +dropBoost *= preset.dropBoostDecay +``` + +**Delete the four helper functions** (`fluidDensityDissipation`, `fluidVelocityDissipation`, +`fluidCurl`, `fluidSplatRadius`) — they become dead code once the preset system is in place. +Note: the settings they previously consumed (`s?.background_persistence`, `s?.drag`, +`s?.blob_life`, `s?.splat_force`) are superseded by the preset's base values. `s?.decay` +and `s?.treble_pulse` are preserved in the preset-aware formulas below. + +Before the `updateFluidConfig` call, compute the two palette colours needed for `BACK_COLOR`: + +```ts +const bgRgb = hexToRgb(paletteRef.current.backgroundColor) +const highlightRgb = hexToRgb(paletteRef.current.highlightColor) +``` + +Replace the `fluidDensityDissipation`, `fluidCurl`, etc. helper calls with preset-aware versions: + +```ts +updateFluidConfig(canvas, { + DENSITY_DISSIPATION: clamp(preset.densityBase - kick * 0.04 * (s?.decay ?? 1), 0.85, 0.999), + VELOCITY_DISSIPATION: clamp(preset.velocityBase - rms * 0.04, 0.80, 0.999), + CURL: clamp(preset.curlBase + treble * 8 + dropBoost * 20, 0, 80), + PRESSURE: clamp(preset.pressureBase + rms * 0.32, 0.40, 0.92), + SPLAT_RADIUS: clamp(preset.splatRadiusBase * clamp(0.6 + kick * 0.8, 0.4, 1.4), 0.05, 0.5), + BLOOM_INTENSITY: fluidBloomIntensity(s, audioIntensity, dropBoost, preset.bloomIntensityBase), + BLOOM_THRESHOLD: fluidBloomThreshold(audioIntensity), + BLOOM_SOFT_KNEE: clamp(0.65 - kick * 0.35, 0.20, 0.72), + SUNRAYS_WEIGHT: clamp(preset.sunraysWeightBase + bass * 1.2 + dropBoost * 0.8, 0.1, 2.0), + BACK_COLOR: dropBoost > 0.08 + ? blendColor(bgRgb, highlightRgb, dropBoost * 0.15) + : bgRgb, + PAUSED: false, +}) +``` + +Update the `fluidBloomIntensity` helper signature to accept `base`: + +```ts +function fluidBloomIntensity(s?: FluidSettings, intensity = 0, dropBoost = 0, base = 0.8): number { + return clamp(base + Math.max(0, s?.treble_pulse ?? 1) * 0.28 + intensity * 1.55 + dropBoost * 0.85, 0.35, 3.5) +} +``` + +Scale the three splat force constants by `preset.ambientForce` / `burstForce` / `sweepForce`: + +```ts +// Ambient: +const force = (0.2 + rms * 0.8 + treble * 0.3 * treblePulse + idle * 0.12) * splatForce * 400 * preset.ambientForce + +// Beat burst: +const force = (0.6 + bass * 1.2 + dropBoost * 0.6) * splatForce * 800 * preset.burstForce + +// Sweep: +const force = (0.28 + kick * 0.5 + phasePulse * 0.4) * splatForce * 500 * preset.sweepForce +``` + +Add a `blendColor` helper: +```ts +function blendColor(a: FluidColor, b: FluidColor, t: number): FluidColor { + return { r: a.r + (b.r - a.r) * t, g: a.g + (b.g - a.g) * t, b: a.b + (b.b - a.b) * t } +} +``` + +### What presets cannot expose (shader literals) + +The sunrays shader has three constants compiled into GLSL — `Density = 0.3`, `Decay = 0.95`, +`Exposure = 0.7`, `ITERATIONS = 16` — not uniforms. These require editing the shader source +in `webgl-fluid.mjs`. Not worth changing for presets since `SUNRAYS_WEIGHT` is sufficient. +Similarly, bloom blur uses fixed `0.25` weights and the display shader's SHADING diffuse +floor/ceiling (`0.7, 1.0`) are shader literals. + +--- + +## F10 — Additional audio-reactive gaps in `FluidVisualizerCanvas.tsx` + +### F10a — `SPLAT_RADIUS` not audio-reactive + +`fluidSplatRadius(s)` maps only `blob_life` setting, constant within a frame. Bass/kick should +expand it — a wide Gaussian on kick creates the signature "ink drop punching into fluid" look. +The F9 preset integration already adds `kick * 0.8` modulation to `SPLAT_RADIUS`. No +additional change needed once F9 is implemented. + +### F10b — Sweep orbit radius fixed + +Beat-phase sweep traces an ellipse `(0.5 ± 0.28, 0.5 ± 0.20)` regardless of audio energy: + +```ts +// Current: +0.5 + Math.cos(phaseAngle) * 0.28, +0.5 + Math.sin(phaseAngle) * 0.20, + +// After: +const orbit = 0.22 + kick * 0.10 + dropBoost * 0.08 +0.5 + Math.cos(phaseAngle) * orbit, +0.5 + Math.sin(phaseAngle) * (orbit * 0.72), +``` + +During drops the sweep orbit expands toward the canvas edges, matching the reference's +behavior where loud sections push dye toward the boundaries. + +### F10c — `minSplatMs` throttle suppresses drop density + +The gate `ts - lastSplatAt < minSplatMs - 1` prevents extra splat passes during drops when +energy is highest and the most visual activity is wanted: + +```ts +// Current: +const minSplatMs = Math.max(minFrameMs, isLow ? 45 : 25) + +// After — bypass throttle on drops: +const minSplatMs = dropBoost > 0.5 + ? minFrameMs + : Math.max(minFrameMs, isLow ? 45 : 25) +``` + +### F10d — `BACK_COLOR` does not react to drops + +Already addressed in F9 via `blendColor(bgRgb, highlightRgb, dropBoost * 0.15)`. + +### F10e — `visualization_preset` missing from fluid interface — add it + +Confirmed absent: `VisualizerBackdrop.tsx` `fluid` block (lines 44–57) has no +`visualization_preset` field. The muon block already has it at line 42. + +In `VisualizerBackdrop.tsx`, add to the `fluid` settings block: + +```ts +fluid?: { + // ... existing fields ... + visualization_preset?: string +} +``` + +The `FluidVisualizerCanvas` receives `settings?.fluid` directly — no other wiring needed. + +--- + +## F11 — Verify vendored `webgl-fluid.mjs` against newer `Fluid-Simulation` repo + +The vendored file is based on `PavelDoGreat/WebGL-Fluid-Simulation`. The user references the +newer `PavelDoGreat/Fluid-Simulation` (`/blob/main/src/`). The newer repo uses a modular +`src/` structure rather than a single `script.js`, which suggests refactoring. Potential +differences: + +- WebGL2-first context (`webgl2` with `webgl` fallback vs. `webgl` only) +- Separate RG16F velocity texture (more efficient than RGBA16F) +- Different bloom or sunray shader parameters +- Fixes to pressure boundary conditions + +**Action**: fetch `https://raw.githubusercontent.com/PavelDoGreat/Fluid-Simulation/main/src/index.js` +(or equivalent entry point) and diff the simulation pipeline against the current vendored +file. Re-vendor from the newer repo if the pipeline differs materially. The `_fluidSplat` +attachment (F1 fix) must be re-applied after re-vendoring. + +--- + +## Fluid edge cases + +### F-E1 — `_fluidSplat` race: sim loads async, splats fire before it's ready + +`initFluidSim` calls `getWebGLFluid().then(WebGL => WebGL(canvas, config))` asynchronously. +`_fluidSplat` is not attached until after `WebGL(canvas, config)` completes. If +`splatFluid` is called before the module loads, `(canvas as any)._fluidSplat` is `undefined` +and the current fix silently drops the splat (the fallback path). This is correct behaviour +— dropping early splats is safe. The `instances.get(canvas)` guard that already exists also +returns early before the instance is registered. + +### F-E2 — Y-axis convention for `_fluidSplat` + +The internal `fe(x, y, dx, dy, color)` uses WebGL texcoord convention: `y = 0` is bottom, +`y = 1` is top. `FluidVisualizerCanvas.tsx` uses `Math.random()` for all positions (as does +the reference's `multipleSplats`), so convention is irrelevant for random splats. + +For the beat-phase sweep, explicit positions are used: +```ts +0.5 + Math.cos(phaseAngle) * 0.28, // x — correct +0.5 + Math.sin(phaseAngle) * 0.20, // y — will be upside-down relative to screen +``` +The sweep traces a clockwise ellipse on screen but counter-clockwise in texcoord space. +Visually this is undetectable. No fix needed unless strict screen-orientation is required. + +### F-E3 — `BLOOM_SOFT_KNEE = 0` causes division by zero in bloom prefilter + +The bloom prefilter shader computes: +```glsl +float knee = threshold * soft_knee + 0.0001; // reference adds epsilon +float curve0 = threshold - knee; // = threshold * (1 - soft_knee) - 0.0001 +``` + +The vendored sim adds `+ 1e-4` to `knee` exactly for this reason (line `const f = _.BLOOM_THRESHOLD * _.BLOOM_SOFT_KNEE + 1e-4`). The `clamp(0.65 - kick * 0.35, 0.20, 0.72)` formula floors at `0.20`, making division-by-zero unreachable. Safe. + +### F-E4 — `SUNRAYS` sunray framebuffers depend on init-time flag + +See note in F3: `SUNRAYS` must be `true` at `initFluidSim` time. Do not attempt to toggle it +via `updateFluidConfig` at runtime — the shader has already been compiled without the +`#define SUNRAYS` keyword and the sunray framebuffers were not created. + +### F-E5 — Direct `fe` velocity vs. current mouse-event velocity + +With the F1 fix, splat velocities increase significantly. The current ambient splat `force` +values (`splatForce * 400`) were tuned against the diluted mouse-event path. After the fix: +- Ambient splats: `force = (0.2 + rms * 0.8 + treble * 0.3) * splatForce * 400` — max ~520 + — matches the reference's `je()` range of 1000 ± 500; reasonable but may be slightly low +- Beat burst: `force = (0.6 + bass * 1.2 + dropBoost * 0.6) * splatForce * 800` — max ~2080 + — significantly more energetic; may look dramatic on drops, which is desirable + +If the ambient fluid is too energetic after the fix, reduce the `400` constant toward `200`. +Tune visually. + +### F-E6 — Multiple burst splats dispatching to the same canvas concurrently + +Beat burst fires up to ~10 `splatFluid` calls in a single frame. With direct `fe` calls, +each call writes to the velocity and dye framebuffers synchronously within the WebGL context. +Since the sim's render loop runs on its own `requestAnimationFrame`, these writes happen +outside the sim's frame boundary. The sim's double-buffer swap protects against reading +while writing: `fe` writes to `write` FBO and calls `.swap()` before reading it again. +Multiple burst calls are safe — each splat accumulates additively in the dye texture. + +--- + +## Files changed + +| File | Change | +|---|---| +| `frontend/src/components/visualizers/muon/noise.ts` | Full replacement — smooth value noise + `curlNoise2D` export | +| `frontend/src/components/visualizers/MuonVisualizerCanvas.tsx` | Import `curlNoise2D`; update `emitParticle`; replace `shadowBlur` with radial gradient; add beat-driven `patternShift`; treble → saturation; ring-first emission loop | +| `frontend/src/components/visualizers/fluid/webgl-fluid.mjs` | Attach `_fluidSplat = fe` on canvas at end of init function (F1) | +| `frontend/src/components/visualizers/fluid/fluidSim.ts` | `splatFluid`: remove `_` from color param; replace mouse-event dispatch with `canvas._fluidSplat` direct call (F1) | +| `frontend/src/components/visualizers/FluidVisualizerCanvas.tsx` | Enable SUNRAYS; add BLOOM_SOFT_KNEE, PRESSURE, quality-aware BLOOM_ITERATIONS/BLOOM_RESOLUTION/SUNRAYS_RESOLUTION/SHADING to init; add SUNRAYS_WEIGHT, BLOOM_SOFT_KNEE, PRESSURE, SPLAT_RADIUS to per-frame updateFluidConfig; add `FLUID_PRESETS`, `blendColor`, preset-aware force scaling, sweep orbit audio-reactivity, drop throttle bypass (F3–F7, F9, F10) | +| `SUMMARIES (Claude).md` | Append entry | + +--- + +## Implementation order + +### Muon changes +1. Replace `noise.ts` (Change 1) — no React changes, easy to verify in isolation +2. Add `t` parameter to `emitParticle` and update its inner `emitOne` closure (Change 6, Step 1) +3. Replace the emit loop with the ring-first structure (Change 6, Step 2) +4. Lower launch velocity and default lifespan (Change 6, Steps 3–4) +5. Verify rings are visible: run the visualiser with a silent audio context; all `divisions` + rings should appear as faint arcs even at zero audio energy +6. Update `emitParticle` to use `curlNoise2D` (Change 2) — import line + noise call +7. Add `lastPatternCycleAt = 0` and beat-driven cycle block (Change 4) +8. Remove `shadowBlur` and replace particle draw with radial gradient (Change 3) +9. Wire `glowIntensity` into gradient radius or remove `glow` variable (E11) +10. Smoke-test Muon: all rings visible simultaneously; soft particle falloff; curl flow + +### Fluid changes +11. Attach `_fluidSplat = fe` in `webgl-fluid.mjs` (F1 Part A) — one line at end of `ft`. + - Add a small marker comment near this line so re-vendor diffs can re-apply safely. +12. Update `splatFluid` in `fluidSim.ts` (F1 Part B) — remove mouse-event block, add direct call. + - Preserve existing orientation by flipping Y for direct calls (`1 - clamp01(y)`). + - Add typed local interface for canvas `_fluidSplat` (avoid raw `any`). +13. Add async init race guard in `fluidSim.ts` (`initToken` / generation id). + - Ensure stale async module load cannot attach a simulator to a replaced/destroyed canvas. +14. Update `FluidVisualizerCanvas.tsx` init config: SUNRAYS, BLOOM_SOFT_KNEE, quality params (F3, F4, F6, F7). + - Note: F5 (PRESSURE audio-reactivity) is handled in step 16 as part of F9 — do not add it here. + - Keep in mind compile-time flags (`SUNRAYS`, `SHADING`, `BLOOM`) require re-init when changed. +15. Add `FLUID_PRESETS` table and `FluidPreset` type; read `visualization_preset` from settings (F9). +16. Replace hardcoded `fluidDensityDissipation` / `fluidCurl` / etc. calls with preset-aware formulas (F9), including `PRESSURE` modulation. +17. Add `blendColor` helper; wire drop flash into `BACK_COLOR` per-frame update (F9, F10d). + - Use consistent color scale with sim expectations (avoid double-normalization to near-black). +18. Add sweep orbit audio-reactivity (F10b); drop throttle bypass (F10c). +19. Add `visualization_preset?: string` to `VisualizerSettings['fluid']` in `VisualizerBackdrop.tsx` (F10e). + - Verify downstream type paths where fluid settings are mirrored/normalized. +20. Smoke-test Fluid: play a track; verify: + - Palette colors appear in the dye (F1 fix confirmed) + - Sunrays visible radiating from bright fluid (F3) + - Bloom flares on kicks (F4) + - Background briefly flashes highlight color on drops (F9/F10d) + - Switching presets changes fluid character (ink vs. storm is dramatic) + - SPLAT_RADIUS expands on bass hits +21. (Optional) Fetch newer `Fluid-Simulation` repo and diff/re-vendor if pipeline differs (F11). +22. Re-open `SUMMARIES (Claude).md` from disk immediately before appending to avoid overwrite/collision with concurrent edits. diff --git a/2026-04-13_SingUI_Pipeline_and_Karaoke_Source_Plan.md b/2026-04-13_SingUI_Pipeline_and_Karaoke_Source_Plan.md new file mode 100644 index 0000000..36467b5 --- /dev/null +++ b/2026-04-13_SingUI_Pipeline_and_Karaoke_Source_Plan.md @@ -0,0 +1,190 @@ +# 2026-04-13 — SingUI Pipeline Selection + Direct Project Karaoke Plan + +## Goal +Make SingUI cleaner by supporting two clear paths: + +1. **Option A:** Add a Settings control to choose which processing pipeline SingUI uses when you click **Package**. +2. **Option B:** Support a workflow that **skips `.sek` packaging** and sings directly from existing StemEngine project stems in `/karaoke`. + +## Implementation status (started 2026-04-13) + +### Completed in this pass + +- ✅ Added server-persisted setting: `singui_process_mode` with validation (`auto` | `separate_only` | `pipeline` | `preset`). +- ✅ Added server-persisted optional selection: `singui_process_preset`. +- ✅ Added Settings UI control under Auto Pipeline to choose/save SingUI package behavior across all four modes. +- ✅ Added conditional preset selector in Settings when mode is `preset`. +- ✅ Wired SingUI package execution to respect `singui_process_mode`: + - `auto` = run configured additional Auto Pipeline steps. + - `separate_only` = stop after separation/project creation. +- ✅ Extended conversion execution logic to include additional process strategies: + - `pipeline` = submit `/api/pipeline` directly (source audio streamed from Subsonic endpoint into upload payload). + - `preset` = run separation using selected preset model/format/instrument refinement + execute that preset step list. +- ✅ Updated SingUI karaoke row actions to prioritize **project** playback when a linked project exists. +- ✅ Updated queue-item preference in project-linked mode to queue project sources before SEK candidates. +- ✅ Updated main action text toward project-centric flow (`Prepare Project`) and aligned status copy across Sing/Tracks/Albums (`Preparing`, `Project Ready`). +- ✅ Validation pass completed after steps 1–3: + - `frontend`: `npm run -s typecheck` (pass) + - `frontend`: `npm run -s build` (pass) +- ✅ Optional preset-storage migration completed: + - Auto Pipeline presets are now read from backend app settings (`auto_presets`) when available. + - Settings preset save/delete now persist to backend settings and keep local fallback. + - One-time backfill migrates existing local preset list to backend when server list is empty. + +### Remaining from full plan + +- None. + +--- + +## Current State (baseline) + +- SingUI package actions currently run a fixed frontend pipeline flow (`import -> separate -> optional steps`) via `useConvertStatus`. +- Auto pipeline configuration is primarily browser-local (`localStorage`) and not centrally selected for SingUI routing. +- Karaoke page already supports **project mode** (`projectId`) and **SEK mode** (`sekPackageId`), but SingUI karaoke list still emphasizes SEK matching in important paths. + +--- + +## Option A — Settings Item: Choose Song Processing Pipeline in SingUI + +### Product outcome +Add a setting like: + +- **Settings → Auto Pipeline → “SingUI Package behavior”** + - `Auto Pipeline (current behavior)` + - `Full /api/pipeline endpoint` + - `Separation only` + - (optional) `Custom preset: ` + +This makes behavior explicit and removes hidden coupling. + +### Implementation plan + +#### A1) Define configuration model (server-persisted) +Add fields to app settings so selection is shared across browsers/users of the same instance: + +- `singui_process_mode`: `'auto' | 'pipeline' | 'separate_only' | 'preset'` +- `singui_process_preset`: `string | null` (used when mode = `preset`) + +Also decide where pipeline presets live long-term: + +- **Phase 1 (fast):** keep existing local presets; setting only chooses execution mode. +- **Phase 2 (clean):** move presets to backend settings for central consistency. + +#### A2) Add Settings UI +In Settings Auto Pipeline section: + +- Add a dropdown for default SingUI process mode. +- If mode is `preset`, show a second dropdown of presets. +- Add inline summary text: “Used by Package buttons in Tracks/Albums/SingUI”. + +#### A3) Wire execution in SingUI packaging path +Refactor package execution into a single strategy resolver: + +- `mode=auto`: current `useConvertStatus` behavior. +- `mode=pipeline`: submit `/api/pipeline` with chosen defaults. +- `mode=separate_only`: only create/update project stems, no post-steps. +- `mode=preset`: execute steps from selected preset. + +Key principle: one orchestrator, multiple strategies, same status UI. + +#### A4) Migration and fallback rules +- If no setting exists, default to `auto`. +- If `preset` points to missing preset, fall back to `auto` + toast warning. +- Maintain backward compatibility with existing localStorage keys. + +#### A5) Testing/validation +- Unit: strategy resolver picks correct execution path. +- Integration: Package button in Tracks, Albums, Sing triggers selected mode. +- Regression: queue/status badges unchanged. + +### Option A acceptance criteria +- Admin/user can choose SingUI package mode in Settings. +- Package actions consistently follow selected mode across pages. +- Existing users continue working without reconfiguration. + +### Option A risk/effort +- **Effort:** Medium +- **Risk:** Low–Medium (mainly migration/consistency between local and server settings) + +--- + +## Option B — Skip `.sek`: Sing directly from existing project stems + +### Product outcome +Allow karaoke flow to use StemEngine projects directly and treat `.sek` as optional export/distribution, not required for local singing. + +### Implementation plan + +#### B1) Clarify source modes in SingUI karaoke +Rename/clarify karaoke source options: + +- `Project-linked` (primary local mode) +- `SEK Folder` (portable package mode) + +In `Project-linked` mode, do **not** depend on SEK matching to render playable rows. + +#### B2) Prioritize project playback path +For each song row: + +1. If linked project exists → show `Sing` immediately (project source). +2. If not linked and user clicks `Package` → run selected processing mode, then auto-link and enable `Sing`. +3. SEK badges can still appear as informational, but not gate playback. + +#### B3) Queue semantics +When user queues from `Project-linked` mode: + +- Create queue items as `source.kind='project'` with `projectId`. +- Keep existing SEK queue behavior unchanged in SEK mode. + +#### B4) UX cleanup +- Labels: “Package (create/update project)” instead of implying SEK export. +- Move `.sek` export language to explicit export actions only. +- Add helper text: “Local karaoke uses project stems; SEK is for portable packages.” + +#### B5) Optional backend support (if needed) +If frontend-only linking is insufficient at scale, add endpoint for stable song→project mapping with confidence score. + +### Option B acceptance criteria +- User can sing from project stems without creating `.sek`. +- Karaoke queue works end-to-end in project mode. +- `.sek` remains available but optional. + +### Option B risk/effort +- **Effort:** Low–Medium +- **Risk:** Low (most underlying project playback support already exists) + +--- + +## Decision guidance + +- Choose **Option A first** if your top priority is explicit, configurable processing behavior. +- Choose **Option B first** if your top priority is faster daily karaoke flow with less packaging friction. +- Best practical sequence: + 1. **B first** (quick UX win, immediate simplification) + 2. **A second** (clean policy control for future growth) + +--- + +## Suggested phased rollout + +### Phase 1 (1–2 days) +- Implement Option B UX/path cleanup for direct project singing. +- Keep current auto pipeline execution behavior unchanged. + +### Phase 2 (2–4 days) +- Implement Option A mode selector + strategy resolver. +- Add migration/fallback logic. + +### Phase 3 (optional hardening) +- Move pipeline preset storage from localStorage to backend settings. +- Add telemetry/toasts for mode/fallback visibility. + +--- + +## Open questions before implementation + +1. Should SingUI mode be global instance-wide or per-user/browser? +2. Do we want `preset` mode in v1, or only `auto/pipeline/separate_only`? +3. Should `Package` update existing linked projects in-place by default? +4. Do we keep SEK matching badges in Project-linked mode, or hide entirely unless SEK mode is selected? diff --git a/2026-04-13_Tendrils_Spirit_Visualizer_Plan.md b/2026-04-13_Tendrils_Spirit_Visualizer_Plan.md new file mode 100644 index 0000000..4b99d92 --- /dev/null +++ b/2026-04-13_Tendrils_Spirit_Visualizer_Plan.md @@ -0,0 +1,1140 @@ +# Tendrils + Spirit Visualizers — Implementation Plan + +Date: 2026-04-13 +References: +- Tendrils: https://github.com/keeffEoghan/keeffeoghan.github.io/tree/master/src/tendrils +- Spirit: https://github.com/edankwan/The-Spirit + +Both visualizers follow the same conventions as `MuonVisualizerCanvas.tsx` and +`FluidVisualizerCanvas.tsx`: a React component receives `featuresRef`, `paletteRef`, `settings`, +`fpsCap`, and `quality` props. Settings are read via ref every frame — no hook restarts on slider +changes. Audio values come from `AudioFeatures`. + +--- + +## Architecture overview + +| Visualizer | Rendering | Particles | Extra deps | +|------------|-----------|-----------|------------| +| Tendrils | Raw WebGL2 — no Three.js | 262 144 (512²) high / 65 536 (256²) low | none | +| Spirit | Three.js + EffectComposer | 65 536 (256²) default / 262 144 (512²) high | `three` | + +### New files + +``` +frontend/src/components/visualizers/ + TendrilsVisualizerCanvas.tsx + SpiritVisualizerCanvas.tsx + tendrils/ + tendrilsSim.ts WebGL2 init + render loop helpers + logic.glsl GPGPU physics fragment shader + render.vert Particle vertex (reads state texture) + render.frag Particle colour/alpha + flow.frag Render particles into flow-feedback FBO + screen.vert Full-screen quad vertex + spirit/ + spiritSim.ts Three.js init + render loop helpers + position.glsl GPGPU position update fragment shader + particle.vert 3D particle vertex (point or triangle) + particle.frag Life-based colour + alpha + curl.glsl 3D divergence-free curl noise (included by position.glsl) +``` + +--- + +## Plan review updates (2026-04-13) + +The following corrections were added after edge-case review. + +1. **Single clock owner per visualizer** + - Avoid dual RAF loops (component + sim) driving the same renderer. + - `tendrilsSim.ts` and `spiritSim.ts` should expose explicit frame-step/render methods, while + `TendrilsVisualizerCanvas.tsx` and `SpiritVisualizerCanvas.tsx` own FPS capping and scheduling. + +2. **WebGL context-loss handling required** + - Add `webglcontextlost` / `webglcontextrestored` listeners for both visualizers. + - On loss: cancel RAF and prevent further GL calls. + - On restore: re-init all GPU resources from current settings. + +3. **Floating render target fallback policy** + - If required float color-buffer extensions are missing, degrade in this order: + - RGBA16F / HalfFloat render targets, + - lower particle resolution, + - Canvas 2D fallback. + - Do not hard-fail initialisation on partial capability devices. + +4. **Quality transitions must reinitialize allocators** + - `rootNum` (Tendrils) and `texSize` (Spirit) are allocation-time constants. + - Changing quality at runtime should trigger full destroy/init, not in-place uniform patching. + +5. **Composer/render-target resize path must be explicit** + - Resize handler must update canvas size, camera aspect, renderer size, composer size, and + all size-coupled FBOs/render targets in one atomic branch. + +--- + +--- + +# Part 1 — Tendrils Visualizer + +## Overview + +Each particle is a GL_LINES segment: its *tail* is read from `statePrev` (last frame's position) +and its *head* from `stateCurr` (this frame). Particles move under three forces: + +1. **Smooth noise** — continuously animated field; gives organic wandering +2. **Flow feedback** — particles sample a shared FBO that accumulates their own trails; popular + paths attract later particles, creating self-reinforcing "tendril" streams +3. **Target orbit** — a beat-phase–driven point that migrates around the canvas, pulling the + swarm gently + +Between frames the canvas is not cleared; a semi-transparent overlay fades old content. + +### GPGPU state texture + +Two RGBA32F textures of size `rootNum × rootNum` are ping-ponged. +Each texel: `(posX, posY, velX, velY)` in NDC `[−1.1, 1.1]`. + +### Flow FBO + +One RGBA16F texture of the same size as the canvas. Particles are rendered into it +as `gl.POINTS`, encoding velocity direction as colour. It is not cleared between frames; +a screen-quad fade with `flowDecay` alpha decays it every frame. + +--- + +## T1 — GPGPU logic shader (`tendrils/logic.glsl`) + +This fragment shader runs once per frame on the ping-pong FBO. + +```glsl +precision highp float; + +uniform sampler2D statePrev; // previous frame [posX, posY, velX, velY] +uniform sampler2D flowTex; // flow feedback field +uniform vec2 stateSize; // rootNum × rootNum +uniform float time; // seconds +uniform float damping; // velocity decay per frame (0.02–0.12) +uniform float speedLimit; // max velocity magnitude +uniform float forceWeight; // global force multiplier +uniform float noiseWeight; // noise force amplitude +uniform float noiseScale; // noise spatial frequency +uniform float noiseSpeed; // noise time scroll speed +uniform float flowWeight; // flow-field force amplitude +uniform vec2 target; // attraction point in NDC (beat-driven) +uniform float targetWeight; // attraction force amplitude + +// ── Inline 2D simplex noise (Ashima Arts) ───────────────────────────────────── +vec3 _perm(vec3 x) { return mod(((x * 34.0) + 1.0) * x, 289.0); } +float snoise2(vec2 v) { + const vec4 C = vec4(0.211324865405187, 0.366025403784439, + -0.577350269189626, 0.024390243902439); + vec2 i = floor(v + dot(v, C.yy)); + vec2 x0 = v - i + dot(i, C.xx); + vec2 i1 = (x0.x > x0.y) ? vec2(1.0, 0.0) : vec2(0.0, 1.0); + vec4 x12 = x0.xyxy + C.xxzz; + x12.xy -= i1; + i = mod(i, 289.0); + vec3 p = _perm(_perm(i.y + vec3(0.0, i1.y, 1.0)) + i.x + vec3(0.0, i1.x, 1.0)); + vec3 m = max(0.5 - vec3(dot(x0,x0), dot(x12.xy,x12.xy), dot(x12.zw,x12.zw)), 0.0); + m = m * m * m * m; + vec3 x = 2.0 * fract(p * C.www) - 1.0; + vec3 h = abs(x) - 0.5; + vec3 ox = floor(x + 0.5); + vec3 a0 = x - ox; + m *= 1.79284291400159 - 0.85373472095314 * (a0*a0 + h*h); + vec3 g; + g.x = a0.x * x0.x + h.x * x0.y; + g.yz = a0.yz * x12.xz + h.yz * x12.yw; + return 130.0 * dot(m, g); +} +// ───────────────────────────────────────────────────────────────────────────── + +void main() { + vec2 uv = gl_FragCoord.xy / stateSize; + vec4 s = texture2D(statePrev, uv); + vec2 pos = s.xy; + vec2 vel = s.zw; + + // 1. Noise force — two perpendicular channels give x/y independently + float t = time * noiseSpeed; + float nx = snoise2(vec2(pos.x * noiseScale + t, pos.y * noiseScale)); + float ny = snoise2(vec2(pos.y * noiseScale, pos.x * noiseScale + t + 31.41)); + vec2 noiseF = vec2(nx, ny) * noiseWeight; + + // 2. Flow feedback — sample flow FBO at particle's screen UV + vec2 uvFlow = clamp(pos * 0.5 + 0.5, 0.0, 1.0); + vec2 flowSmp = texture2D(flowTex, uvFlow).xy; + vec2 flowF = (flowSmp * 2.0 - 1.0) * flowWeight; + + // 3. Target attraction + vec2 toTarget = target - pos; + vec2 targetF = toTarget * targetWeight; + + // Euler integration + vel = vel * (1.0 - damping) + (noiseF + flowF + targetF) * forceWeight; + float spd = length(vel); + if (spd > speedLimit) vel *= speedLimit / spd; + pos += vel; + + // Soft boundary: bounce inward + if (abs(pos.x) > 1.1) { vel.x *= -0.5; pos.x = clamp(pos.x, -1.1, 1.1); } + if (abs(pos.y) > 1.1) { vel.y *= -0.5; pos.y = clamp(pos.y, -1.1, 1.1); } + + gl_FragColor = vec4(pos, vel); +} +``` + +--- + +## T2 — Particle vertex shader (`tendrils/render.vert`) + +Each particle draws one GL_LINES segment (2 vertices). `vid` encodes which vertex: +`floor(vid/2)` = particle index, `mod(vid,2)` = tail (0) or head (1). + +```glsl +precision highp float; +attribute float vid; // 0 … numParticles*2-1 + +uniform sampler2D stateCurr; // [posX, posY, velX, velY] current frame +uniform sampler2D statePrev; // previous frame +uniform vec2 stateSize; // rootNum × rootNum +uniform float aspect; // canvas W/H + +varying vec4 vState; // current [pos, vel] — passed to frag +varying float vIsHead; + +vec2 idxToUV(float idx) { + return (vec2(mod(idx, stateSize.x), floor(idx / stateSize.x)) + 0.5) / stateSize; +} + +void main() { + float pidx = floor(vid / 2.0); + vIsHead = mod(vid, 2.0); + vec2 uv = idxToUV(pidx); + vState = texture2D(stateCurr, uv); + vec4 prev = texture2D(statePrev, uv); + vec2 pos = (vIsHead > 0.5) ? vState.xy : prev.xy; + // Correct for non-square canvas + pos.x /= max(1.0, aspect); + gl_Position = vec4(pos, 0.0, 1.0); +} +``` + +--- + +## T3 — Particle fragment shader (`tendrils/render.frag`) + +```glsl +precision mediump float; +uniform vec3 baseColor; // palette seed colour +uniform vec3 flowColor; // velocity-direction tint +uniform float colorMapAlpha; // blend weight of velocity colour +uniform float speedAlpha; // minimum speed for alpha (inverse power) +uniform float hueShift; // per-frame hue rotation (audio-driven) + +varying vec4 vState; +varying float vIsHead; + +void main() { + vec2 vel = vState.zw; + float speed = length(vel); + + // Alpha: faster = more visible; head brighter than tail + float a = smoothstep(0.0, max(0.00001, speedAlpha), speed); + a *= (vIsHead > 0.5) ? 1.0 : 0.25; + + // Colour: mix base colour with velocity-direction encoding + vec2 dir = speed > 0.0001 ? vel / speed : vec2(0.0); + vec3 vCol = vec3(dir.x * 0.5 + 0.5, dir.y * 0.5 + 0.5, length(dir) * 0.5); + vec3 color = mix(baseColor, vCol + flowColor, colorMapAlpha); + + gl_FragColor = vec4(color, a); +} +``` + +--- + +## T4 — Flow FBO fragment shader (`tendrils/flow.frag`) + +Rendered with the same vertex shader as T2. Additive blending; alpha controls accumulation rate. + +```glsl +precision mediump float; +varying vec4 vState; +varying float vIsHead; + +void main() { + if (vIsHead < 0.5) discard; // only head contributes to flow + vec2 vel = vState.zw; + // Encode velocity into [0,1] RG so logic.glsl can decode it + gl_FragColor = vec4(vel * 0.5 + 0.5, 0.0, 0.08); +} +``` + +--- + +## T5 — Screen-quad vertex shader (`tendrils/screen.vert`) + +Used for both the auto-fade overlay and any fullscreen blit passes. + +```glsl +attribute vec2 aPos; // quad corners: ±1 +void main() { gl_Position = vec4(aPos, 0.0, 1.0); } +``` + +--- + +## T6 — `tendrilsSim.ts` — WebGL2 setup module + +Export surface mirrors `fluidSim.ts`: + +```ts +export interface TendrilsConfig { + rootNum: number // GPGPU grid size (256 or 512) + damping: number + speedLimit: number + forceWeight: number + noiseWeight: number + noiseScale: number + noiseSpeed: number + flowWeight: number + flowDecay: number // how fast the flow FBO fades (0.002–0.02) + targetWeight: number + lineWidth: number // gl.lineWidth (clamped to 1 on most desktop GPUs) + speedAlpha: number + colorMapAlpha: number +} + +export interface TendrilsInstance { + canvas: HTMLCanvasElement + config: TendrilsConfig + // internal WebGL objects not exposed +} + +const instances = new WeakMap() + +export function initTendrilsSim(canvas: HTMLCanvasElement, cfg: TendrilsConfig): void +export function updateTendrilsConfig(canvas: HTMLCanvasElement, patch: Partial): void +export function splatTendrils(canvas: HTMLCanvasElement, ...): void // not needed for this sim +export function destroyTendrilsSim(canvas: HTMLCanvasElement): void +``` + +### Internal render loop structure (inside `initTendrilsSim`) + +```ts +// State textures +const stateA = createFloatTexture(gl, rootNum, rootNum) // RGBA32F +const stateB = createFloatTexture(gl, rootNum, rootNum) +// FBOs +const fboA = createFBO(gl, stateA) +const fboB = createFBO(gl, stateB) +const flowFBO = createFBO(gl, createFloatTexture(gl, canvas.width, canvas.height)) +// Shaders +const logicProg = createProgram(gl, screenVert, logicGLSL) +const renderProg = createProgram(gl, renderVert, renderFrag) +const flowProg = createProgram(gl, renderVert, flowFrag) +// Vertex buffer: vid = [0, 1, 2, ..., numParticles*2-1] +const vidBuf = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vidBuf) +gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(rootNum*rootNum*2).map((_,i)=>i), gl.STATIC_DRAW) + +let currFBO = fboA, currTex = stateA, prevFBO = fboB, prevTex = stateB + +function tick(ts: number) { + // 1. GPGPU logic pass — write to currFBO, read from prevTex + flowFBO.tex + // 2. Swap: [currFBO, currTex, prevFBO, prevTex] = [prevFBO, prevTex, currFBO, currTex] + // 3. Flow fade — screen quad with flowDecay alpha → flowFBO + // 4. Flow render — draw particles into flowFBO with additive blend + // 5. Canvas fade — screen quad with trailAlpha → canvas + // 6. View render — draw GL_LINES from prevTex+currTex onto canvas +} +``` + +--- + +## T7 — `TendrilsVisualizerCanvas.tsx` + +Structure is identical to `FluidVisualizerCanvas.tsx` — `useEffect` with `featuresRef`/`paletteRef` +dependency only; settings read via ref each frame. + +```tsx +export default function TendrilsVisualizerCanvas({ + featuresRef, paletteRef, width, height, settings, fpsCap, quality +}: TendrilsVisualizerCanvasProps) { + // ... canvasRef, settingsRef pattern ... + useEffect(() => { + initTendrilsSim(canvas, buildInitConfig(settings, quality)) + let raf = 0 + let dropBoost = 0, lastDropAt = 0, prevBeatPhase = 0 + + const render = (ts: number) => { + raf = window.requestAnimationFrame(render) + // FPS cap ... + const f = featuresRef.current, s = settingsRef.current, p = paletteRef.current + // derive audio values same as MuonVisualizerCanvas + // updateTendrilsConfig(canvas, { ... }) + // tendrilsTick(canvas) // advance one sim + render frame + } + raf = window.requestAnimationFrame(render) + return () => { window.cancelAnimationFrame(raf); destroyTendrilsSim(canvas) } + }, [featuresRef, paletteRef]) + return