/* StemEngine iframe-local feature extractor for Muon render graphs. */ ;(function stemengineFeatureExtractor() { 'use strict' var ENERGY_HISTORY_SIZE = 43 var BEAT_TIME_HISTORY_SIZE = 24 var BUILDUP_WINDOW = 40 var DROP_SENSITIVITY = 2.5 var SILENCE_FLOOR = 0.008 // D-3 fix: per-band peak tracker with slow decay, mirrors host's useAudioFeatures.ts:287-350. // Raw band magnitudes from getFloatFrequencyData() are tiny (0.005-0.05); without this // normalization, render-graph shader uniforms see effectively-zero values and the // visualizer doesn't visibly react. PEAK_DECAY=0.9998 matches the host's value so behavior // is consistent across both pipelines. var PEAK_DECAY = 0.9998 var PEAK_FLOOR = 0.001 var rafId = null var running = false var energyHistory = [] var beatTimes = [] var lastBeatTime = 0 var previousEnergy = 0 var bpm = 0 var beatThreshold = 1.5 var magnitudeData = null var bandPeaks = { sub: PEAK_FLOOR, bass: PEAK_FLOOR, lowMid: PEAK_FLOOR, mid: PEAK_FLOOR, highMid: PEAK_FLOOR, presence: PEAK_FLOOR, brilliance: PEAK_FLOOR, } // Energy peak tracker — same idea as bands, applied to broadband energy. var energyPeak = PEAK_FLOOR var rmsPeak = PEAK_FLOOR var FREQUENCY_BANDS = { sub: [20, 60], bass: [60, 250], lowMid: [250, 500], mid: [500, 2000], highMid: [2000, 4000], presence: [4000, 6000], brilliance: [6000, 20000], } function clamp01(value) { var n = Number(value) if (!isFinite(n)) return 0 return Math.max(0, Math.min(1, n)) } function calculateAverageAndVariance(values) { if (!values.length) return { average: 0, variance: 0 } var sum = 0 var i = 0 for (i = 0; i < values.length; i += 1) sum += values[i] var average = sum / values.length var varianceSum = 0 for (i = 0; i < values.length; i += 1) { var d = values[i] - average varianceSum += d * d } return { average: average, variance: varianceSum / values.length } } function estimateBpm() { if (beatTimes.length < 2) return bpm var intervals = [] for (var i = 1; i < beatTimes.length; i += 1) { var dt = beatTimes[i] - beatTimes[i - 1] if (dt > 0.2 && dt < 1.5) intervals.push(dt) } if (!intervals.length) return bpm var sum = 0 for (var j = 0; j < intervals.length; j += 1) sum += intervals[j] var avg = sum / intervals.length var computed = 60 / avg return Math.max(40, Math.min(220, computed)) } function extractFrequencyBands(sampleRate, magnitude) { var out = { sub: 0, bass: 0, lowMid: 0, mid: 0, highMid: 0, presence: 0, brilliance: 0, } var nyquist = sampleRate / 2 var binCount = magnitude.length || 1 var hzPerBin = nyquist / binCount var keys = ['sub', 'bass', 'lowMid', 'mid', 'highMid', 'presence', 'brilliance'] for (var k = 0; k < keys.length; k += 1) { var key = keys[k] var lowHz = FREQUENCY_BANDS[key][0] var highHz = FREQUENCY_BANDS[key][1] var start = Math.max(0, Math.floor(lowHz / hzPerBin)) var end = Math.min(binCount, Math.ceil(highHz / hzPerBin)) var span = Math.max(1, end - start) var sumSquares = 0 for (var i = start; i < end; i += 1) { var m = magnitude[i] sumSquares += m * m } var raw = Math.sqrt(sumSquares / span) // D-3 fix: divide by a slowly-decaying peak so the output is in a perceptually // useful 0..1 range. Without this, render-graph shaders see ~0.04 for a peak bass hit. bandPeaks[key] = Math.max(bandPeaks[key] * PEAK_DECAY, raw, PEAK_FLOOR) out[key] = clamp01(raw / bandPeaks[key]) } return out } function detectBeat(energy) { var now = performance.now() / 1000 energyHistory.push(energy) if (energyHistory.length > ENERGY_HISTORY_SIZE) energyHistory.shift() var stats = calculateAverageAndVariance(energyHistory) var average = stats.average var stdDev = Math.sqrt(Math.max(stats.variance, 1e-9)) var zScore = stdDev > 0 ? (energy - average) / stdDev : 0 var minInterval = 0.18 var isCandidate = energy > average * beatThreshold && zScore > 0.35 var isBeat = isCandidate && (now - lastBeatTime) > minInterval if (isBeat) { lastBeatTime = now beatTimes.push(now) if (beatTimes.length > BEAT_TIME_HISTORY_SIZE) beatTimes.shift() bpm = estimateBpm() } var beatPeriod = bpm > 0 ? 60 / bpm : 0 var timeSinceLastBeat = lastBeatTime > 0 ? now - lastBeatTime : 0 var beatPhase = beatPeriod > 0 ? (timeSinceLastBeat % beatPeriod) / beatPeriod : 0 var confidence = clamp01((zScore / 3) * 0.6 + ((energy / Math.max(average, 1e-6)) - 1) * 0.4) return { isBeat: isBeat, bpm: bpm, confidence: confidence, beatPhase: clamp01(beatPhase), timeSinceLastBeat: timeSinceLastBeat, } } function detectPatterns(energy, isBeat) { var energyDelta = energy - previousEnergy previousEnergy = energy var tail = energyHistory.slice(-BUILDUP_WINDOW) var avg = energy if (tail.length) { var sum = 0 for (var i = 0; i < tail.length; i += 1) sum += tail[i] avg = sum / tail.length } var isSilence = energy < SILENCE_FLOOR var isBuildUp = !isSilence && energyDelta > 0.01 && energy > avg * 1.02 var isBeatDrop = isBeat && !isSilence && energy > avg * DROP_SENSITIVITY return { isBeatDrop: isBeatDrop, isBuildUp: isBuildUp, isSilence: isSilence, intensity: clamp01(energy * 8), energyDelta: energyDelta, } } function extractFrame(analyser, ctx, frequencyData, timeData) { analyser.getFloatFrequencyData(frequencyData) analyser.getFloatTimeDomainData(timeData) if (!magnitudeData || magnitudeData.length !== frequencyData.length) { magnitudeData = new Float32Array(frequencyData.length) } var bins = frequencyData.length || 1 var nyquist = ctx.sampleRate / 2 var powerSum = 0 var magSum = 0 var weightedBinSum = 0 var totalEnergy = 0 for (var i = 0; i < frequencyData.length; i += 1) { var db = isFinite(frequencyData[i]) ? frequencyData[i] : -100 var mag = Math.pow(10, db / 20) magnitudeData[i] = mag var power = mag * mag powerSum += power magSum += mag weightedBinSum += i * mag totalEnergy += mag } var rawRms = Math.sqrt(powerSum / bins) var rawEnergy = totalEnergy / bins // D-3 fix: peak-normalize broadband energy and RMS too (same approach as bands). rmsPeak = Math.max(rmsPeak * PEAK_DECAY, rawRms, PEAK_FLOOR) energyPeak = Math.max(energyPeak * PEAK_DECAY, rawEnergy, PEAK_FLOOR) var rms = clamp01(rawRms / rmsPeak) var energy = clamp01(rawEnergy / energyPeak) var centroidBin = magSum > 0 ? weightedBinSum / magSum : 0 var spectralCentroidHz = (centroidBin / bins) * nyquist var spectralCentroid = clamp01(spectralCentroidHz / nyquist) var geoLogSum = 0 for (var j = 0; j < magnitudeData.length; j += 1) { geoLogSum += Math.log(Math.max(1e-12, magnitudeData[j])) } var geometricMean = Math.exp(geoLogSum / bins) var arithmeticMean = magSum / bins var spectralFlatness = arithmeticMean > 0 ? geometricMean / arithmeticMean : 0 var zc = 0 for (var t = 1; t < timeData.length; t += 1) { var prev = timeData[t - 1] var cur = timeData[t] if ((prev >= 0 && cur < 0) || (prev < 0 && cur >= 0)) zc += 1 } var zcr = timeData.length > 1 ? zc / (timeData.length - 1) : 0 var frequencyBands = extractFrequencyBands(ctx.sampleRate, magnitudeData) // detectBeat uses RAW (un-normalized) energy so the peak detector compares apples to apples // across recent history. Using normalized 0..1 energy would defeat the variance check. var beatInfo = detectBeat(rawEnergy) var patternInfo = detectPatterns(rawEnergy, beatInfo.isBeat) // Treble = mean of (presence, brilliance) — matches host's useAudioFeatures convention. // Previously used (highMid + presence + brilliance)/3 which spilled mid-range into treble. window.__STEMENGINE_MUON_AUDIO__ = { bass: frequencyBands.bass, treble: clamp01((frequencyBands.presence + frequencyBands.brilliance) / 2), rms: rms, energy: energy, beat: beatInfo.isBeat, drumIntensity: clamp01(frequencyBands.sub * 0.7 + frequencyBands.bass * 0.3), sub: frequencyBands.sub, beatPhase: beatInfo.beatPhase, bpm: beatInfo.bpm, beatConfidence: beatInfo.confidence, isBeatDrop: patternInfo.isBeatDrop, isBuildUp: patternInfo.isBuildUp, isSilence: patternInfo.isSilence, energyDelta: patternInfo.energyDelta, spectralCentroid: spectralCentroid, spectralFlatness: clamp01(spectralFlatness), zcr: clamp01(zcr), frequencyBands: frequencyBands, fft: frequencyData, waveform: timeData, __seUpdatedAt: Date.now(), } } function stop() { running = false if (rafId !== null) { cancelAnimationFrame(rafId) rafId = null } } function start() { var engine = window.__StemEngineAudioEngine__ if (!engine || typeof engine.getAnalyser !== 'function' || typeof engine.getContext !== 'function') return var analyser = engine.getAnalyser() var ctx = engine.getContext() if (!analyser || !ctx) return stop() running = true var frequencyData = new Float32Array(analyser.frequencyBinCount) var timeData = new Float32Array(analyser.fftSize) function loop() { if (!running) return extractFrame(analyser, ctx, frequencyData, timeData) rafId = requestAnimationFrame(loop) } rafId = requestAnimationFrame(loop) } function reset() { energyHistory = [] beatTimes = [] lastBeatTime = 0 previousEnergy = 0 bpm = 0 rmsPeak = PEAK_FLOOR energyPeak = PEAK_FLOOR bandPeaks.sub = PEAK_FLOOR bandPeaks.bass = PEAK_FLOOR bandPeaks.lowMid = PEAK_FLOOR bandPeaks.mid = PEAK_FLOOR bandPeaks.highMid = PEAK_FLOOR bandPeaks.presence = PEAK_FLOOR bandPeaks.brilliance = PEAK_FLOOR } window.__StemEngineFeatureExtractor__ = { start: start, stop: stop, reset: reset, setBeatThreshold: function (threshold) { var t = Number(threshold) if (!isFinite(t)) return beatThreshold = Math.max(0, Math.min(3, t)) }, } })()