332 lines
10 KiB
JavaScript
332 lines
10 KiB
JavaScript
/* StemEngine iframe-local feature extractor for Muon render graphs. */
|
|
;(function stemengineFeatureExtractor() {
|
|
'use strict'
|
|
|
|
var ENERGY_HISTORY_SIZE = 43
|
|
var BEAT_TIME_HISTORY_SIZE = 24
|
|
var BUILDUP_WINDOW = 40
|
|
var DROP_SENSITIVITY = 2.5
|
|
var SILENCE_FLOOR = 0.008
|
|
// D-3 fix: per-band peak tracker with slow decay, mirrors host's useAudioFeatures.ts:287-350.
|
|
// Raw band magnitudes from getFloatFrequencyData() are tiny (0.005-0.05); without this
|
|
// normalization, render-graph shader uniforms see effectively-zero values and the
|
|
// visualizer doesn't visibly react. PEAK_DECAY=0.9998 matches the host's value so behavior
|
|
// is consistent across both pipelines.
|
|
var PEAK_DECAY = 0.9998
|
|
var PEAK_FLOOR = 0.001
|
|
|
|
var rafId = null
|
|
var running = false
|
|
var energyHistory = []
|
|
var beatTimes = []
|
|
var lastBeatTime = 0
|
|
var previousEnergy = 0
|
|
var bpm = 0
|
|
var beatThreshold = 1.5
|
|
var magnitudeData = null
|
|
var bandPeaks = {
|
|
sub: PEAK_FLOOR,
|
|
bass: PEAK_FLOOR,
|
|
lowMid: PEAK_FLOOR,
|
|
mid: PEAK_FLOOR,
|
|
highMid: PEAK_FLOOR,
|
|
presence: PEAK_FLOOR,
|
|
brilliance: PEAK_FLOOR,
|
|
}
|
|
// Energy peak tracker — same idea as bands, applied to broadband energy.
|
|
var energyPeak = PEAK_FLOOR
|
|
var rmsPeak = PEAK_FLOOR
|
|
|
|
var FREQUENCY_BANDS = {
|
|
sub: [20, 60],
|
|
bass: [60, 250],
|
|
lowMid: [250, 500],
|
|
mid: [500, 2000],
|
|
highMid: [2000, 4000],
|
|
presence: [4000, 6000],
|
|
brilliance: [6000, 20000],
|
|
}
|
|
|
|
function clamp01(value) {
|
|
var n = Number(value)
|
|
if (!isFinite(n)) return 0
|
|
return Math.max(0, Math.min(1, n))
|
|
}
|
|
|
|
function calculateAverageAndVariance(values) {
|
|
if (!values.length) return { average: 0, variance: 0 }
|
|
var sum = 0
|
|
var i = 0
|
|
for (i = 0; i < values.length; i += 1) sum += values[i]
|
|
var average = sum / values.length
|
|
var varianceSum = 0
|
|
for (i = 0; i < values.length; i += 1) {
|
|
var d = values[i] - average
|
|
varianceSum += d * d
|
|
}
|
|
return { average: average, variance: varianceSum / values.length }
|
|
}
|
|
|
|
function estimateBpm() {
|
|
if (beatTimes.length < 2) return bpm
|
|
var intervals = []
|
|
for (var i = 1; i < beatTimes.length; i += 1) {
|
|
var dt = beatTimes[i] - beatTimes[i - 1]
|
|
if (dt > 0.2 && dt < 1.5) intervals.push(dt)
|
|
}
|
|
if (!intervals.length) return bpm
|
|
var sum = 0
|
|
for (var j = 0; j < intervals.length; j += 1) sum += intervals[j]
|
|
var avg = sum / intervals.length
|
|
var computed = 60 / avg
|
|
return Math.max(40, Math.min(220, computed))
|
|
}
|
|
|
|
function extractFrequencyBands(sampleRate, magnitude) {
|
|
var out = {
|
|
sub: 0,
|
|
bass: 0,
|
|
lowMid: 0,
|
|
mid: 0,
|
|
highMid: 0,
|
|
presence: 0,
|
|
brilliance: 0,
|
|
}
|
|
|
|
var nyquist = sampleRate / 2
|
|
var binCount = magnitude.length || 1
|
|
var hzPerBin = nyquist / binCount
|
|
var keys = ['sub', 'bass', 'lowMid', 'mid', 'highMid', 'presence', 'brilliance']
|
|
|
|
for (var k = 0; k < keys.length; k += 1) {
|
|
var key = keys[k]
|
|
var lowHz = FREQUENCY_BANDS[key][0]
|
|
var highHz = FREQUENCY_BANDS[key][1]
|
|
var start = Math.max(0, Math.floor(lowHz / hzPerBin))
|
|
var end = Math.min(binCount, Math.ceil(highHz / hzPerBin))
|
|
var span = Math.max(1, end - start)
|
|
var sumSquares = 0
|
|
for (var i = start; i < end; i += 1) {
|
|
var m = magnitude[i]
|
|
sumSquares += m * m
|
|
}
|
|
var raw = Math.sqrt(sumSquares / span)
|
|
// D-3 fix: divide by a slowly-decaying peak so the output is in a perceptually
|
|
// useful 0..1 range. Without this, render-graph shaders see ~0.04 for a peak bass hit.
|
|
bandPeaks[key] = Math.max(bandPeaks[key] * PEAK_DECAY, raw, PEAK_FLOOR)
|
|
out[key] = clamp01(raw / bandPeaks[key])
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
function detectBeat(energy) {
|
|
var now = performance.now() / 1000
|
|
|
|
energyHistory.push(energy)
|
|
if (energyHistory.length > ENERGY_HISTORY_SIZE) energyHistory.shift()
|
|
|
|
var stats = calculateAverageAndVariance(energyHistory)
|
|
var average = stats.average
|
|
var stdDev = Math.sqrt(Math.max(stats.variance, 1e-9))
|
|
var zScore = stdDev > 0 ? (energy - average) / stdDev : 0
|
|
|
|
var minInterval = 0.18
|
|
var isCandidate = energy > average * beatThreshold && zScore > 0.35
|
|
var isBeat = isCandidate && (now - lastBeatTime) > minInterval
|
|
|
|
if (isBeat) {
|
|
lastBeatTime = now
|
|
beatTimes.push(now)
|
|
if (beatTimes.length > BEAT_TIME_HISTORY_SIZE) beatTimes.shift()
|
|
bpm = estimateBpm()
|
|
}
|
|
|
|
var beatPeriod = bpm > 0 ? 60 / bpm : 0
|
|
var timeSinceLastBeat = lastBeatTime > 0 ? now - lastBeatTime : 0
|
|
var beatPhase = beatPeriod > 0 ? (timeSinceLastBeat % beatPeriod) / beatPeriod : 0
|
|
|
|
var confidence = clamp01((zScore / 3) * 0.6 + ((energy / Math.max(average, 1e-6)) - 1) * 0.4)
|
|
|
|
return {
|
|
isBeat: isBeat,
|
|
bpm: bpm,
|
|
confidence: confidence,
|
|
beatPhase: clamp01(beatPhase),
|
|
timeSinceLastBeat: timeSinceLastBeat,
|
|
}
|
|
}
|
|
|
|
function detectPatterns(energy, isBeat) {
|
|
var energyDelta = energy - previousEnergy
|
|
previousEnergy = energy
|
|
|
|
var tail = energyHistory.slice(-BUILDUP_WINDOW)
|
|
var avg = energy
|
|
if (tail.length) {
|
|
var sum = 0
|
|
for (var i = 0; i < tail.length; i += 1) sum += tail[i]
|
|
avg = sum / tail.length
|
|
}
|
|
|
|
var isSilence = energy < SILENCE_FLOOR
|
|
var isBuildUp = !isSilence && energyDelta > 0.01 && energy > avg * 1.02
|
|
var isBeatDrop = isBeat && !isSilence && energy > avg * DROP_SENSITIVITY
|
|
|
|
return {
|
|
isBeatDrop: isBeatDrop,
|
|
isBuildUp: isBuildUp,
|
|
isSilence: isSilence,
|
|
intensity: clamp01(energy * 8),
|
|
energyDelta: energyDelta,
|
|
}
|
|
}
|
|
|
|
function extractFrame(analyser, ctx, frequencyData, timeData) {
|
|
analyser.getFloatFrequencyData(frequencyData)
|
|
analyser.getFloatTimeDomainData(timeData)
|
|
|
|
if (!magnitudeData || magnitudeData.length !== frequencyData.length) {
|
|
magnitudeData = new Float32Array(frequencyData.length)
|
|
}
|
|
|
|
var bins = frequencyData.length || 1
|
|
var nyquist = ctx.sampleRate / 2
|
|
|
|
var powerSum = 0
|
|
var magSum = 0
|
|
var weightedBinSum = 0
|
|
var totalEnergy = 0
|
|
|
|
for (var i = 0; i < frequencyData.length; i += 1) {
|
|
var db = isFinite(frequencyData[i]) ? frequencyData[i] : -100
|
|
var mag = Math.pow(10, db / 20)
|
|
magnitudeData[i] = mag
|
|
|
|
var power = mag * mag
|
|
powerSum += power
|
|
magSum += mag
|
|
weightedBinSum += i * mag
|
|
totalEnergy += mag
|
|
}
|
|
|
|
var rawRms = Math.sqrt(powerSum / bins)
|
|
var rawEnergy = totalEnergy / bins
|
|
// D-3 fix: peak-normalize broadband energy and RMS too (same approach as bands).
|
|
rmsPeak = Math.max(rmsPeak * PEAK_DECAY, rawRms, PEAK_FLOOR)
|
|
energyPeak = Math.max(energyPeak * PEAK_DECAY, rawEnergy, PEAK_FLOOR)
|
|
var rms = clamp01(rawRms / rmsPeak)
|
|
var energy = clamp01(rawEnergy / energyPeak)
|
|
var centroidBin = magSum > 0 ? weightedBinSum / magSum : 0
|
|
var spectralCentroidHz = (centroidBin / bins) * nyquist
|
|
var spectralCentroid = clamp01(spectralCentroidHz / nyquist)
|
|
|
|
var geoLogSum = 0
|
|
for (var j = 0; j < magnitudeData.length; j += 1) {
|
|
geoLogSum += Math.log(Math.max(1e-12, magnitudeData[j]))
|
|
}
|
|
var geometricMean = Math.exp(geoLogSum / bins)
|
|
var arithmeticMean = magSum / bins
|
|
var spectralFlatness = arithmeticMean > 0 ? geometricMean / arithmeticMean : 0
|
|
|
|
var zc = 0
|
|
for (var t = 1; t < timeData.length; t += 1) {
|
|
var prev = timeData[t - 1]
|
|
var cur = timeData[t]
|
|
if ((prev >= 0 && cur < 0) || (prev < 0 && cur >= 0)) zc += 1
|
|
}
|
|
var zcr = timeData.length > 1 ? zc / (timeData.length - 1) : 0
|
|
|
|
var frequencyBands = extractFrequencyBands(ctx.sampleRate, magnitudeData)
|
|
// detectBeat uses RAW (un-normalized) energy so the peak detector compares apples to apples
|
|
// across recent history. Using normalized 0..1 energy would defeat the variance check.
|
|
var beatInfo = detectBeat(rawEnergy)
|
|
var patternInfo = detectPatterns(rawEnergy, beatInfo.isBeat)
|
|
|
|
// Treble = mean of (presence, brilliance) — matches host's useAudioFeatures convention.
|
|
// Previously used (highMid + presence + brilliance)/3 which spilled mid-range into treble.
|
|
window.__STEMENGINE_MUON_AUDIO__ = {
|
|
bass: frequencyBands.bass,
|
|
treble: clamp01((frequencyBands.presence + frequencyBands.brilliance) / 2),
|
|
rms: rms,
|
|
energy: energy,
|
|
beat: beatInfo.isBeat,
|
|
drumIntensity: clamp01(frequencyBands.sub * 0.7 + frequencyBands.bass * 0.3),
|
|
sub: frequencyBands.sub,
|
|
beatPhase: beatInfo.beatPhase,
|
|
bpm: beatInfo.bpm,
|
|
beatConfidence: beatInfo.confidence,
|
|
isBeatDrop: patternInfo.isBeatDrop,
|
|
isBuildUp: patternInfo.isBuildUp,
|
|
isSilence: patternInfo.isSilence,
|
|
energyDelta: patternInfo.energyDelta,
|
|
spectralCentroid: spectralCentroid,
|
|
spectralFlatness: clamp01(spectralFlatness),
|
|
zcr: clamp01(zcr),
|
|
frequencyBands: frequencyBands,
|
|
fft: frequencyData,
|
|
waveform: timeData,
|
|
__seUpdatedAt: Date.now(),
|
|
}
|
|
}
|
|
|
|
function stop() {
|
|
running = false
|
|
if (rafId !== null) {
|
|
cancelAnimationFrame(rafId)
|
|
rafId = null
|
|
}
|
|
}
|
|
|
|
function start() {
|
|
var engine = window.__StemEngineAudioEngine__
|
|
if (!engine || typeof engine.getAnalyser !== 'function' || typeof engine.getContext !== 'function') return
|
|
|
|
var analyser = engine.getAnalyser()
|
|
var ctx = engine.getContext()
|
|
if (!analyser || !ctx) return
|
|
|
|
stop()
|
|
running = true
|
|
|
|
var frequencyData = new Float32Array(analyser.frequencyBinCount)
|
|
var timeData = new Float32Array(analyser.fftSize)
|
|
|
|
function loop() {
|
|
if (!running) return
|
|
extractFrame(analyser, ctx, frequencyData, timeData)
|
|
rafId = requestAnimationFrame(loop)
|
|
}
|
|
|
|
rafId = requestAnimationFrame(loop)
|
|
}
|
|
|
|
function reset() {
|
|
energyHistory = []
|
|
beatTimes = []
|
|
lastBeatTime = 0
|
|
previousEnergy = 0
|
|
bpm = 0
|
|
rmsPeak = PEAK_FLOOR
|
|
energyPeak = PEAK_FLOOR
|
|
bandPeaks.sub = PEAK_FLOOR
|
|
bandPeaks.bass = PEAK_FLOOR
|
|
bandPeaks.lowMid = PEAK_FLOOR
|
|
bandPeaks.mid = PEAK_FLOOR
|
|
bandPeaks.highMid = PEAK_FLOOR
|
|
bandPeaks.presence = PEAK_FLOOR
|
|
bandPeaks.brilliance = PEAK_FLOOR
|
|
}
|
|
|
|
window.__StemEngineFeatureExtractor__ = {
|
|
start: start,
|
|
stop: stop,
|
|
reset: reset,
|
|
setBeatThreshold: function (threshold) {
|
|
var t = Number(threshold)
|
|
if (!isFinite(t)) return
|
|
beatThreshold = Math.max(0, Math.min(3, t))
|
|
},
|
|
}
|
|
})()
|