Files
stemengine_old/visualizer-iframes/shared/stemengine-feature-extractor.js
T
2026-04-28 17:51:01 +09:00

333 lines
10 KiB
JavaScript

/* StemEngine iframe-local feature extractor for Muon render graphs. */
;(function stemengineFeatureExtractor() {
'use strict'
var ENERGY_HISTORY_SIZE = 43
var BEAT_TIME_HISTORY_SIZE = 24
var BUILDUP_WINDOW = 40
var DROP_SENSITIVITY = 2.5
var SILENCE_FLOOR = 0.008
// D-3 fix: per-band peak tracker with slow decay, mirrors host's useAudioFeatures.ts:287-350.
// Raw band magnitudes from getFloatFrequencyData() are tiny (0.005-0.05); without this
// normalization, render-graph shader uniforms see effectively-zero values and the
// visualizer doesn't visibly react. PEAK_DECAY=0.9998 matches the host's value so behavior
// is consistent across both pipelines.
var PEAK_DECAY = 0.9998
var PEAK_FLOOR = 0.001
var rafId = null
var running = false
var energyHistory = []
var beatTimes = []
var lastBeatTime = 0
var previousEnergy = 0
var bpm = 0
var beatThreshold = 1.5
var magnitudeData = null
var bandPeaks = {
sub: PEAK_FLOOR,
bass: PEAK_FLOOR,
lowMid: PEAK_FLOOR,
mid: PEAK_FLOOR,
highMid: PEAK_FLOOR,
presence: PEAK_FLOOR,
brilliance: PEAK_FLOOR,
}
// Energy peak tracker — same idea as bands, applied to broadband energy.
var energyPeak = PEAK_FLOOR
var rmsPeak = PEAK_FLOOR
var FREQUENCY_BANDS = {
sub: [20, 60],
bass: [60, 250],
lowMid: [250, 500],
mid: [500, 2000],
highMid: [2000, 4000],
presence: [4000, 6000],
brilliance: [6000, 20000],
}
function clamp01(value) {
var n = Number(value)
if (!isFinite(n)) return 0
return Math.max(0, Math.min(1, n))
}
function calculateAverageAndVariance(values) {
if (!values.length) return { average: 0, variance: 0 }
var sum = 0
var i = 0
for (i = 0; i < values.length; i += 1) sum += values[i]
var average = sum / values.length
var varianceSum = 0
for (i = 0; i < values.length; i += 1) {
var d = values[i] - average
varianceSum += d * d
}
return { average: average, variance: varianceSum / values.length }
}
function estimateBpm() {
if (beatTimes.length < 2) return bpm
var intervals = []
for (var i = 1; i < beatTimes.length; i += 1) {
var dt = beatTimes[i] - beatTimes[i - 1]
if (dt > 0.2 && dt < 1.5) intervals.push(dt)
}
if (!intervals.length) return bpm
var sum = 0
for (var j = 0; j < intervals.length; j += 1) sum += intervals[j]
var avg = sum / intervals.length
var computed = 60 / avg
return Math.max(40, Math.min(220, computed))
}
function extractFrequencyBands(sampleRate, magnitude) {
var out = {
sub: 0,
bass: 0,
lowMid: 0,
mid: 0,
highMid: 0,
presence: 0,
brilliance: 0,
}
var nyquist = sampleRate / 2
var binCount = magnitude.length || 1
var hzPerBin = nyquist / binCount
var keys = ['sub', 'bass', 'lowMid', 'mid', 'highMid', 'presence', 'brilliance']
for (var k = 0; k < keys.length; k += 1) {
var key = keys[k]
var lowHz = FREQUENCY_BANDS[key][0]
var highHz = FREQUENCY_BANDS[key][1]
var start = Math.max(0, Math.floor(lowHz / hzPerBin))
var end = Math.min(binCount, Math.ceil(highHz / hzPerBin))
var span = Math.max(1, end - start)
var sumSquares = 0
for (var i = start; i < end; i += 1) {
var m = magnitude[i]
sumSquares += m * m
}
var raw = Math.sqrt(sumSquares / span)
// D-3 fix: divide by a slowly-decaying peak so the output is in a perceptually
// useful 0..1 range. Without this, render-graph shaders see ~0.04 for a peak bass hit.
bandPeaks[key] = Math.max(bandPeaks[key] * PEAK_DECAY, raw, PEAK_FLOOR)
out[key] = clamp01(raw / bandPeaks[key])
}
return out
}
function detectBeat(energy) {
var now = performance.now() / 1000
energyHistory.push(energy)
if (energyHistory.length > ENERGY_HISTORY_SIZE) energyHistory.shift()
var stats = calculateAverageAndVariance(energyHistory)
var average = stats.average
var stdDev = Math.sqrt(Math.max(stats.variance, 1e-9))
var zScore = stdDev > 0 ? (energy - average) / stdDev : 0
var minInterval = 0.18
var isCandidate = energy > average * beatThreshold && zScore > 0.35
var isBeat = isCandidate && (now - lastBeatTime) > minInterval
if (isBeat) {
lastBeatTime = now
beatTimes.push(now)
if (beatTimes.length > BEAT_TIME_HISTORY_SIZE) beatTimes.shift()
bpm = estimateBpm()
}
var beatPeriod = bpm > 0 ? 60 / bpm : 0
var timeSinceLastBeat = lastBeatTime > 0 ? now - lastBeatTime : 0
var beatPhase = beatPeriod > 0 ? (timeSinceLastBeat % beatPeriod) / beatPeriod : 0
var confidence = clamp01((zScore / 3) * 0.6 + ((energy / Math.max(average, 1e-6)) - 1) * 0.4)
return {
isBeat: isBeat,
bpm: bpm,
confidence: confidence,
beatPhase: clamp01(beatPhase),
timeSinceLastBeat: timeSinceLastBeat,
}
}
function detectPatterns(energy, isBeat) {
var energyDelta = energy - previousEnergy
previousEnergy = energy
var tail = energyHistory.slice(-BUILDUP_WINDOW)
var avg = energy
if (tail.length) {
var sum = 0
for (var i = 0; i < tail.length; i += 1) sum += tail[i]
avg = sum / tail.length
}
var isSilence = energy < SILENCE_FLOOR
var isBuildUp = !isSilence && energyDelta > 0.01 && energy > avg * 1.02
var isBeatDrop = isBeat && !isSilence && energy > avg * DROP_SENSITIVITY
return {
isBeatDrop: isBeatDrop,
isBuildUp: isBuildUp,
isSilence: isSilence,
intensity: clamp01(energy * 8),
energyDelta: energyDelta,
}
}
function extractFrame(analyser, ctx, frequencyData, timeData) {
analyser.getFloatFrequencyData(frequencyData)
analyser.getFloatTimeDomainData(timeData)
if (!magnitudeData || magnitudeData.length !== frequencyData.length) {
magnitudeData = new Float32Array(frequencyData.length)
}
var bins = frequencyData.length || 1
var nyquist = ctx.sampleRate / 2
var powerSum = 0
var magSum = 0
var weightedBinSum = 0
var totalEnergy = 0
for (var i = 0; i < frequencyData.length; i += 1) {
var db = isFinite(frequencyData[i]) ? frequencyData[i] : -100
var mag = Math.pow(10, db / 20)
magnitudeData[i] = mag
var power = mag * mag
powerSum += power
magSum += mag
weightedBinSum += i * mag
totalEnergy += mag
}
var rawRms = Math.sqrt(powerSum / bins)
var rawEnergy = totalEnergy / bins
// D-3 fix: peak-normalize broadband energy and RMS too (same approach as bands).
rmsPeak = Math.max(rmsPeak * PEAK_DECAY, rawRms, PEAK_FLOOR)
energyPeak = Math.max(energyPeak * PEAK_DECAY, rawEnergy, PEAK_FLOOR)
var rms = clamp01(rawRms / rmsPeak)
var energy = clamp01(rawEnergy / energyPeak)
var centroidBin = magSum > 0 ? weightedBinSum / magSum : 0
var spectralCentroidHz = (centroidBin / bins) * nyquist
var spectralCentroid = clamp01(spectralCentroidHz / nyquist)
var geoLogSum = 0
for (var j = 0; j < magnitudeData.length; j += 1) {
geoLogSum += Math.log(Math.max(1e-12, magnitudeData[j]))
}
var geometricMean = Math.exp(geoLogSum / bins)
var arithmeticMean = magSum / bins
var spectralFlatness = arithmeticMean > 0 ? geometricMean / arithmeticMean : 0
var zc = 0
for (var t = 1; t < timeData.length; t += 1) {
var prev = timeData[t - 1]
var cur = timeData[t]
if ((prev >= 0 && cur < 0) || (prev < 0 && cur >= 0)) zc += 1
}
var zcr = timeData.length > 1 ? zc / (timeData.length - 1) : 0
var frequencyBands = extractFrequencyBands(ctx.sampleRate, magnitudeData)
// detectBeat uses RAW (un-normalized) energy so the peak detector compares apples to apples
// across recent history. Using normalized 0..1 energy would defeat the variance check.
var beatInfo = detectBeat(rawEnergy)
var patternInfo = detectPatterns(rawEnergy, beatInfo.isBeat)
// Treble = mean of (presence, brilliance) — matches host's useAudioFeatures convention.
// Previously used (highMid + presence + brilliance)/3 which spilled mid-range into treble.
var payload = {
bass: frequencyBands.bass,
treble: clamp01((frequencyBands.presence + frequencyBands.brilliance) / 2),
rms: rms,
energy: energy,
beat: beatInfo.isBeat,
drumIntensity: clamp01(frequencyBands.sub * 0.7 + frequencyBands.bass * 0.3),
sub: frequencyBands.sub,
beatPhase: beatInfo.beatPhase,
bpm: beatInfo.bpm,
beatConfidence: beatInfo.confidence,
isBeatDrop: patternInfo.isBeatDrop,
isBuildUp: patternInfo.isBuildUp,
isSilence: patternInfo.isSilence,
energyDelta: patternInfo.energyDelta,
spectralCentroid: spectralCentroid,
spectralFlatness: clamp01(spectralFlatness),
zcr: clamp01(zcr),
frequencyBands: frequencyBands,
fft: frequencyData,
waveform: timeData,
__seUpdatedAt: Date.now(),
}
window.__STEMENGINE_MUON_AUDIO__ = payload
window.__STEMENGINE_VIZ_AUDIO__ = payload
}
function stop() {
running = false
if (rafId !== null) {
cancelAnimationFrame(rafId)
rafId = null
}
}
function start() {
var engine = window.__StemEngineAudioEngine__
if (!engine || typeof engine.getAnalyser !== 'function' || typeof engine.getContext !== 'function') return
var analyser = engine.getAnalyser()
var ctx = engine.getContext()
if (!analyser || !ctx) return
stop()
running = true
var frequencyData = new Float32Array(analyser.frequencyBinCount)
var timeData = new Float32Array(analyser.fftSize)
function loop() {
if (!running) return
extractFrame(analyser, ctx, frequencyData, timeData)
rafId = requestAnimationFrame(loop)
}
rafId = requestAnimationFrame(loop)
}
function reset() {
energyHistory = []
beatTimes = []
lastBeatTime = 0
previousEnergy = 0
bpm = 0
rmsPeak = PEAK_FLOOR
energyPeak = PEAK_FLOOR
bandPeaks.sub = PEAK_FLOOR
bandPeaks.bass = PEAK_FLOOR
bandPeaks.lowMid = PEAK_FLOOR
bandPeaks.mid = PEAK_FLOOR
bandPeaks.highMid = PEAK_FLOOR
bandPeaks.presence = PEAK_FLOOR
bandPeaks.brilliance = PEAK_FLOOR
}
window.__StemEngineFeatureExtractor__ = {
start: start,
stop: stop,
reset: reset,
setBeatThreshold: function (threshold) {
var t = Number(threshold)
if (!isFinite(t)) return
beatThreshold = Math.max(0, Math.min(3, t))
},
}
})()