mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Final piece of the review-page speed work (#1024 → #1028 → #1029): the Waveforms-mode strips no longer require downloading + WebAudio-decoding the entire vocals stem (~7.5MB, ~4–6s) — the backend now serves a **precomputed 400-peaks/s envelope as ~150KB of uint8/base64 JSON**, cached in GCS and pre-warmed at prep time, so strips paint sub-second. ## How - **`WaveformGenerator.generate_peaks()`** — numpy max-abs envelope, bucket-for-bucket identical to the frontend's `computePeaks` (same drift-free fractional-stride bucketing; exact match vs a naive reference in tests). - **`AudioAnalysisService.get_vocals_peaks()`** — GCS cache at `review-audio/vocals_peaks_400.json`, validated against the stem path; miss decodes the small transcoded OGG; upload best-effort. - **`GET /api/review/{job_id}/vocals-peaks`** — off the event loop under the shared waveform semaphore; `202 Retry-After: 15` while separation runs (mirrors `/audio/vocals`); `Cache-Control: private, max-age=600`; `peaks_per_second` clamped 50–1000. - **`screens_worker`** pre-computes the envelope next to the existing OGG + waveform pre-warm, so it's a pure cache read by the time a job reaches review. - **Frontend** — `VocalsAudioDataLoader` prefers the peaks endpoint (one quick retry on transient errors) and **falls back to the original full download+decode** on any terminal peaks failure, so a peaks-side problem can never cost a user their waveforms. Local mode (no peaks URL) uses the old path unchanged. Vocal stem selection factored into `backend/utils/stems.py` (shared by routes + worker). ## Testing - Package: 4 tests incl. exact naive-reference parity (**106 passed**) - Backend: 4 service + 4 endpoint tests (**4518 passed**) - Frontend: `fetchVocalsPeaks` units + 5 loader-behavior tests (prefers peaks / falls back / one-retry-then-fallback / 202 polling / audio-only mode) (**1307 passed**) - `review-fast-load.spec.ts` prod E2E extended with a cached vocals-peaks latency/shape check - Post-deploy: will verify the endpoint on a real in-review job and confirm payload size + latency ## Notes - Existing in-review jobs have no cache yet — their first Waveforms open computes + caches lazily (bounded); every open after that is sub-second. New jobs are pre-warmed. - 8-bit amplitude quantization is visually lossless at strip heights (14px). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
149 lines
5.6 KiB
TypeScript
149 lines
5.6 KiB
TypeScript
export interface AudioData {
|
|
duration: number
|
|
// Downsampled absolute-amplitude envelope: one peak per bucket. Storing the
|
|
// envelope instead of the raw PCM keeps a whole vocal track in ~hundreds of KB
|
|
// (peaksPerSecond * duration * 4 bytes) rather than the tens of MB a full
|
|
// Float32Array of decoded samples would occupy, which noticeably slows the app.
|
|
peaks: Float32Array
|
|
peaksPerSecond: number
|
|
}
|
|
|
|
// Resolution of the precomputed peak envelope. High enough to stay crisp when a
|
|
// short segment is zoomed across the full timeline width, low enough to keep the
|
|
// whole track tiny in memory. 400/s over a 4-minute song ≈ 96k floats (~384 KB).
|
|
const PEAKS_PER_SECOND = 400
|
|
|
|
// The vocals endpoint returns 202 while audio separation is still running (the
|
|
// user can open lyrics review before the vocal stem has been uploaded). Callers
|
|
// should catch this and retry after a delay rather than giving up.
|
|
export class AudioNotReadyError extends Error {
|
|
constructor() {
|
|
super('Vocals audio not ready yet (separation in progress)')
|
|
this.name = 'AudioNotReadyError'
|
|
}
|
|
}
|
|
|
|
// Non-2xx response from the vocals endpoint. Carries the status so callers can
|
|
// tell transient overload (5xx/429 — worth retrying) from terminal answers
|
|
// (404: this job has no vocal stem; retrying would never succeed).
|
|
export class AudioFetchError extends Error {
|
|
constructor(public readonly status: number) {
|
|
super(`Failed to fetch vocals audio: ${status}`)
|
|
this.name = 'AudioFetchError'
|
|
}
|
|
}
|
|
|
|
export function isTransientAudioError(error: unknown): boolean {
|
|
if (error instanceof AudioFetchError) {
|
|
return error.status >= 500 || error.status === 429
|
|
}
|
|
// fetch() rejects with a TypeError on network-level failures (offline,
|
|
// connection reset, CORS blip) — all worth retrying.
|
|
return error instanceof TypeError
|
|
}
|
|
|
|
// Minimal structural subset of AudioBuffer, so the peak math is testable
|
|
// without constructing a real (browser-only) AudioContext.
|
|
export interface DecodedAudioLike {
|
|
duration: number
|
|
length: number
|
|
numberOfChannels: number
|
|
getChannelData(channel: number): Float32Array
|
|
}
|
|
|
|
export function computePeaks(decoded: DecodedAudioLike): AudioData {
|
|
const { duration, numberOfChannels, length } = decoded
|
|
const bucketCount = Math.max(1, Math.ceil(duration * PEAKS_PER_SECOND))
|
|
// Fractional on purpose: at a 44.1 kHz AudioContext this is 110.25. Flooring
|
|
// it to an integer stride made the envelope's clock run ~0.23% fast, so the
|
|
// waveform drifted visibly late as the song progressed (~0.4s by the 3-minute
|
|
// mark). With floored per-bucket boundaries the remaining error is bounded by
|
|
// the ceil() rounding of bucketCount: under one bucket (2.5 ms) anywhere in
|
|
// the track, non-accumulating and sub-pixel at any timeline zoom.
|
|
const samplesPerBucket = length / bucketCount
|
|
const peaks = new Float32Array(bucketCount)
|
|
|
|
// Collapse all channels into a single envelope by taking the maximum
|
|
// absolute sample value in each bucket. Iterate per bucket (not per sample)
|
|
// so we avoid a division + Math.floor on every one of the ~millions of
|
|
// samples, which would otherwise freeze the main thread while loading.
|
|
// The raw decoded buffer is released when this function returns; only the
|
|
// compact `peaks` array is retained.
|
|
for (let channelIdx = 0; channelIdx < numberOfChannels; channelIdx++) {
|
|
const channelData = decoded.getChannelData(channelIdx)
|
|
for (let bucket = 0; bucket < bucketCount; bucket++) {
|
|
const start = Math.floor(bucket * samplesPerBucket)
|
|
if (start >= length) break
|
|
const end = bucket === bucketCount - 1 ? length : Math.min(length, Math.floor((bucket + 1) * samplesPerBucket))
|
|
let peak = peaks[bucket]
|
|
for (let i = start; i < end; i++) {
|
|
const amplitude = Math.abs(channelData[i])
|
|
if (amplitude > peak) {
|
|
peak = amplitude
|
|
}
|
|
}
|
|
peaks[bucket] = peak
|
|
}
|
|
}
|
|
|
|
return {
|
|
duration,
|
|
peaks,
|
|
peaksPerSecond: PEAKS_PER_SECOND
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetch the server-side pre-computed vocals peak envelope (~150 KB JSON) and
|
|
* adapt it to the same AudioData shape computePeaks produces — so waveform
|
|
* strips paint sub-second instead of after a multi-MB download + decode.
|
|
* Throws AudioNotReadyError on 202 (separation still running) and
|
|
* AudioFetchError on other non-2xx, mirroring fetchAudioData.
|
|
*/
|
|
export async function fetchVocalsPeaks(url: string): Promise<AudioData> {
|
|
const response = await fetch(url)
|
|
if (response.status === 202) {
|
|
throw new AudioNotReadyError()
|
|
}
|
|
if (!response.ok) {
|
|
throw new AudioFetchError(response.status)
|
|
}
|
|
const data = await response.json()
|
|
if (data?.encoding !== 'u8' || typeof data.peaks_b64 !== 'string' || !data.peaks_b64) {
|
|
throw new AudioFetchError(500) // unexpected payload — treat as transient server trouble
|
|
}
|
|
const raw = atob(data.peaks_b64)
|
|
const peaks = new Float32Array(raw.length)
|
|
for (let i = 0; i < raw.length; i++) {
|
|
peaks[i] = raw.charCodeAt(i) / 255
|
|
}
|
|
return {
|
|
duration: Number(data.duration_seconds) || 0,
|
|
peaks,
|
|
peaksPerSecond: Number(data.peaks_per_second) || PEAKS_PER_SECOND,
|
|
}
|
|
}
|
|
|
|
export async function fetchAudioData(url: string): Promise<AudioData> {
|
|
// fetch() resolves even for 4xx/5xx; the vocals endpoint 404s when a job has
|
|
// no vocal stem. Fail explicitly so the caller sees "no vocals" rather than an
|
|
// opaque decodeAudioData error on the JSON/HTML error body.
|
|
const response = await fetch(url)
|
|
if (response.status === 202) {
|
|
throw new AudioNotReadyError()
|
|
}
|
|
if (!response.ok) {
|
|
throw new AudioFetchError(response.status)
|
|
}
|
|
const arrayBuffer = await response.arrayBuffer()
|
|
|
|
const context = new AudioContext()
|
|
|
|
try {
|
|
const decoded = await context.decodeAudioData(arrayBuffer)
|
|
return computePeaks(decoded)
|
|
} finally {
|
|
context.close()
|
|
}
|
|
}
|