diff --git a/backend/api/routes/internal.py b/backend/api/routes/internal.py index ac171d60..af8c9487 100644 --- a/backend/api/routes/internal.py +++ b/backend/api/routes/internal.py @@ -27,7 +27,7 @@ from backend.workers.video_worker import generate_video from backend.workers.render_video_worker import process_render_video from backend.api.dependencies import require_admin from backend.services.auth_service import AuthResult, UserType -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, PREP_PHASE_STATUSES from backend.services.tracing import ( extract_trace_context, start_span_with_context, @@ -1078,11 +1078,12 @@ async def recover_stuck_jobs( except Exception as e: logger.warning(f"[job:{job_id}] render re-trigger failed: {e}") - # --- DOWNLOADING with lyrics done but screens never triggered: lost screens trigger --- + # --- Prep phase with lyrics done but screens never triggered: lost screens trigger --- # The lyrics worker triggers screen generation once transcription completes # (audio separation is decoupled). If that dispatch is lost — historically a # fire-and-forget task cancelled on Cloud Run Job loop teardown, now awaited, - # but a Cloud Tasks enqueue can still fail — the job sits at `downloading` with + # but a Cloud Tasks enqueue can still fail — the job sits in its prep status + # (`downloading`, or `audio_edit_complete` for audio-edited jobs) with # lyrics_complete=True and never advances, with no error. Re-trigger is # idempotent (advance_to_screens_if_ready status-guards + the screens worker # no-ops a duplicate). Mirrors the REVIEW_COMPLETE lost-render recovery above. @@ -1092,7 +1093,7 @@ async def recover_stuck_jobs( # in. DOWNLOADING is a transient processing state (jobs move through in # minutes), so this ceiling is far above any realistic concurrent population. dl_query = jobs_ref.where( - filter=FieldFilter("status", "==", JobStatus.DOWNLOADING.value) + filter=FieldFilter("status", "in", sorted(s.value for s in PREP_PHASE_STATUSES)) ).limit(SCREENS_RECOVERY_SCAN_LIMIT).stream() for doc in dl_query: if len(screens_retriggered) >= SCREENS_RETRIGGERS_PER_TICK: @@ -1104,7 +1105,7 @@ async def recover_stuck_jobs( if not _prep_screens_stalled(job): continue logger.warning( - f"[job:{job_id}] DOWNLOADING stalled >10 min with lyrics complete but " + f"[job:{job_id}] {job.status} stalled >10 min with lyrics complete but " "no screens — re-triggering" ) try: @@ -1167,7 +1168,7 @@ def _job_updated_age_seconds(job): def _prep_screens_stalled(job) -> bool: - """True when a DOWNLOADING job has lyrics complete but has sat >10 min without + """True when a prep-phase job has lyrics complete but has sat >10 min without advancing to screen generation — i.e. the screens trigger was lost. While audio separation is still running it keeps ``updated_at`` fresh, so this diff --git a/backend/services/job_manager.py b/backend/services/job_manager.py index e0715c57..d5b269d4 100644 --- a/backend/services/job_manager.py +++ b/backend/services/job_manager.py @@ -25,6 +25,17 @@ from backend.services.storage_service import StorageService logger = logging.getLogger(__name__) +# Statuses a job sits in while its prep workers (audio separation + lyrics) run, +# i.e. the states from which the lyrics→screens handoff may fire. Normal jobs +# prep at DOWNLOADING; jobs that went through the audio editor prep at +# AUDIO_EDIT_COMPLETE (the edit-submit endpoint triggers the workers without +# passing back through DOWNLOADING). +PREP_PHASE_STATUSES = frozenset({ + JobStatus.DOWNLOADING, + JobStatus.AUDIO_EDIT_COMPLETE, +}) + + def _mask_email(email: str) -> str: """Mask email for logging to protect PII. Shows first char + domain.""" if not email or "@" not in email: @@ -1160,7 +1171,8 @@ class JobManager: Screen generation is gated on LYRICS only (audio separation is decoupled so reviewers can start while it finishes), so this no-ops until lyrics are - done, and only fires while the job is still at ``downloading``. + done, and only fires while the job is still in its prep phase + (``downloading``, or ``audio_edit_complete`` for audio-edited jobs). Returns: True iff a screens dispatch was issued. @@ -1171,13 +1183,13 @@ class JobManager: if not job.state_data.get('lyrics_complete', False): # Lyrics not finished yet — whichever worker finishes lyrics triggers. return False - if job.status != JobStatus.DOWNLOADING: + if job.status not in PREP_PHASE_STATUSES: # Already advanced past prep (or parked in a special state). Keeps the # audio-worker fallback from re-triggering a job that already reached # screens/review. logger.info( f"Job {job_id}: screens not (re)triggered — status={job.status} " - "(expected downloading)" + "(expected downloading or audio_edit_complete)" ) return False from backend.services.worker_service import get_worker_service diff --git a/backend/tests/test_audio_download_worker.py b/backend/tests/test_audio_download_worker.py index 2531ecd3..fccd1643 100644 --- a/backend/tests/test_audio_download_worker.py +++ b/backend/tests/test_audio_download_worker.py @@ -547,6 +547,100 @@ class TestAudioEditPreGeneration: assert any(c.kwargs.get('new_status') == JobStatus.AWAITING_AUDIO_EDIT for c in calls) +class TestAudioEditPrepResume: + """A retry that finds an audio-edit job stuck at DOWNLOADING must finish the + editor prep instead of skipping (job 2c1b922e: first run OOM-killed while + generating the waveform, retry skipped, job orphaned at `downloading`).""" + + @pytest.mark.asyncio + async def test_retry_resumes_unfinished_audio_edit_prep(self): + job = _make_job( + status=JobStatus.DOWNLOADING, + state_data={'requires_audio_edit': True}, + input_media_gcs_path="uploads/test-job-123/audio/song.flac", + ) + + with patch("backend.workers.audio_download_worker.JobManager") as mock_jm_cls, \ + patch("backend.workers.audio_download_worker.StorageService"), \ + patch("backend.workers.audio_download_worker._download_audio", new_callable=AsyncMock) as mock_dl, \ + patch("backend.workers.audio_download_worker.get_worker_service") as mock_ws_factory, \ + patch("backend.services.audio_transcoding_service.AudioTranscodingService") as MockTS, \ + patch("backend.services.audio_analysis_service.AudioAnalysisService") as MockAS: + + mock_jm = MagicMock() + mock_jm.get_job.return_value = job + mock_jm.transition_to_state.return_value = True + mock_jm_cls.return_value = mock_jm + mock_ws = AsyncMock() + mock_ws_factory.return_value = mock_ws + MockAS.return_value.cache_waveform_data.return_value = ([0.1], 1.0) + + result = await process_audio_download("test-job-123") + + assert result is True + mock_dl.assert_not_awaited() + MockTS.return_value.transcode_if_needed.assert_called_once_with( + "uploads/test-job-123/audio/song.flac" + ) + mock_jm.transition_to_state.assert_called_once() + assert mock_jm.transition_to_state.call_args.kwargs["new_status"] == JobStatus.AWAITING_AUDIO_EDIT + mock_ws.trigger_audio_worker.assert_not_awaited() + mock_ws.trigger_lyrics_worker.assert_not_awaited() + mock_jm.fail_job.assert_not_called() + + @pytest.mark.asyncio + async def test_concurrent_run_already_in_editor_is_success(self): + """If another run moved the job to AWAITING_AUDIO_EDIT first, the losing + transition must not fail the job.""" + stuck = _make_job( + status=JobStatus.DOWNLOADING, + state_data={'requires_audio_edit': True}, + input_media_gcs_path="uploads/test-job-123/audio/song.flac", + ) + advanced = _make_job(status=JobStatus.AWAITING_AUDIO_EDIT) + + with patch("backend.workers.audio_download_worker.JobManager") as mock_jm_cls, \ + patch("backend.workers.audio_download_worker.StorageService"), \ + patch("backend.services.audio_transcoding_service.AudioTranscodingService"), \ + patch("backend.services.audio_analysis_service.AudioAnalysisService"): + + mock_jm = MagicMock() + mock_jm.get_job.side_effect = [stuck, advanced] + mock_jm.transition_to_state.return_value = False + mock_jm_cls.return_value = mock_jm + + result = await process_audio_download("test-job-123") + + assert result is True + assert mock_jm.transition_to_state.call_args.kwargs["raise_on_invalid"] is False + mock_jm.fail_job.assert_not_called() + + @pytest.mark.asyncio + async def test_retry_without_audio_edit_still_skips(self): + """Non-edit jobs at DOWNLOADING may already have downstream workers + triggered — keep skipping to avoid double-triggering.""" + job = _make_job( + status=JobStatus.DOWNLOADING, + input_media_gcs_path="uploads/test-job-123/audio/song.flac", + ) + + with patch("backend.workers.audio_download_worker.JobManager") as mock_jm_cls, \ + patch("backend.workers.audio_download_worker.StorageService"), \ + patch("backend.workers.audio_download_worker._download_audio", new_callable=AsyncMock) as mock_dl, \ + patch("backend.workers.audio_download_worker._enter_audio_edit") as mock_enter: + + mock_jm = MagicMock() + mock_jm.get_job.return_value = job + mock_jm_cls.return_value = mock_jm + + result = await process_audio_download("test-job-123") + + assert result is True + mock_dl.assert_not_awaited() + mock_enter.assert_not_called() + mock_jm.transition_to_state.assert_not_called() + + class TestFfprobeSeconds: """Tests for the _ffprobe_seconds helper in duration_reconciliation.""" diff --git a/backend/tests/test_modal_gcp_migration.py b/backend/tests/test_modal_gcp_migration.py index 969b90b9..a115b89f 100644 --- a/backend/tests/test_modal_gcp_migration.py +++ b/backend/tests/test_modal_gcp_migration.py @@ -120,6 +120,22 @@ class TestPipelineDecoupling: assert result is True mock_ws.trigger_screens_worker.assert_awaited_once_with("test-001") + @pytest.mark.asyncio + async def test_advance_triggers_screens_for_audio_edited_job(self, job_manager, mock_firestore_service): + """Audio-edited jobs prep at AUDIO_EDIT_COMPLETE (the edit-submit endpoint + triggers the workers without passing back through DOWNLOADING). The + handoff must still fire — job 2c1b922e stalled here.""" + job = _make_job(state_data={"lyrics_complete": True}, status=JobStatus.AUDIO_EDIT_COMPLETE) + mock_firestore_service.get_job.return_value = job + mock_ws = MagicMock() + mock_ws.trigger_screens_worker = AsyncMock(return_value=True) + + with patch("backend.services.worker_service.get_worker_service", return_value=mock_ws): + result = await job_manager.advance_to_screens_if_ready("test-001") + + assert result is True + mock_ws.trigger_screens_worker.assert_awaited_once_with("test-001") + @pytest.mark.asyncio async def test_advance_noop_when_lyrics_not_done(self, job_manager, mock_firestore_service): """No screens trigger until lyrics complete, even if audio is done.""" diff --git a/backend/tests/test_recover_stuck_jobs_repark.py b/backend/tests/test_recover_stuck_jobs_repark.py index 58a9d4a1..8634e919 100644 --- a/backend/tests/test_recover_stuck_jobs_repark.py +++ b/backend/tests/test_recover_stuck_jobs_repark.py @@ -196,6 +196,37 @@ def test_stuck_downloading_with_lyrics_done_retriggers_screens(client): mock_jm.advance_to_screens_if_ready.assert_awaited_once_with("s1") +def test_stuck_audio_edit_complete_with_lyrics_done_retriggers_screens(client): + """Audio-edited jobs prep at AUDIO_EDIT_COMPLETE, not DOWNLOADING. A lost + screens trigger there must be recovered too (job 2c1b922e sat stuck).""" + job = _job("ae1", JobStatus.AUDIO_EDIT_COMPLETE, minutes_stale=15) + job.state_data["lyrics_complete"] = True + mock_jm = _mock_jm(downloading_prep=[_doc("ae1", JobStatus.AUDIO_EDIT_COMPLETE)], jobs={"ae1": job}) + mock_jm.advance_to_screens_if_ready = AsyncMock(return_value=True) + resp, _ = _run(client, mock_jm) + assert resp.json()["screens_retriggered_jobs"] == ["ae1"] + mock_jm.advance_to_screens_if_ready.assert_awaited_once_with("ae1") + + +def test_lost_screens_sweep_queries_all_prep_statuses(client): + """The lost-screens sweep must query both prep statuses, not just DOWNLOADING.""" + mock_jm = _mock_jm() + # Record FieldFilter args directly — other test modules may stub the + # firestore package, so the real FieldFilter's attributes aren't reliable. + with patch("google.cloud.firestore_v1.FieldFilter", + side_effect=lambda field, op, value: (field, op, value)): + _run(client, mock_jm) + filters = [ + c.kwargs["filter"] + for c in mock_jm.firestore.db.collection.return_value.where.call_args_list + ] + in_filters = [f for f in filters if f[1] == "in"] + assert len(in_filters) == 1 + assert set(in_filters[0][2]) == { + JobStatus.DOWNLOADING.value, JobStatus.AUDIO_EDIT_COMPLETE.value, + } + + def test_fresh_downloading_not_retriggered(client): """A DOWNLOADING job updated 2 min ago (audio still separating) is NOT re-triggered.""" job = _job("s2", JobStatus.DOWNLOADING, minutes_stale=2) diff --git a/backend/workers/audio_download_worker.py b/backend/workers/audio_download_worker.py index 2252ddcc..53370b1b 100644 --- a/backend/workers/audio_download_worker.py +++ b/backend/workers/audio_download_worker.py @@ -181,6 +181,22 @@ async def process_audio_download(job_id: str) -> bool: # Cloud Run Job auto-retries of a successful task — re-downloading # would double-trigger downstream workers and break state transitions. if job.status in DOWNLOAD_ALREADY_DONE_STATUSES: + # Exception: an audio-edit job still at DOWNLOADING means the previous + # run died between the download and entering AWAITING_AUDIO_EDIT + # (e.g. OOM-killed while pre-generating editor assets — job 2c1b922e). + # Nothing downstream is triggered on that path, so finishing it here + # is safe — skipping would orphan the job at `downloading` forever. + if ( + job.status == JobStatus.DOWNLOADING + and job.state_data.get('requires_audio_edit') + and job.input_media_gcs_path + ): + logger.info( + f"[job:{job_id}] Download complete but audio-edit prep unfinished; " + f"resuming" + ) + _enter_audio_edit(job_manager, job_id, job.input_media_gcs_path) + return True logger.info( f"[job:{job_id}] Download already complete (status={job.status}), " f"skipping re-run" @@ -296,35 +312,7 @@ async def process_audio_download(job_id: str) -> bool: # Check if audio editing was requested — enter blocking state instead of processing job = job_manager.get_job(job_id) if job and job.state_data.get('requires_audio_edit'): - # Preserve the original input path before any edits - job_manager.update_state_data(job_id, 'original_input_media_gcs_path', audio_gcs_path) - - # Pre-generate audio editor assets (OGG transcode + waveform data) - # so the editor UI loads instantly. Non-fatal if this fails. - waveform_cache_path = f"jobs/{job_id}/audio_edit/waveform_original.json" - try: - from backend.services.audio_transcoding_service import AudioTranscodingService - from backend.services.audio_analysis_service import AudioAnalysisService - - transcoding = AudioTranscodingService() - transcoding.transcode_if_needed(audio_gcs_path) - logger.info(f"[job:{job_id}] Pre-transcoded input audio to OGG for editor") - - analysis = AudioAnalysisService() - analysis.cache_waveform_data(audio_gcs_path, job_id, waveform_cache_path) - logger.info(f"[job:{job_id}] Pre-generated waveform data for editor") - - job_manager.update_state_data(job_id, 'audio_edit_waveform_cache_path', waveform_cache_path) - except Exception as e: - logger.warning(f"[job:{job_id}] Failed to pre-generate editor assets (non-fatal): {e}") - - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.AWAITING_AUDIO_EDIT, - progress=15, - message="Audio downloaded. Please review and edit the audio before processing." - ) - logger.info(f"[job:{job_id}] Audio download complete, awaiting audio edit") + _enter_audio_edit(job_manager, job_id, audio_gcs_path) return True # Reconcile credit charge against actual audio duration before processing. @@ -356,6 +344,55 @@ async def process_audio_download(job_id: str) -> bool: return False +def _enter_audio_edit(job_manager: JobManager, job_id: str, audio_gcs_path: str) -> None: + """Prepare editor assets and park the job at AWAITING_AUDIO_EDIT.""" + # Preserve the original input path before any edits + job_manager.update_state_data(job_id, 'original_input_media_gcs_path', audio_gcs_path) + + # Pre-generate audio editor assets (OGG transcode + waveform data) + # so the editor UI loads instantly. Non-fatal if this fails. + waveform_cache_path = f"jobs/{job_id}/audio_edit/waveform_original.json" + try: + from backend.services.audio_transcoding_service import AudioTranscodingService + from backend.services.audio_analysis_service import AudioAnalysisService + + transcoding = AudioTranscodingService() + transcoding.transcode_if_needed(audio_gcs_path) + logger.info(f"[job:{job_id}] Pre-transcoded input audio to OGG for editor") + + analysis = AudioAnalysisService() + analysis.cache_waveform_data(audio_gcs_path, job_id, waveform_cache_path) + logger.info(f"[job:{job_id}] Pre-generated waveform data for editor") + + job_manager.update_state_data(job_id, 'audio_edit_waveform_cache_path', waveform_cache_path) + except Exception as e: + logger.warning(f"[job:{job_id}] Failed to pre-generate editor assets (non-fatal): {e}") + + # raise_on_invalid=False: a concurrent run (e.g. manual retry racing a Cloud + # Run auto-retry) may already have moved the job on — that's success, and + # raising would reach the caller's fail_job and clobber the advanced job. + transitioned = job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.AWAITING_AUDIO_EDIT, + progress=15, + message="Audio downloaded. Please review and edit the audio before processing.", + raise_on_invalid=False, + ) + if not transitioned: + latest = job_manager.get_job(job_id) + if latest and latest.status in DOWNLOAD_ALREADY_DONE_STATUSES and latest.status != JobStatus.DOWNLOADING: + logger.info( + f"[job:{job_id}] Concurrent run already advanced the job " + f"(status={latest.status}); treating as idempotent success" + ) + return + raise DownloadError( + f"Could not transition to AWAITING_AUDIO_EDIT " + f"(status={latest.status if latest else 'missing'})" + ) + logger.info(f"[job:{job_id}] Audio download complete, awaiting audio edit") + + async def _download_audio( job_id: str, source_name: str, diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index a01b7dfd..e0a95baa 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -152,6 +152,19 @@ curl -X POST "https://api.nomadkaraoke.com/api/jobs/YOUR_JOB_ID/retry" \ --- +## Job stuck at `downloading` or `audio_edit_complete` (no error) + +Two known signatures (both fixed in v0.238.5, job 2c1b922e): + +- **Audio-edit job stuck at `downloading`, `requires_audio_edit=true`, never reached `awaiting_audio_edit`.** The download worker was OOM-killed while pre-generating the editor waveform (a 24-bit/96kHz FLAC decoded at native resolution peaked at ~1 GB in the 1 GiB `audio-download-job`), and the Cloud Run auto-retry saw `downloading` and skipped. Since v0.238.5 the waveform decodes at 16 kHz mono and a retry resumes the unfinished editor prep. Manual fix: `POST /api/admin/jobs/{id}/reset` with `{"target_state": "awaiting_audio_edit"}`. +- **Audio-edited job stuck at `audio_edit_complete` with `lyrics_complete=true`.** Logs show `screens not (re)triggered — status=audio_edit_complete (expected downloading)`. The lyrics→screens handoff only accepted `downloading`; since v0.238.5 it (and the `recover-stuck-jobs` lost-screens sweep) also accepts `audio_edit_complete`. Manual fix — dispatch the screens worker directly (don't use `regenerate-screens`: it restores the original status afterwards, so the job never reaches review): + +```bash +curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/screens" \ + -H "Authorization: Bearer $KARAOKE_ADMIN_TOKEN" -H "Content-Type: application/json" \ + -d '{"job_id": ""}' +``` + ## Job stuck at `encoding` status **Cause:** A Cloud Run deployment killed the poller mid-encoding. The GCE worker finished but nobody received the result. The health service will flag this as `encoding_stuck` after 50 minutes. diff --git a/karaoke_gen/instrumental_review/waveform.py b/karaoke_gen/instrumental_review/waveform.py index c1fba6ab..4ae41a5c 100644 --- a/karaoke_gen/instrumental_review/waveform.py +++ b/karaoke_gen/instrumental_review/waveform.py @@ -7,6 +7,7 @@ images suitable for display in the instrumental review UI. import logging import math +import subprocess from pathlib import Path from typing import List, Optional, Tuple @@ -22,6 +23,13 @@ from .models import AudibleSegment, MuteRegion logger = logging.getLogger(__name__) +# Sample rate used to decode audio for the RMS envelope in generate_data_only. +_ENVELOPE_SAMPLE_RATE = 16000 +_PCM16_MAX_AMPLITUDE = 32768.0 +# Decoding a full track at 16 kHz mono takes seconds; bound pathological inputs. +_DECODE_TIMEOUT_SECONDS = 300 + + class WaveformGenerator: """ Generates waveform visualization images from audio files. @@ -182,32 +190,60 @@ class WaveformGenerator: path = Path(audio_path) if not path.exists(): raise FileNotFoundError(f"Audio file not found: {audio_path}") - - audio = AudioSegment.from_file(audio_path) - duration_seconds = len(audio) / 1000.0 - - if audio.channels > 1: - audio = audio.set_channels(1) - + + # Decode straight to low-rate mono 16-bit PCM rather than loading the + # source at native resolution: a 6-minute 24-bit/96kHz stereo FLAC + # peaked at ~1 GB RSS through pydub and OOM-killed the 1 GiB + # audio-download Cloud Run Job (job 2c1b922e). A 1000-point RMS + # envelope doesn't need more than a few kHz of bandwidth. + samples = self._decode_mono_pcm16(audio_path, _ENVELOPE_SAMPLE_RATE) + samples_per_ms = _ENVELOPE_SAMPLE_RATE // 1000 + duration_ms = int(round(len(samples) * 1000 / _ENVELOPE_SAMPLE_RATE)) + duration_seconds = duration_ms / 1000.0 + # Calculate window size to get desired number of points - duration_ms = len(audio) window_ms = max(1, duration_ms // num_points) - - amplitudes = [] - for start_ms in range(0, duration_ms, window_ms): - end_ms = min(start_ms + window_ms, duration_ms) - window = audio[start_ms:end_ms] - - if window.rms > 0: - db = 20 * math.log10(window.rms / window.max_possible_amplitude) - else: - db = -100.0 - - # Normalize to 0-1 range (mapping -60dB to 0dB -> 0 to 1) - normalized = max(0.0, min(1.0, (db + 60) / 60)) - amplitudes.append(normalized) - - return amplitudes, duration_seconds + + starts = np.arange(0, duration_ms, window_ms, dtype=np.int64) * samples_per_ms + starts = starts[starts < len(samples)] + if not len(starts): + return [], duration_seconds + + squared = samples.astype(np.float64) ** 2 + sums = np.add.reduceat(squared, starts) + widths = np.diff(np.append(starts, len(samples))) + rms = np.sqrt(sums / widths) + + with np.errstate(divide="ignore"): + db = np.where(rms > 0, 20 * np.log10(rms / _PCM16_MAX_AMPLITUDE), -100.0) + + # Normalize to 0-1 range (mapping -60dB to 0dB -> 0 to 1) + amplitudes = np.clip((db + 60) / 60, 0.0, 1.0) + return [float(a) for a in amplitudes], duration_seconds + + @staticmethod + def _decode_mono_pcm16(audio_path: str, sample_rate: int) -> np.ndarray: + """Decode any ffmpeg-readable file to mono int16 samples at ``sample_rate``.""" + cmd = [ + AudioSegment.converter, "-nostdin", "-v", "error", + "-i", audio_path, "-vn", + "-ac", "1", "-ar", str(sample_rate), + "-f", "s16le", "-acodec", "pcm_s16le", "-", + ] + try: + result = subprocess.run( + cmd, capture_output=True, check=False, timeout=_DECODE_TIMEOUT_SECONDS + ) + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"ffmpeg failed to decode {audio_path}: timed out after {e.timeout}s" + ) from e + if result.returncode != 0: + raise RuntimeError( + f"ffmpeg failed to decode {audio_path}: " + f"{result.stderr.decode(errors='replace').strip()}" + ) + return np.frombuffer(result.stdout, dtype=np.int16) def generate_peaks( self, diff --git a/pyproject.toml b/pyproject.toml index 98731341..d6971f10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.238.4" +version = "0.238.5" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/test_instrumental_review/test_waveform.py b/tests/unit/test_instrumental_review/test_waveform.py index 612fb3a0..b6603389 100644 --- a/tests/unit/test_instrumental_review/test_waveform.py +++ b/tests/unit/test_instrumental_review/test_waveform.py @@ -12,6 +12,7 @@ Tests cover: import os import pytest from pathlib import Path +from pydub import AudioSegment from karaoke_gen.instrumental_review import ( WaveformGenerator, @@ -413,3 +414,78 @@ class TestCustomDimensions: ) assert os.path.exists(output_path) + + +class TestDataOnlyHiResDecode: + """generate_data_only must not decode the source at native resolution. + + A 6-minute 24-bit/96kHz stereo FLAC peaked at ~1 GB through pydub and + OOM-killed the 1 GiB audio-download Cloud Run Job (job 2c1b922e). + """ + + @pytest.fixture + def hires_audio_path(self, temp_dir): + import subprocess + + path = os.path.join(temp_dir, "hires.flac") + subprocess.run( + [AudioSegment.converter, "-nostdin", "-v", "error", "-y", + "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=96000:duration=5", + "-af", "volume=-10dB", "-ac", "2", + "-sample_fmt", "s32", "-bits_per_raw_sample", "24", path], + check=True, + ) + return path + + def test_decodes_at_low_rate_mono(self, hires_audio_path): + import subprocess + from unittest.mock import patch + + real_run = subprocess.run + with patch("karaoke_gen.instrumental_review.waveform.subprocess.run", + side_effect=real_run) as mock_run: + WaveformGenerator().generate_data_only(hires_audio_path, num_points=100) + + cmd = mock_run.call_args.args[0] + assert cmd[cmd.index("-ac") + 1] == "1" + assert cmd[cmd.index("-ar") + 1] == "16000" + + def test_matches_native_resolution_envelope(self, hires_audio_path): + """Low-rate envelope ≈ the pydub native-resolution RMS envelope.""" + import math + + amplitudes, duration = WaveformGenerator().generate_data_only( + hires_audio_path, num_points=100 + ) + + audio = AudioSegment.from_file(hires_audio_path).set_channels(1) + window_ms = len(audio) // 100 + expected = [] + for start in range(0, len(audio), window_ms): + w = audio[start:start + window_ms] + db = 20 * math.log10(w.rms / w.max_possible_amplitude) if w.rms else -100.0 + expected.append(max(0.0, min(1.0, (db + 60) / 60))) + + assert abs(duration - 5.0) < 0.01 + assert len(amplitudes) == len(expected) + assert max(abs(a - e) for a, e in zip(amplitudes, expected)) < 0.01 + + def test_silent_audio_is_zero(self, silent_audio_path): + amplitudes, _ = WaveformGenerator().generate_data_only(silent_audio_path, num_points=50) + assert amplitudes and all(a == 0.0 for a in amplitudes) + + def test_undecodable_file_raises(self, temp_dir): + path = os.path.join(temp_dir, "garbage.flac") + with open(path, "wb") as f: + f.write(b"not audio at all") + with pytest.raises(RuntimeError, match="ffmpeg failed"): + WaveformGenerator().generate_data_only(path) + + def test_decode_timeout_raises_runtime_error(self, loud_audio_path): + import subprocess + from unittest.mock import patch + + with patch("karaoke_gen.instrumental_review.waveform.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="ffmpeg", timeout=300)): + with pytest.raises(RuntimeError, match="timed out after 300"): + WaveformGenerator().generate_data_only(loud_audio_path)