mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(jobs): unstick audio-edit jobs — waveform OOM, retry skip, screens handoff (#1049)
## Summary Job `2c1b922e` (Dave Matthews Band – Why I Am, 24-bit/96kHz FLAC, audio edit requested) stalled twice in prod. Three bugs: 1. **Waveform OOM** — `WaveformGenerator.generate_data_only` decoded the full source at native resolution through pydub. For a 6-min hi-res FLAC that peaked at ~1 GB RSS and OOM-killed the 1 GiB `audio-download-job` while pre-generating editor assets. It now decodes straight to 16 kHz mono PCM via ffmpeg and computes the envelope with numpy: **156 MB vs 1.07 GB peak** on the real file, amplitudes within 0.003 of the old output. The same function backs the editor's on-demand waveform in the API. 2. **Retry skipped unfinished audio-edit prep** — the Cloud Run auto-retry saw `downloading`, treated the download as done and exited 0, leaving the job at `downloading` forever. A retry that finds an audio-edit job still at `downloading` now resumes the editor prep. That path triggers nothing downstream, so it's safe. Non-edit jobs keep the existing skip (downstream may already be triggered). 3. **Screens handoff ignored `audio_edit_complete`** — `advance_to_screens_if_ready` (added in #1002) and the `recover-stuck-jobs` lost-screens sweep only accepted `downloading`. Audio-edited jobs prep at `audio_edit_complete`, so **every audio-edited job since Sept 15 stalled** after lyrics and separation finished (log: `screens not (re)triggered — status=audio_edit_complete`). Both now use a shared `PREP_PHASE_STATUSES`. 2c1b922e was unstuck manually (admin reset → `awaiting_audio_edit`, then a direct `/api/internal/workers/screens` dispatch). Added a TROUBLESHOOTING entry covering both signatures and the manual fixes. ## Testing - New tests: hi-res 96k/24-bit decode path (low-rate mono args, envelope matches pydub reference, silent → 0, undecodable → error); download-worker resume vs skip; handoff at `audio_edit_complete`; watchdog recovers `audio_edit_complete` and queries both statuses. - `pytest tests/unit/` → 1824 passed, 8 skipped - `pytest backend/tests/ --ignore=backend/tests/emulator` → 4586 passed, 36 skipped - Version 0.238.4 → 0.238.5 🤖 Generated with [Claude Code](https://claude.com/claude-code) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Audio-edit jobs can resume preparation after an interruption without redownloading existing audio. * Screen generation can proceed after audio editing is complete, and stalled screen-generation triggers can be recovered. * Waveform previews are more reliable for high-resolution and silent audio. Invalid audio and decoding timeouts now produce clear errors. * **Documentation** * Added troubleshooting guidance for jobs stuck during audio editing or before screen generation. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
daa46347f7
commit
bcdfd6b6bb
10 files changed
+379
-63
No files matched your search
@@ -27,7 +27,7 @@ from backend.workers.video_worker import generate_video
|
||||
from backend.workers.render_video_worker import process_render_video
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.auth_service import AuthResult, UserType
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.job_manager import JobManager, PREP_PHASE_STATUSES
|
||||
from backend.services.tracing import (
|
||||
extract_trace_context,
|
||||
start_span_with_context,
|
||||
@@ -1078,11 +1078,12 @@ async def recover_stuck_jobs(
|
||||
except Exception as e:
|
||||
logger.warning(f"[job:{job_id}] render re-trigger failed: {e}")
|
||||
|
||||
# --- DOWNLOADING with lyrics done but screens never triggered: lost screens trigger ---
|
||||
# --- Prep phase with lyrics done but screens never triggered: lost screens trigger ---
|
||||
# The lyrics worker triggers screen generation once transcription completes
|
||||
# (audio separation is decoupled). If that dispatch is lost — historically a
|
||||
# fire-and-forget task cancelled on Cloud Run Job loop teardown, now awaited,
|
||||
# but a Cloud Tasks enqueue can still fail — the job sits at `downloading` with
|
||||
# but a Cloud Tasks enqueue can still fail — the job sits in its prep status
|
||||
# (`downloading`, or `audio_edit_complete` for audio-edited jobs) with
|
||||
# lyrics_complete=True and never advances, with no error. Re-trigger is
|
||||
# idempotent (advance_to_screens_if_ready status-guards + the screens worker
|
||||
# no-ops a duplicate). Mirrors the REVIEW_COMPLETE lost-render recovery above.
|
||||
@@ -1092,7 +1093,7 @@ async def recover_stuck_jobs(
|
||||
# in. DOWNLOADING is a transient processing state (jobs move through in
|
||||
# minutes), so this ceiling is far above any realistic concurrent population.
|
||||
dl_query = jobs_ref.where(
|
||||
filter=FieldFilter("status", "==", JobStatus.DOWNLOADING.value)
|
||||
filter=FieldFilter("status", "in", sorted(s.value for s in PREP_PHASE_STATUSES))
|
||||
).limit(SCREENS_RECOVERY_SCAN_LIMIT).stream()
|
||||
for doc in dl_query:
|
||||
if len(screens_retriggered) >= SCREENS_RETRIGGERS_PER_TICK:
|
||||
@@ -1104,7 +1105,7 @@ async def recover_stuck_jobs(
|
||||
if not _prep_screens_stalled(job):
|
||||
continue
|
||||
logger.warning(
|
||||
f"[job:{job_id}] DOWNLOADING stalled >10 min with lyrics complete but "
|
||||
f"[job:{job_id}] {job.status} stalled >10 min with lyrics complete but "
|
||||
"no screens — re-triggering"
|
||||
)
|
||||
try:
|
||||
@@ -1167,7 +1168,7 @@ def _job_updated_age_seconds(job):
|
||||
|
||||
|
||||
def _prep_screens_stalled(job) -> bool:
|
||||
"""True when a DOWNLOADING job has lyrics complete but has sat >10 min without
|
||||
"""True when a prep-phase job has lyrics complete but has sat >10 min without
|
||||
advancing to screen generation — i.e. the screens trigger was lost.
|
||||
|
||||
While audio separation is still running it keeps ``updated_at`` fresh, so this
|
||||
|
||||
@@ -25,6 +25,17 @@ from backend.services.storage_service import StorageService
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Statuses a job sits in while its prep workers (audio separation + lyrics) run,
|
||||
# i.e. the states from which the lyrics→screens handoff may fire. Normal jobs
|
||||
# prep at DOWNLOADING; jobs that went through the audio editor prep at
|
||||
# AUDIO_EDIT_COMPLETE (the edit-submit endpoint triggers the workers without
|
||||
# passing back through DOWNLOADING).
|
||||
PREP_PHASE_STATUSES = frozenset({
|
||||
JobStatus.DOWNLOADING,
|
||||
JobStatus.AUDIO_EDIT_COMPLETE,
|
||||
})
|
||||
|
||||
|
||||
def _mask_email(email: str) -> str:
|
||||
"""Mask email for logging to protect PII. Shows first char + domain."""
|
||||
if not email or "@" not in email:
|
||||
@@ -1160,7 +1171,8 @@ class JobManager:
|
||||
|
||||
Screen generation is gated on LYRICS only (audio separation is decoupled
|
||||
so reviewers can start while it finishes), so this no-ops until lyrics are
|
||||
done, and only fires while the job is still at ``downloading``.
|
||||
done, and only fires while the job is still in its prep phase
|
||||
(``downloading``, or ``audio_edit_complete`` for audio-edited jobs).
|
||||
|
||||
Returns:
|
||||
True iff a screens dispatch was issued.
|
||||
@@ -1171,13 +1183,13 @@ class JobManager:
|
||||
if not job.state_data.get('lyrics_complete', False):
|
||||
# Lyrics not finished yet — whichever worker finishes lyrics triggers.
|
||||
return False
|
||||
if job.status != JobStatus.DOWNLOADING:
|
||||
if job.status not in PREP_PHASE_STATUSES:
|
||||
# Already advanced past prep (or parked in a special state). Keeps the
|
||||
# audio-worker fallback from re-triggering a job that already reached
|
||||
# screens/review.
|
||||
logger.info(
|
||||
f"Job {job_id}: screens not (re)triggered — status={job.status} "
|
||||
"(expected downloading)"
|
||||
"(expected downloading or audio_edit_complete)"
|
||||
)
|
||||
return False
|
||||
from backend.services.worker_service import get_worker_service
|
||||
|
||||
@@ -547,6 +547,100 @@ class TestAudioEditPreGeneration:
|
||||
assert any(c.kwargs.get('new_status') == JobStatus.AWAITING_AUDIO_EDIT for c in calls)
|
||||
|
||||
|
||||
class TestAudioEditPrepResume:
|
||||
"""A retry that finds an audio-edit job stuck at DOWNLOADING must finish the
|
||||
editor prep instead of skipping (job 2c1b922e: first run OOM-killed while
|
||||
generating the waveform, retry skipped, job orphaned at `downloading`)."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_resumes_unfinished_audio_edit_prep(self):
|
||||
job = _make_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data={'requires_audio_edit': True},
|
||||
input_media_gcs_path="uploads/test-job-123/audio/song.flac",
|
||||
)
|
||||
|
||||
with patch("backend.workers.audio_download_worker.JobManager") as mock_jm_cls, \
|
||||
patch("backend.workers.audio_download_worker.StorageService"), \
|
||||
patch("backend.workers.audio_download_worker._download_audio", new_callable=AsyncMock) as mock_dl, \
|
||||
patch("backend.workers.audio_download_worker.get_worker_service") as mock_ws_factory, \
|
||||
patch("backend.services.audio_transcoding_service.AudioTranscodingService") as MockTS, \
|
||||
patch("backend.services.audio_analysis_service.AudioAnalysisService") as MockAS:
|
||||
|
||||
mock_jm = MagicMock()
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.transition_to_state.return_value = True
|
||||
mock_jm_cls.return_value = mock_jm
|
||||
mock_ws = AsyncMock()
|
||||
mock_ws_factory.return_value = mock_ws
|
||||
MockAS.return_value.cache_waveform_data.return_value = ([0.1], 1.0)
|
||||
|
||||
result = await process_audio_download("test-job-123")
|
||||
|
||||
assert result is True
|
||||
mock_dl.assert_not_awaited()
|
||||
MockTS.return_value.transcode_if_needed.assert_called_once_with(
|
||||
"uploads/test-job-123/audio/song.flac"
|
||||
)
|
||||
mock_jm.transition_to_state.assert_called_once()
|
||||
assert mock_jm.transition_to_state.call_args.kwargs["new_status"] == JobStatus.AWAITING_AUDIO_EDIT
|
||||
mock_ws.trigger_audio_worker.assert_not_awaited()
|
||||
mock_ws.trigger_lyrics_worker.assert_not_awaited()
|
||||
mock_jm.fail_job.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_run_already_in_editor_is_success(self):
|
||||
"""If another run moved the job to AWAITING_AUDIO_EDIT first, the losing
|
||||
transition must not fail the job."""
|
||||
stuck = _make_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
state_data={'requires_audio_edit': True},
|
||||
input_media_gcs_path="uploads/test-job-123/audio/song.flac",
|
||||
)
|
||||
advanced = _make_job(status=JobStatus.AWAITING_AUDIO_EDIT)
|
||||
|
||||
with patch("backend.workers.audio_download_worker.JobManager") as mock_jm_cls, \
|
||||
patch("backend.workers.audio_download_worker.StorageService"), \
|
||||
patch("backend.services.audio_transcoding_service.AudioTranscodingService"), \
|
||||
patch("backend.services.audio_analysis_service.AudioAnalysisService"):
|
||||
|
||||
mock_jm = MagicMock()
|
||||
mock_jm.get_job.side_effect = [stuck, advanced]
|
||||
mock_jm.transition_to_state.return_value = False
|
||||
mock_jm_cls.return_value = mock_jm
|
||||
|
||||
result = await process_audio_download("test-job-123")
|
||||
|
||||
assert result is True
|
||||
assert mock_jm.transition_to_state.call_args.kwargs["raise_on_invalid"] is False
|
||||
mock_jm.fail_job.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_without_audio_edit_still_skips(self):
|
||||
"""Non-edit jobs at DOWNLOADING may already have downstream workers
|
||||
triggered — keep skipping to avoid double-triggering."""
|
||||
job = _make_job(
|
||||
status=JobStatus.DOWNLOADING,
|
||||
input_media_gcs_path="uploads/test-job-123/audio/song.flac",
|
||||
)
|
||||
|
||||
with patch("backend.workers.audio_download_worker.JobManager") as mock_jm_cls, \
|
||||
patch("backend.workers.audio_download_worker.StorageService"), \
|
||||
patch("backend.workers.audio_download_worker._download_audio", new_callable=AsyncMock) as mock_dl, \
|
||||
patch("backend.workers.audio_download_worker._enter_audio_edit") as mock_enter:
|
||||
|
||||
mock_jm = MagicMock()
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm_cls.return_value = mock_jm
|
||||
|
||||
result = await process_audio_download("test-job-123")
|
||||
|
||||
assert result is True
|
||||
mock_dl.assert_not_awaited()
|
||||
mock_enter.assert_not_called()
|
||||
mock_jm.transition_to_state.assert_not_called()
|
||||
|
||||
|
||||
class TestFfprobeSeconds:
|
||||
"""Tests for the _ffprobe_seconds helper in duration_reconciliation."""
|
||||
|
||||
|
||||
@@ -120,6 +120,22 @@ class TestPipelineDecoupling:
|
||||
assert result is True
|
||||
mock_ws.trigger_screens_worker.assert_awaited_once_with("test-001")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_advance_triggers_screens_for_audio_edited_job(self, job_manager, mock_firestore_service):
|
||||
"""Audio-edited jobs prep at AUDIO_EDIT_COMPLETE (the edit-submit endpoint
|
||||
triggers the workers without passing back through DOWNLOADING). The
|
||||
handoff must still fire — job 2c1b922e stalled here."""
|
||||
job = _make_job(state_data={"lyrics_complete": True}, status=JobStatus.AUDIO_EDIT_COMPLETE)
|
||||
mock_firestore_service.get_job.return_value = job
|
||||
mock_ws = MagicMock()
|
||||
mock_ws.trigger_screens_worker = AsyncMock(return_value=True)
|
||||
|
||||
with patch("backend.services.worker_service.get_worker_service", return_value=mock_ws):
|
||||
result = await job_manager.advance_to_screens_if_ready("test-001")
|
||||
|
||||
assert result is True
|
||||
mock_ws.trigger_screens_worker.assert_awaited_once_with("test-001")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_advance_noop_when_lyrics_not_done(self, job_manager, mock_firestore_service):
|
||||
"""No screens trigger until lyrics complete, even if audio is done."""
|
||||
|
||||
@@ -196,6 +196,37 @@ def test_stuck_downloading_with_lyrics_done_retriggers_screens(client):
|
||||
mock_jm.advance_to_screens_if_ready.assert_awaited_once_with("s1")
|
||||
|
||||
|
||||
def test_stuck_audio_edit_complete_with_lyrics_done_retriggers_screens(client):
|
||||
"""Audio-edited jobs prep at AUDIO_EDIT_COMPLETE, not DOWNLOADING. A lost
|
||||
screens trigger there must be recovered too (job 2c1b922e sat stuck)."""
|
||||
job = _job("ae1", JobStatus.AUDIO_EDIT_COMPLETE, minutes_stale=15)
|
||||
job.state_data["lyrics_complete"] = True
|
||||
mock_jm = _mock_jm(downloading_prep=[_doc("ae1", JobStatus.AUDIO_EDIT_COMPLETE)], jobs={"ae1": job})
|
||||
mock_jm.advance_to_screens_if_ready = AsyncMock(return_value=True)
|
||||
resp, _ = _run(client, mock_jm)
|
||||
assert resp.json()["screens_retriggered_jobs"] == ["ae1"]
|
||||
mock_jm.advance_to_screens_if_ready.assert_awaited_once_with("ae1")
|
||||
|
||||
|
||||
def test_lost_screens_sweep_queries_all_prep_statuses(client):
|
||||
"""The lost-screens sweep must query both prep statuses, not just DOWNLOADING."""
|
||||
mock_jm = _mock_jm()
|
||||
# Record FieldFilter args directly — other test modules may stub the
|
||||
# firestore package, so the real FieldFilter's attributes aren't reliable.
|
||||
with patch("google.cloud.firestore_v1.FieldFilter",
|
||||
side_effect=lambda field, op, value: (field, op, value)):
|
||||
_run(client, mock_jm)
|
||||
filters = [
|
||||
c.kwargs["filter"]
|
||||
for c in mock_jm.firestore.db.collection.return_value.where.call_args_list
|
||||
]
|
||||
in_filters = [f for f in filters if f[1] == "in"]
|
||||
assert len(in_filters) == 1
|
||||
assert set(in_filters[0][2]) == {
|
||||
JobStatus.DOWNLOADING.value, JobStatus.AUDIO_EDIT_COMPLETE.value,
|
||||
}
|
||||
|
||||
|
||||
def test_fresh_downloading_not_retriggered(client):
|
||||
"""A DOWNLOADING job updated 2 min ago (audio still separating) is NOT re-triggered."""
|
||||
job = _job("s2", JobStatus.DOWNLOADING, minutes_stale=2)
|
||||
|
||||
@@ -181,6 +181,22 @@ async def process_audio_download(job_id: str) -> bool:
|
||||
# Cloud Run Job auto-retries of a successful task — re-downloading
|
||||
# would double-trigger downstream workers and break state transitions.
|
||||
if job.status in DOWNLOAD_ALREADY_DONE_STATUSES:
|
||||
# Exception: an audio-edit job still at DOWNLOADING means the previous
|
||||
# run died between the download and entering AWAITING_AUDIO_EDIT
|
||||
# (e.g. OOM-killed while pre-generating editor assets — job 2c1b922e).
|
||||
# Nothing downstream is triggered on that path, so finishing it here
|
||||
# is safe — skipping would orphan the job at `downloading` forever.
|
||||
if (
|
||||
job.status == JobStatus.DOWNLOADING
|
||||
and job.state_data.get('requires_audio_edit')
|
||||
and job.input_media_gcs_path
|
||||
):
|
||||
logger.info(
|
||||
f"[job:{job_id}] Download complete but audio-edit prep unfinished; "
|
||||
f"resuming"
|
||||
)
|
||||
_enter_audio_edit(job_manager, job_id, job.input_media_gcs_path)
|
||||
return True
|
||||
logger.info(
|
||||
f"[job:{job_id}] Download already complete (status={job.status}), "
|
||||
f"skipping re-run"
|
||||
@@ -296,35 +312,7 @@ async def process_audio_download(job_id: str) -> bool:
|
||||
# Check if audio editing was requested — enter blocking state instead of processing
|
||||
job = job_manager.get_job(job_id)
|
||||
if job and job.state_data.get('requires_audio_edit'):
|
||||
# Preserve the original input path before any edits
|
||||
job_manager.update_state_data(job_id, 'original_input_media_gcs_path', audio_gcs_path)
|
||||
|
||||
# Pre-generate audio editor assets (OGG transcode + waveform data)
|
||||
# so the editor UI loads instantly. Non-fatal if this fails.
|
||||
waveform_cache_path = f"jobs/{job_id}/audio_edit/waveform_original.json"
|
||||
try:
|
||||
from backend.services.audio_transcoding_service import AudioTranscodingService
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
transcoding = AudioTranscodingService()
|
||||
transcoding.transcode_if_needed(audio_gcs_path)
|
||||
logger.info(f"[job:{job_id}] Pre-transcoded input audio to OGG for editor")
|
||||
|
||||
analysis = AudioAnalysisService()
|
||||
analysis.cache_waveform_data(audio_gcs_path, job_id, waveform_cache_path)
|
||||
logger.info(f"[job:{job_id}] Pre-generated waveform data for editor")
|
||||
|
||||
job_manager.update_state_data(job_id, 'audio_edit_waveform_cache_path', waveform_cache_path)
|
||||
except Exception as e:
|
||||
logger.warning(f"[job:{job_id}] Failed to pre-generate editor assets (non-fatal): {e}")
|
||||
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.AWAITING_AUDIO_EDIT,
|
||||
progress=15,
|
||||
message="Audio downloaded. Please review and edit the audio before processing."
|
||||
)
|
||||
logger.info(f"[job:{job_id}] Audio download complete, awaiting audio edit")
|
||||
_enter_audio_edit(job_manager, job_id, audio_gcs_path)
|
||||
return True
|
||||
|
||||
# Reconcile credit charge against actual audio duration before processing.
|
||||
@@ -356,6 +344,55 @@ async def process_audio_download(job_id: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _enter_audio_edit(job_manager: JobManager, job_id: str, audio_gcs_path: str) -> None:
|
||||
"""Prepare editor assets and park the job at AWAITING_AUDIO_EDIT."""
|
||||
# Preserve the original input path before any edits
|
||||
job_manager.update_state_data(job_id, 'original_input_media_gcs_path', audio_gcs_path)
|
||||
|
||||
# Pre-generate audio editor assets (OGG transcode + waveform data)
|
||||
# so the editor UI loads instantly. Non-fatal if this fails.
|
||||
waveform_cache_path = f"jobs/{job_id}/audio_edit/waveform_original.json"
|
||||
try:
|
||||
from backend.services.audio_transcoding_service import AudioTranscodingService
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
transcoding = AudioTranscodingService()
|
||||
transcoding.transcode_if_needed(audio_gcs_path)
|
||||
logger.info(f"[job:{job_id}] Pre-transcoded input audio to OGG for editor")
|
||||
|
||||
analysis = AudioAnalysisService()
|
||||
analysis.cache_waveform_data(audio_gcs_path, job_id, waveform_cache_path)
|
||||
logger.info(f"[job:{job_id}] Pre-generated waveform data for editor")
|
||||
|
||||
job_manager.update_state_data(job_id, 'audio_edit_waveform_cache_path', waveform_cache_path)
|
||||
except Exception as e:
|
||||
logger.warning(f"[job:{job_id}] Failed to pre-generate editor assets (non-fatal): {e}")
|
||||
|
||||
# raise_on_invalid=False: a concurrent run (e.g. manual retry racing a Cloud
|
||||
# Run auto-retry) may already have moved the job on — that's success, and
|
||||
# raising would reach the caller's fail_job and clobber the advanced job.
|
||||
transitioned = job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.AWAITING_AUDIO_EDIT,
|
||||
progress=15,
|
||||
message="Audio downloaded. Please review and edit the audio before processing.",
|
||||
raise_on_invalid=False,
|
||||
)
|
||||
if not transitioned:
|
||||
latest = job_manager.get_job(job_id)
|
||||
if latest and latest.status in DOWNLOAD_ALREADY_DONE_STATUSES and latest.status != JobStatus.DOWNLOADING:
|
||||
logger.info(
|
||||
f"[job:{job_id}] Concurrent run already advanced the job "
|
||||
f"(status={latest.status}); treating as idempotent success"
|
||||
)
|
||||
return
|
||||
raise DownloadError(
|
||||
f"Could not transition to AWAITING_AUDIO_EDIT "
|
||||
f"(status={latest.status if latest else 'missing'})"
|
||||
)
|
||||
logger.info(f"[job:{job_id}] Audio download complete, awaiting audio edit")
|
||||
|
||||
|
||||
async def _download_audio(
|
||||
job_id: str,
|
||||
source_name: str,
|
||||
|
||||
@@ -152,6 +152,19 @@ curl -X POST "https://api.nomadkaraoke.com/api/jobs/YOUR_JOB_ID/retry" \
|
||||
|
||||
---
|
||||
|
||||
## Job stuck at `downloading` or `audio_edit_complete` (no error)
|
||||
|
||||
Two known signatures (both fixed in v0.238.5, job 2c1b922e):
|
||||
|
||||
- **Audio-edit job stuck at `downloading`, `requires_audio_edit=true`, never reached `awaiting_audio_edit`.** The download worker was OOM-killed while pre-generating the editor waveform (a 24-bit/96kHz FLAC decoded at native resolution peaked at ~1 GB in the 1 GiB `audio-download-job`), and the Cloud Run auto-retry saw `downloading` and skipped. Since v0.238.5 the waveform decodes at 16 kHz mono and a retry resumes the unfinished editor prep. Manual fix: `POST /api/admin/jobs/{id}/reset` with `{"target_state": "awaiting_audio_edit"}`.
|
||||
- **Audio-edited job stuck at `audio_edit_complete` with `lyrics_complete=true`.** Logs show `screens not (re)triggered — status=audio_edit_complete (expected downloading)`. The lyrics→screens handoff only accepted `downloading`; since v0.238.5 it (and the `recover-stuck-jobs` lost-screens sweep) also accepts `audio_edit_complete`. Manual fix — dispatch the screens worker directly (don't use `regenerate-screens`: it restores the original status afterwards, so the job never reaches review):
|
||||
|
||||
```bash
|
||||
curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/screens" \
|
||||
-H "Authorization: Bearer $KARAOKE_ADMIN_TOKEN" -H "Content-Type: application/json" \
|
||||
-d '{"job_id": "<job_id>"}'
|
||||
```
|
||||
|
||||
## Job stuck at `encoding` status
|
||||
|
||||
**Cause:** A Cloud Run deployment killed the poller mid-encoding. The GCE worker finished but nobody received the result. The health service will flag this as `encoding_stuck` after 50 minutes.
|
||||
|
||||
@@ -7,6 +7,7 @@ images suitable for display in the instrumental review UI.
|
||||
|
||||
import logging
|
||||
import math
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
@@ -22,6 +23,13 @@ from .models import AudibleSegment, MuteRegion
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Sample rate used to decode audio for the RMS envelope in generate_data_only.
|
||||
_ENVELOPE_SAMPLE_RATE = 16000
|
||||
_PCM16_MAX_AMPLITUDE = 32768.0
|
||||
# Decoding a full track at 16 kHz mono takes seconds; bound pathological inputs.
|
||||
_DECODE_TIMEOUT_SECONDS = 300
|
||||
|
||||
|
||||
class WaveformGenerator:
|
||||
"""
|
||||
Generates waveform visualization images from audio files.
|
||||
@@ -182,32 +190,60 @@ class WaveformGenerator:
|
||||
path = Path(audio_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
|
||||
audio = AudioSegment.from_file(audio_path)
|
||||
duration_seconds = len(audio) / 1000.0
|
||||
|
||||
if audio.channels > 1:
|
||||
audio = audio.set_channels(1)
|
||||
|
||||
|
||||
# Decode straight to low-rate mono 16-bit PCM rather than loading the
|
||||
# source at native resolution: a 6-minute 24-bit/96kHz stereo FLAC
|
||||
# peaked at ~1 GB RSS through pydub and OOM-killed the 1 GiB
|
||||
# audio-download Cloud Run Job (job 2c1b922e). A 1000-point RMS
|
||||
# envelope doesn't need more than a few kHz of bandwidth.
|
||||
samples = self._decode_mono_pcm16(audio_path, _ENVELOPE_SAMPLE_RATE)
|
||||
samples_per_ms = _ENVELOPE_SAMPLE_RATE // 1000
|
||||
duration_ms = int(round(len(samples) * 1000 / _ENVELOPE_SAMPLE_RATE))
|
||||
duration_seconds = duration_ms / 1000.0
|
||||
|
||||
# Calculate window size to get desired number of points
|
||||
duration_ms = len(audio)
|
||||
window_ms = max(1, duration_ms // num_points)
|
||||
|
||||
amplitudes = []
|
||||
for start_ms in range(0, duration_ms, window_ms):
|
||||
end_ms = min(start_ms + window_ms, duration_ms)
|
||||
window = audio[start_ms:end_ms]
|
||||
|
||||
if window.rms > 0:
|
||||
db = 20 * math.log10(window.rms / window.max_possible_amplitude)
|
||||
else:
|
||||
db = -100.0
|
||||
|
||||
# Normalize to 0-1 range (mapping -60dB to 0dB -> 0 to 1)
|
||||
normalized = max(0.0, min(1.0, (db + 60) / 60))
|
||||
amplitudes.append(normalized)
|
||||
|
||||
return amplitudes, duration_seconds
|
||||
|
||||
starts = np.arange(0, duration_ms, window_ms, dtype=np.int64) * samples_per_ms
|
||||
starts = starts[starts < len(samples)]
|
||||
if not len(starts):
|
||||
return [], duration_seconds
|
||||
|
||||
squared = samples.astype(np.float64) ** 2
|
||||
sums = np.add.reduceat(squared, starts)
|
||||
widths = np.diff(np.append(starts, len(samples)))
|
||||
rms = np.sqrt(sums / widths)
|
||||
|
||||
with np.errstate(divide="ignore"):
|
||||
db = np.where(rms > 0, 20 * np.log10(rms / _PCM16_MAX_AMPLITUDE), -100.0)
|
||||
|
||||
# Normalize to 0-1 range (mapping -60dB to 0dB -> 0 to 1)
|
||||
amplitudes = np.clip((db + 60) / 60, 0.0, 1.0)
|
||||
return [float(a) for a in amplitudes], duration_seconds
|
||||
|
||||
@staticmethod
|
||||
def _decode_mono_pcm16(audio_path: str, sample_rate: int) -> np.ndarray:
|
||||
"""Decode any ffmpeg-readable file to mono int16 samples at ``sample_rate``."""
|
||||
cmd = [
|
||||
AudioSegment.converter, "-nostdin", "-v", "error",
|
||||
"-i", audio_path, "-vn",
|
||||
"-ac", "1", "-ar", str(sample_rate),
|
||||
"-f", "s16le", "-acodec", "pcm_s16le", "-",
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, check=False, timeout=_DECODE_TIMEOUT_SECONDS
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise RuntimeError(
|
||||
f"ffmpeg failed to decode {audio_path}: timed out after {e.timeout}s"
|
||||
) from e
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"ffmpeg failed to decode {audio_path}: "
|
||||
f"{result.stderr.decode(errors='replace').strip()}"
|
||||
)
|
||||
return np.frombuffer(result.stdout, dtype=np.int16)
|
||||
|
||||
def generate_peaks(
|
||||
self,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.238.4"
|
||||
version = "0.238.5"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -12,6 +12,7 @@ Tests cover:
|
||||
import os
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from pydub import AudioSegment
|
||||
|
||||
from karaoke_gen.instrumental_review import (
|
||||
WaveformGenerator,
|
||||
@@ -413,3 +414,78 @@ class TestCustomDimensions:
|
||||
)
|
||||
|
||||
assert os.path.exists(output_path)
|
||||
|
||||
|
||||
class TestDataOnlyHiResDecode:
|
||||
"""generate_data_only must not decode the source at native resolution.
|
||||
|
||||
A 6-minute 24-bit/96kHz stereo FLAC peaked at ~1 GB through pydub and
|
||||
OOM-killed the 1 GiB audio-download Cloud Run Job (job 2c1b922e).
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def hires_audio_path(self, temp_dir):
|
||||
import subprocess
|
||||
|
||||
path = os.path.join(temp_dir, "hires.flac")
|
||||
subprocess.run(
|
||||
[AudioSegment.converter, "-nostdin", "-v", "error", "-y",
|
||||
"-f", "lavfi", "-i", "sine=frequency=440:sample_rate=96000:duration=5",
|
||||
"-af", "volume=-10dB", "-ac", "2",
|
||||
"-sample_fmt", "s32", "-bits_per_raw_sample", "24", path],
|
||||
check=True,
|
||||
)
|
||||
return path
|
||||
|
||||
def test_decodes_at_low_rate_mono(self, hires_audio_path):
|
||||
import subprocess
|
||||
from unittest.mock import patch
|
||||
|
||||
real_run = subprocess.run
|
||||
with patch("karaoke_gen.instrumental_review.waveform.subprocess.run",
|
||||
side_effect=real_run) as mock_run:
|
||||
WaveformGenerator().generate_data_only(hires_audio_path, num_points=100)
|
||||
|
||||
cmd = mock_run.call_args.args[0]
|
||||
assert cmd[cmd.index("-ac") + 1] == "1"
|
||||
assert cmd[cmd.index("-ar") + 1] == "16000"
|
||||
|
||||
def test_matches_native_resolution_envelope(self, hires_audio_path):
|
||||
"""Low-rate envelope ≈ the pydub native-resolution RMS envelope."""
|
||||
import math
|
||||
|
||||
amplitudes, duration = WaveformGenerator().generate_data_only(
|
||||
hires_audio_path, num_points=100
|
||||
)
|
||||
|
||||
audio = AudioSegment.from_file(hires_audio_path).set_channels(1)
|
||||
window_ms = len(audio) // 100
|
||||
expected = []
|
||||
for start in range(0, len(audio), window_ms):
|
||||
w = audio[start:start + window_ms]
|
||||
db = 20 * math.log10(w.rms / w.max_possible_amplitude) if w.rms else -100.0
|
||||
expected.append(max(0.0, min(1.0, (db + 60) / 60)))
|
||||
|
||||
assert abs(duration - 5.0) < 0.01
|
||||
assert len(amplitudes) == len(expected)
|
||||
assert max(abs(a - e) for a, e in zip(amplitudes, expected)) < 0.01
|
||||
|
||||
def test_silent_audio_is_zero(self, silent_audio_path):
|
||||
amplitudes, _ = WaveformGenerator().generate_data_only(silent_audio_path, num_points=50)
|
||||
assert amplitudes and all(a == 0.0 for a in amplitudes)
|
||||
|
||||
def test_undecodable_file_raises(self, temp_dir):
|
||||
path = os.path.join(temp_dir, "garbage.flac")
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"not audio at all")
|
||||
with pytest.raises(RuntimeError, match="ffmpeg failed"):
|
||||
WaveformGenerator().generate_data_only(path)
|
||||
|
||||
def test_decode_timeout_raises_runtime_error(self, loud_audio_path):
|
||||
import subprocess
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch("karaoke_gen.instrumental_review.waveform.subprocess.run",
|
||||
side_effect=subprocess.TimeoutExpired(cmd="ffmpeg", timeout=300)):
|
||||
with pytest.raises(RuntimeError, match="timed out after 300"):
|
||||
WaveformGenerator().generate_data_only(loud_audio_path)
|
||||
Reference in new issue
Block a user