Fix three production job failure modes with robust error handling (#565)

@coderabbitai ignore

## Summary

Fixes three distinct production job failures identified from
investigating jobs 503a4ad3, 7be8d1e5, and 1868f323:

- **0-segment lyrics guard**: Block review submission when transcription
produces no lyrics (e.g. karaoke input with no vocals). Backend
validation + frontend warning with guidance. CDG orchestrator gracefully
skips when LRC has no content.
- **Audio processor stage 2 validation**: Validate backing vocals
separation produced all expected stems (mirrors existing stage 1
validation). Defensive null checks prevent confusing NoneType crashes.
Full tracebacks in error logs.
- **AudioShake retry with exponential backoff**: 5 attempts via
tenacity, ~40 min spread (60s base, 3x multiplier), retries on
5xx/429/connection errors. Applied to all HTTP call points. Fixed file
handle exhaustion bug on upload retry.

Also includes:
- Troubleshooting runbooks for all 3 failure modes
- Lessons learned entry on pipeline robustness patterns
- Seed planning doc for Modal-to-GCP audio separation migration
- Version bump to 0.147.0

## Test plan

- [x] 2696 backend tests passing (54 new/modified)
- [x] 712 frontend tests passing (5 new for ReviewChangesModal)
- [x] HTTP endpoint test verifies 400 response for 0-segment review
submission
- [x] AudioShake retry tests verify 502 retry, 400 no-retry, connection
error retry
- [x] Audio processor tests verify stage 2 validation, null safety,
combined instrumentals
- [x] CDG orchestrator tests verify graceful skip on empty LRC, fail on
valid LRC errors
- [x] LRC timestamp regex tests for millisecond, centisecond, and
no-decimal formats

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.6 authored and GitHub committed 2026-03-21 17:58:31 -04:00
1 parent de02570514
commit 25aa2ef8ef
16 files changed
+970 -60

No files matched your search

+20
View File
@@ -1284,6 +1284,26 @@ async def complete_review(
detail=f"Job not in review state (current status: {job.status})"
)
# Check for 0-segment lyrics — fail early with user-friendly message
# Only check when lyrics pipeline has run (lyrics_metadata exists in state_data)
state_data = job.state_data or {}
lyrics_metadata = state_data.get('lyrics_metadata')
if lyrics_metadata is not None:
segment_count = lyrics_metadata.get('segment_count', 0)
corrected_lyrics = state_data.get('corrected_lyrics', {})
corrected_segments = corrected_lyrics.get('corrected_segments', [])
if segment_count == 0 and len(corrected_segments) == 0:
raise HTTPException(
status_code=400,
detail=(
"Cannot complete review: no lyrics were detected in the audio. "
"This can happen when the input audio has no vocals (e.g. a karaoke track or instrumental). "
"You can go back and paste lyrics manually using 'Replace All', "
"or cancel this job and try again with an audio file that contains vocals."
)
)
try:
# Store instrumental selection if provided (combined review flow)
instrumental_selection = body.instrumental_selection if body else None
@@ -0,0 +1,134 @@
"""
Tests for audio processor robustness fixes.
Covers:
- Stage 2 validation catches missing stems
- _normalize_audio_files handles missing clean instrumental path
- _generate_combined_instrumentals handles missing backing vocals path
"""
import pytest
from unittest.mock import MagicMock, patch
from karaoke_gen.audio_processor import AudioProcessor
def _make_processor():
"""Create an AudioProcessor with mocked dependencies."""
processor = AudioProcessor.__new__(AudioProcessor)
processor.logger = MagicMock()
processor.clean_instrumental_model = "model_bs_roformer"
processor.backing_vocals_models = ["mel_band_roformer_karaoke"]
processor.other_stems_models = []
processor.lossless_output_format = "FLAC"
processor.ffmpeg_base_command = "ffmpeg -y"
return processor
class TestNormalizeAudioFilesNullSafety:
"""Tests for _normalize_audio_files defensive checks."""
def test_raises_on_missing_clean_instrumental(self):
processor = _make_processor()
separation_result = {
"clean_instrumental": {}, # missing "instrumental" key
"combined_instrumentals": {},
}
with pytest.raises(ValueError, match="clean instrumental path is missing"):
processor._normalize_audio_files(separation_result, "Artist - Title", "/tmp/output")
def test_raises_on_none_clean_instrumental(self):
processor = _make_processor()
separation_result = {
"clean_instrumental": {"instrumental": None},
"combined_instrumentals": {},
}
with pytest.raises(ValueError, match="clean instrumental path is missing"):
processor._normalize_audio_files(separation_result, "Artist - Title", "/tmp/output")
def test_works_with_valid_path(self, tmp_path):
processor = _make_processor()
# Create a real file so _file_exists returns True
audio_file = tmp_path / "instrumental.flac"
audio_file.write_bytes(b"fake audio data")
separation_result = {
"clean_instrumental": {"instrumental": str(audio_file), "vocals": str(tmp_path / "vocals.flac")},
"combined_instrumentals": {},
"other_stems": {},
"backing_vocals": {},
}
# Mock _normalize_audio to avoid actually running ffmpeg
with patch.object(processor, "_normalize_audio"):
processor._normalize_audio_files(separation_result, "Artist - Title", str(tmp_path))
# Should have logged success (no exception)
assert processor.logger.info.called
def test_normalizes_combined_instrumentals(self, tmp_path):
processor = _make_processor()
instrumental = tmp_path / "instrumental.flac"
instrumental.write_bytes(b"fake instrumental")
combined = tmp_path / "combined.flac"
combined.write_bytes(b"fake combined")
separation_result = {
"clean_instrumental": {"instrumental": str(instrumental), "vocals": str(tmp_path / "v.flac")},
"combined_instrumentals": {"mel_band_roformer_karaoke": str(combined)},
}
with patch.object(processor, "_normalize_audio") as mock_normalize:
processor._normalize_audio_files(separation_result, "Artist - Title", str(tmp_path))
# Both clean instrumental and combined instrumental should be normalized
assert mock_normalize.call_count == 2
normalized_paths = [call.args[0] for call in mock_normalize.call_args_list]
assert str(instrumental) in normalized_paths
assert str(combined) in normalized_paths
class TestGenerateCombinedInstrumentalsNullSafety:
"""Tests for _generate_combined_instrumentals defensive checks."""
def test_skips_model_with_missing_backing_vocals(self, tmp_path):
processor = _make_processor()
instrumental_path = str(tmp_path / "instrumental.flac")
# Model has lead_vocals but not backing_vocals
backing_vocals_result = {
"mel_band_roformer_karaoke": {"lead_vocals": "/tmp/lead.flac"}
# "backing_vocals" key is missing
}
result = processor._generate_combined_instrumentals(
instrumental_path, backing_vocals_result, "Artist - Title", str(tmp_path)
)
assert result == {}
processor.logger.error.assert_called_once()
assert "backing_vocals path is missing" in processor.logger.error.call_args[0][0]
class TestStage2Validation:
"""Tests for stage 2 remote results validation in _process_audio_separation_remote."""
def test_organize_stage2_missing_backing_vocals_stem(self):
"""Test that _organize_stage2_remote_results correctly handles partial results."""
processor = _make_processor()
# Simulate only getting the Vocals file, not the Instrumental file
# This is what happened in the production failure
import tempfile, os
with tempfile.TemporaryDirectory() as tmpdir:
stems_dir = os.path.join(tmpdir, "stems")
os.makedirs(stems_dir)
# Create only the vocals file (missing the instrumental/backing vocals file)
vocals_file = os.path.join(tmpdir, "audio_(Vocals)_mel_band_roformer_karaoke.flac")
with open(vocals_file, "w") as f:
f.write("fake")
result = processor._organize_stage2_remote_results(
[vocals_file], "Artist - Title", stems_dir
)
# Should have the model with only lead_vocals, no backing_vocals
assert "mel_band_roformer_karaoke" in result
assert "lead_vocals" in result["mel_band_roformer_karaoke"]
assert "backing_vocals" not in result["mel_band_roformer_karaoke"]
+192
View File
@@ -0,0 +1,192 @@
"""
Tests for AudioShake API retry logic.
Covers:
- Retry on 5xx server errors
- Retry on 429 rate limit
- Retry on connection errors
- No retry on 4xx client errors (except 429)
- Retry exhaustion raises the original error
"""
import pytest
from unittest.mock import MagicMock, patch, call
import requests
from tenacity import wait_none
from karaoke_gen.lyrics_transcriber.transcribers.audioshake import (
AudioShakeAPI,
AudioShakeConfig,
_is_retryable_error,
)
@pytest.fixture(autouse=True)
def _no_retry_wait():
"""Disable retry wait times in all tests to avoid sleeping."""
# Temporarily replace the wait strategy with no-wait for both retry-decorated methods
methods = [AudioShakeAPI._request_with_retry, AudioShakeAPI._upload_file_with_retry]
original_waits = [m.retry.wait for m in methods]
for m in methods:
m.retry.wait = wait_none()
yield
for m, w in zip(methods, original_waits):
m.retry.wait = w
class TestIsRetryableError:
"""Tests for the _is_retryable_error predicate."""
def test_connection_error_is_retryable(self):
assert _is_retryable_error(requests.exceptions.ConnectionError()) is True
def test_timeout_is_retryable(self):
assert _is_retryable_error(requests.exceptions.Timeout()) is True
def test_502_is_retryable(self):
response = MagicMock()
response.status_code = 502
error = requests.exceptions.HTTPError(response=response)
assert _is_retryable_error(error) is True
def test_503_is_retryable(self):
response = MagicMock()
response.status_code = 503
error = requests.exceptions.HTTPError(response=response)
assert _is_retryable_error(error) is True
def test_429_is_retryable(self):
response = MagicMock()
response.status_code = 429
error = requests.exceptions.HTTPError(response=response)
assert _is_retryable_error(error) is True
def test_400_is_not_retryable(self):
response = MagicMock()
response.status_code = 400
error = requests.exceptions.HTTPError(response=response)
assert _is_retryable_error(error) is False
def test_401_is_not_retryable(self):
response = MagicMock()
response.status_code = 401
error = requests.exceptions.HTTPError(response=response)
assert _is_retryable_error(error) is False
def test_404_is_not_retryable(self):
response = MagicMock()
response.status_code = 404
error = requests.exceptions.HTTPError(response=response)
assert _is_retryable_error(error) is False
def test_value_error_is_not_retryable(self):
assert _is_retryable_error(ValueError("bad")) is False
class TestAudioShakeAPIRetry:
"""Tests for AudioShakeAPI._request_with_retry."""
def _make_api(self):
config = AudioShakeConfig(api_token="test-token")
logger = MagicMock()
return AudioShakeAPI(config, logger)
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
def test_succeeds_on_first_try(self, mock_request):
api = self._make_api()
mock_response = MagicMock()
mock_response.raise_for_status = MagicMock()
mock_request.return_value = mock_response
result = api._request_with_retry("GET", "https://api.example.com/test")
assert result == mock_response
mock_request.assert_called_once()
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
def test_retries_on_502_then_succeeds(self, mock_request):
api = self._make_api()
# First call: 502 error
bad_response = MagicMock()
bad_response.status_code = 502
bad_response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=bad_response)
# Second call: success
good_response = MagicMock()
good_response.raise_for_status = MagicMock()
mock_request.side_effect = [bad_response, good_response]
result = api._request_with_retry("GET", "https://api.example.com/test")
assert result == good_response
assert mock_request.call_count == 2
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
def test_does_not_retry_on_400(self, mock_request):
api = self._make_api()
bad_response = MagicMock()
bad_response.status_code = 400
bad_response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=bad_response)
mock_request.return_value = bad_response
with pytest.raises(requests.exceptions.HTTPError):
api._request_with_retry("GET", "https://api.example.com/test")
# Should NOT retry — only 1 call
mock_request.assert_called_once()
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
def test_retries_on_connection_error(self, mock_request):
api = self._make_api()
good_response = MagicMock()
good_response.raise_for_status = MagicMock()
mock_request.side_effect = [
requests.exceptions.ConnectionError("Connection refused"),
good_response,
]
result = api._request_with_retry("GET", "https://api.example.com/test")
assert result == good_response
assert mock_request.call_count == 2
class TestAudioShakeUploadRetry:
"""Tests for upload_file using retry."""
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.post")
def test_upload_file_retries_on_502(self, mock_post, tmp_path):
config = AudioShakeConfig(api_token="test-token")
logger = MagicMock()
api = AudioShakeAPI(config, logger)
# Create a test file with known content
test_file = tmp_path / "test.flac"
test_file.write_bytes(b"fake audio data")
# First call: 502
bad_response = MagicMock()
bad_response.status_code = 502
bad_response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=bad_response)
# Second call: success
good_response = MagicMock()
good_response.raise_for_status = MagicMock()
good_response.json.return_value = {"id": "asset-123"}
good_response.status_code = 200
good_response.text = '{"id": "asset-123"}'
mock_post.side_effect = [bad_response, good_response]
asset_id = api.upload_file(str(test_file))
assert asset_id == "asset-123"
assert mock_post.call_count == 2
# Verify file was re-opened on each attempt (both calls got file data)
for call_args in mock_post.call_args_list:
files_arg = call_args[1].get("files") or call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("files")
assert files_arg is not None, "Each retry should pass files"
+89
View File
@@ -6,6 +6,8 @@ Tests that require full backend imports are in the emulator integration tests.
"""
import pytest
import json
from datetime import datetime, UTC
from unittest.mock import MagicMock, AsyncMock, patch
class TestJobStatusTransitionsForReview:
@@ -277,6 +279,93 @@ class TestReviewResubmissionClearsWorkerProgress:
assert callable(getattr(jm, 'delete_state_data_keys'))
class TestCompleteReviewZeroSegmentValidation:
"""Tests for the 0-segment lyrics guard in complete_review endpoint.
These test the actual HTTP endpoint, not just the validation logic.
"""
@pytest.fixture
def _patched_client(self):
"""Create TestClient with mocked services for complete_review."""
from backend.models.job import Job, JobStatus
mock_job_manager = MagicMock()
mock_job_manager.transition_to_state.return_value = True
mock_worker_service = MagicMock()
mock_creds = MagicMock()
mock_creds.universe_domain = "googleapis.com"
with patch("backend.api.routes.jobs.job_manager", mock_job_manager), \
patch("backend.api.routes.jobs.worker_service", mock_worker_service), \
patch("backend.services.firestore_service.firestore"), \
patch("backend.services.storage_service.storage"), \
patch("google.auth.default", return_value=(mock_creds, "test-project")):
from backend.main import app
from fastapi.testclient import TestClient
yield TestClient(app), mock_job_manager
def _make_job(self, state_data=None):
from backend.models.job import Job, JobStatus
return Job(
job_id="job-zero-seg",
status=JobStatus.AWAITING_REVIEW,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
artist="Test Artist",
title="Test Song",
state_data=state_data,
)
def test_endpoint_rejects_zero_segment_lyrics(self, _patched_client):
"""POST complete-review returns 400 when lyrics pipeline produced 0 segments."""
client, mock_jm = _patched_client
job = self._make_job(state_data={
'lyrics_metadata': {'segment_count': 0, 'ready_for_review': True},
'corrected_lyrics': {'corrected_segments': [], 'original_segments': []},
})
mock_jm.get_job.return_value = job
response = client.post(
"/api/jobs/job-zero-seg/complete-review",
headers={"Authorization": "Bearer test-admin-token"},
)
assert response.status_code == 400
assert "no lyrics were detected" in response.json()["detail"]
mock_jm.transition_to_state.assert_not_called()
def test_endpoint_allows_nonzero_segment_lyrics(self, _patched_client):
"""POST complete-review succeeds when lyrics have segments."""
client, mock_jm = _patched_client
job = self._make_job(state_data={
'lyrics_metadata': {'segment_count': 42},
'corrected_lyrics': {'corrected_segments': [{'text': 'hello'}] * 42},
})
mock_jm.get_job.return_value = job
response = client.post(
"/api/jobs/job-zero-seg/complete-review",
headers={"Authorization": "Bearer test-admin-token"},
)
assert response.status_code == 200
mock_jm.transition_to_state.assert_called_once()
def test_endpoint_allows_job_without_lyrics_metadata(self, _patched_client):
"""POST complete-review succeeds for jobs without lyrics_metadata (e.g. finalise-only)."""
client, mock_jm = _patched_client
job = self._make_job(state_data={}) # No lyrics pipeline ran
mock_jm.get_job.return_value = job
response = client.post(
"/api/jobs/job-zero-seg/complete-review",
headers={"Authorization": "Bearer test-admin-token"},
)
assert response.status_code == 200
class TestPreviewStyleLoading:
"""Tests for the unified style loader used in preview video generation.
@@ -399,8 +399,8 @@ class TestPackagingValidation:
"""
@pytest.mark.asyncio
async def test_cdg_validation_fails_when_enabled_but_not_produced(self):
"""Test that orchestrator raises when CDG enabled but not produced."""
async def test_cdg_validation_fails_when_enabled_but_not_produced_and_lrc_has_content(self):
"""Test that orchestrator raises when CDG enabled but not produced and LRC has lyrics."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
@@ -417,6 +417,7 @@ class TestPackagingValidation:
pass # CDG generation "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=True), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
result = await orchestrator.run()
@@ -426,8 +427,37 @@ class TestPackagingValidation:
mock_encoding.assert_not_called() # Should not reach encoding
@pytest.mark.asyncio
async def test_txt_validation_fails_when_enabled_but_not_produced(self):
"""Test that orchestrator raises when TXT enabled but not produced."""
async def test_cdg_skipped_gracefully_when_lrc_has_no_content(self):
"""Test that CDG failure is a warning (not error) when LRC has no lyrics content."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=True,
)
orchestrator = VideoWorkerOrchestrator(config)
async def mock_packaging():
pass # CDG generation "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=False), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock):
result = await orchestrator.run()
assert result.success is True # Pipeline continues
mock_encoding.assert_called_once() # Encoding still runs
@pytest.mark.asyncio
async def test_txt_validation_fails_when_enabled_but_not_produced_and_lrc_has_content(self):
"""Test that orchestrator raises when TXT enabled but not produced and LRC has lyrics."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
@@ -443,6 +473,7 @@ class TestPackagingValidation:
pass # TXT generation "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=True), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
result = await orchestrator.run()
@@ -506,7 +537,7 @@ class TestPackagingValidation:
@pytest.mark.asyncio
async def test_both_cdg_and_txt_validation_fail_together(self):
"""Test that orchestrator catches CDG failure when both CDG and TXT enabled."""
"""Test that orchestrator catches CDG failure when both CDG and TXT enabled and LRC has content."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
@@ -523,6 +554,7 @@ class TestPackagingValidation:
pass # Both CDG and TXT "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=True), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
result = await orchestrator.run()
@@ -1709,3 +1741,57 @@ class TestDistributionWarnings:
assert len(orchestrator.result.distribution_warnings) == 1
assert "Dropbox upload failed" in orchestrator.result.distribution_warnings[0]
class TestLrcHasLyricsContent:
"""Tests for _lrc_has_lyrics_content helper."""
def _make_orchestrator(self, lrc_file_path=None):
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
lrc_file_path=lrc_file_path,
)
return VideoWorkerOrchestrator(config)
def test_returns_false_when_no_lrc_path(self):
orchestrator = self._make_orchestrator(lrc_file_path=None)
assert orchestrator._lrc_has_lyrics_content() is False
def test_returns_false_when_lrc_file_missing(self):
orchestrator = self._make_orchestrator(lrc_file_path="/nonexistent/path.lrc")
assert orchestrator._lrc_has_lyrics_content() is False
def test_returns_false_for_empty_lrc(self, tmp_path):
lrc_file = tmp_path / "empty.lrc"
lrc_file.write_text("[re:MidiCo]\n")
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
assert orchestrator._lrc_has_lyrics_content() is False
def test_returns_true_for_lrc_with_lyrics(self, tmp_path):
lrc_file = tmp_path / "valid.lrc"
lrc_file.write_text("[re:MidiCo]\n[00:12.345]Hello world\n[00:15.678]Second line\n")
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
assert orchestrator._lrc_has_lyrics_content() is True
def test_returns_true_for_centisecond_timestamps(self, tmp_path):
lrc_file = tmp_path / "centisecond.lrc"
lrc_file.write_text("[01:02.34]Lyric line\n")
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
assert orchestrator._lrc_has_lyrics_content() is True
def test_returns_true_for_no_decimal_timestamps(self, tmp_path):
lrc_file = tmp_path / "nodecimal.lrc"
lrc_file.write_text("[01:30]Lyric line\n")
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
assert orchestrator._lrc_has_lyrics_content() is True
def test_returns_false_for_metadata_only_lrc(self, tmp_path):
lrc_file = tmp_path / "meta.lrc"
lrc_file.write_text("[ti:Song Title]\n[ar:Artist Name]\n[re:MidiCo]\n")
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
assert orchestrator._lrc_has_lyrics_content() is False
+1 -1
View File
@@ -439,7 +439,7 @@ async def process_audio_separation(job_id: str) -> bool:
except Exception as e:
duration = time.time() - start_time
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}")
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}", exc_info=True)
job_manager.mark_job_failed(
job_id=job_id,
error_message=f"Audio separation failed: {str(e)}",
+36 -8
View File
@@ -18,6 +18,7 @@ Pipeline stages:
import logging
import os
import re
import time
from dataclasses import dataclass, field
from typing import Optional, Dict, Any, List
@@ -234,16 +235,31 @@ class VideoWorkerOrchestrator:
await self._run_packaging()
# Validate required packaging outputs before expensive encoding
# If LRC has no lyrics content (e.g. input audio had no vocals),
# skip CDG/TXT gracefully instead of failing the entire job
lrc_has_content = self._lrc_has_lyrics_content()
if self.config.enable_cdg and not self.result.final_karaoke_cdg_zip:
raise RuntimeError(
"CDG generation was enabled but failed. "
"Check worker logs for CDG-related errors."
)
if lrc_has_content:
raise RuntimeError(
"CDG generation was enabled but failed. "
"Check worker logs for CDG-related errors."
)
else:
self.job_log.warning(
"CDG generation skipped: LRC file has no lyrics content. "
"Video will be generated without CDG package."
)
if self.config.enable_txt and not self.result.final_karaoke_txt_zip:
raise RuntimeError(
"TXT generation was enabled but failed. "
"Check worker logs for TXT-related errors."
)
if lrc_has_content:
raise RuntimeError(
"TXT generation was enabled but failed. "
"Check worker logs for TXT-related errors."
)
else:
self.job_log.warning(
"TXT generation skipped: LRC file has no lyrics content. "
"Video will be generated without TXT package."
)
# Stage 2: Encoding
await self._run_encoding()
@@ -276,6 +292,18 @@ class VideoWorkerOrchestrator:
return self.result
def _lrc_has_lyrics_content(self) -> bool:
"""Check if the LRC file has actual lyrics content (timestamp lines), not just metadata."""
if not self.config.lrc_file_path or not os.path.isfile(self.config.lrc_file_path):
return False
try:
with open(self.config.lrc_file_path, "r", encoding="utf-8") as f:
content = f.read()
# Match LRC timestamp lines: [MM:SS.xx] or [MM:SS.xxx] or [M:SS.xx]
return bool(re.search(r"\[\d{1,2}:\d{2}(?:\.\d{1,3})?\]", content))
except Exception:
return False
async def _run_packaging(self):
"""Run the packaging stage (CDG/TXT generation)."""
self.job_log.info("Starting packaging stage (CDG/TXT)")
+6
View File
@@ -20,6 +20,12 @@ Autocomplete on the artist/title input fields confused users — it wasn't clear
### Fuzzy Search: Artist Tiebreaker for Title-Based Fallback (Mar 2026)
When fuzzy-matching by title returns multiple artists with the same song name (e.g. "Bruises" by Lewis Capaldi vs Fox Stevenson), use the user's garbled artist input as a tiebreaker — `stringSimilarity(userInput, track.artist_name) * 0.2` as a bonus score. Also: Spotify's `q` parameter searches track names, not artist names, so `searchCatalogTracks(artist, artist, 20)` (artist as both query and filter) returns nothing. Use `searchCatalogTracks(artist, undefined, 20)` to get the artist's tracks via general search.
### Pipeline Robustness Patterns (Mar 2026)
Three patterns from investigating production job failures where errors cascaded far from their root cause:
1. **Validate every stage's outputs, not just some** — stage 1 of audio separation had validation for expected stems, stage 2 didn't. The missing validation let a transient Modal API failure propagate through 3 more functions before crashing with a confusing NoneType error.
2. **Block impossible downstream work early** — AudioShake returning 0 lyrics segments was treated as "success", letting the user submit an empty review that only failed at CDG generation. Validate at the boundary: if transcription returns 0 segments, fail with a user-friendly error, not 4 steps later.
3. **File handles in retry loops** — when wrapping HTTP calls with retry (e.g. tenacity), ensure file handles are re-opened on each attempt. A `with open()` wrapping a retried `requests.post(files=...)` exhausts the file handle on the first attempt; subsequent retries send empty data.
---
## Infrastructure
+30
View File
@@ -136,3 +136,33 @@ python scripts/backfill_gdrive_uploads.py --all-missing
# Specific jobs
python scripts/backfill_gdrive_uploads.py --job-ids JOB1,JOB2
```
---
## Job failed: "CDG generation was enabled but failed"
**Cause:** The LRC file has no lyrics content (just metadata like `[re:MidiCo]`). This happens when AudioShake returns 0 segments — typically because the input audio has no vocals (e.g. user uploaded a karaoke track).
**Fix (v0.135+):** The video orchestrator now skips CDG/TXT gracefully when the LRC has no content. The `complete_review` endpoint also blocks 0-segment submissions with a user-friendly error, and the frontend shows a warning with guidance.
**For older jobs:** Retry won't help unless the user provides different audio or pastes lyrics manually via "Replace All".
---
## Job failed: "Audio separation failed: expected str, bytes or os.PathLike object, not NoneType"
**Cause:** Modal API intermittently returned fewer stems than expected from stage 2 (backing vocals) separation. Missing stems caused NoneType crashes in downstream processing.
**Fix (v0.135+):** Stage 2 now validates that all expected stems are present (matching existing stage 1 validation). Defensive null checks prevent confusing TypeError messages. Full tracebacks are logged with `exc_info=True`.
**Recovery:** Retry the job — the failure is transient (Modal API intermittent). Long-term fix: migrate audio separation from Modal to GCP (see `docs/archive/2026-03-21-modal-to-gcp-migration-plan.md`).
---
## Job failed: "Lyrics transcription failed: 502 Server Error: Bad Gateway for url: https://api.audioshake.ai/tasks"
**Cause:** AudioShake API returned a transient 5xx error.
**Fix (v0.135+):** AudioShake API calls now retry up to 5 times with exponential backoff (60s base, 3x multiplier, ~40 minutes total spread). Retries on 5xx, 429, connection errors, and timeouts.
**For older jobs:** Simply retry the job via admin dashboard.
@@ -0,0 +1,121 @@
# Modal to GCP Audio Separation Migration Plan
**Date:** 2026-03-21
**Status:** Planning seed (not yet started)
**Priority:** Medium - reduces external dependency risk and cost
## Why Migrate
1. **Reliability** - Modal API intermittently fails to return stem files ("no files were downloaded" errors), causing job failures that require manual retries.
2. **Latency** - Round-trip to Modal (upload audio, wait for GPU separation, download stems) adds network overhead. Local GCP processing eliminates cross-provider data transfer.
3. **Cost** - Modal charges for GPU compute on their platform. GCP GPU VMs or spot instances may be cheaper, especially with sustained use discounts.
4. **Dependency reduction** - Modal is the only third-party compute dependency. Removing it means the entire pipeline runs within GCP.
## Current State
### How Modal Is Used
The audio worker (`backend/workers/audio_worker.py`) runs on Cloud Run and calls Modal via the `audio-separator[remote]` Python package:
1. Cloud Run audio worker downloads the source audio from GCS
2. `AudioProcessor.process_audio_separation()` checks `AUDIO_SEPARATOR_API_URL` env var
3. Calls `AudioSeparatorAPIClient.separate_audio()` which POSTs the audio file to Modal's HTTP API
4. Modal runs GPU separation and returns stem files
5. Worker downloads stems, post-processes, and uploads to GCS
**API endpoint:** `https://nomadkaraoke--audio-separator-api.modal.run`
**Configured in:** `infrastructure/modules/cloud_run.py` (env var on Cloud Run service)
### Models Used
- **Stage 1 (Clean instrumental):** `model_bs_roformer_ep_317_sdr_12.9755.ckpt` (BS-Roformer)
- **Stage 1 (Other stems):** `htdemucs_6s.yaml` (Demucs 6-stem)
- **Stage 2 (Backing vocals):** `mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt` (Mel-Band Roformer)
### Processing Time
- Stage 1: ~3-5 minutes
- Stage 2: ~2-3 minutes
- Total audio worker timeout: 30 minutes (includes download/upload overhead)
### Existing GCP Infrastructure
- **Encoding worker VM** (`c4d-highcpu-32`, `us-central1-c`) - CPU-only, handles video encoding
- Packer-based image build, GCS-based deployment (wheel + startup.sh)
- Pulumi IaC in `infrastructure/`
- The `audio-separator` package already supports local processing (just needs GPU + models)
## Target State
Run audio separation on a GCP GPU VM instead of Modal. The `audio-separator` package already supports both local and remote modes - we just need to point it at our own GPU endpoint (or run locally on the GPU VM).
## Options
### Option A: Dedicated GCE GPU VM (Recommended)
Similar pattern to the encoding worker. A persistent VM with GPU that runs the audio-separator API.
- **Pros:** Simple, mirrors existing encoding worker pattern, models stay loaded in memory, fast cold start after first load
- **Cons:** Pays for GPU even when idle, need to handle scaling
- **GPU:** NVIDIA T4 (~$0.35/hr) or L4 (~$0.70/hr) sufficient for these models
- **Cost estimate:** T4 spot instance ~$100-150/month if running 24/7 (vs Modal pay-per-use)
### Option B: GCE Spot GPU VM (Start/Stop)
Same as A but stop the VM when not processing, start on demand.
- **Pros:** Only pay when processing, cheapest option
- **Cons:** Cold start (VM boot + model loading ~2-5 min), spot can be preempted, more complex orchestration
- **Best if:** Job volume is low/bursty
### Option C: Cloud Run Jobs with GPU
Cloud Run now supports GPU (L4) for jobs.
- **Pros:** Serverless, auto-scaling, no idle cost, Cloud Run already used for other workers
- **Cons:** GPU availability can be limited, cold start on each job, 1-hour max timeout, model download on each cold start unless cached
- **Cost estimate:** Pay only for job duration, but model loading adds per-job overhead
### Option D: Run Locally on Audio Worker (Cloud Run)
Run separation directly in the Cloud Run audio worker with a GPU.
- **Pros:** Simplest code change (just remove AUDIO_SEPARATOR_API_URL, models load locally)
- **Cons:** Cloud Run GPU instances are expensive if kept warm, model loading on cold start
## Recommendation
**Option A (dedicated GCE GPU VM)** for initial migration, with a path to Option B if cost is a concern.
Rationale:
- Mirrors the proven encoding worker pattern (Packer image, GCS deploy, Pulumi)
- Models stay loaded = fast separation with no cold start penalty
- Simplest to implement and debug
- Can add start/stop automation later (Option B) if idle cost is too high
## Key Considerations
- **Model storage:** ~2-3 GB total for the three models. Bake into Packer image or download on first start to a persistent disk.
- **GPU memory:** These models fit comfortably in a T4 (16 GB VRAM). L4 (24 GB) provides headroom.
- **API compatibility:** Can reuse the existing `AudioSeparatorAPIClient` by running the audio-separator API server on the GCE VM and updating `AUDIO_SEPARATOR_API_URL` to point to it.
- **Fallback:** Keep Modal config as a fallback during migration. The `AUDIO_SEPARATOR_API_URL` env var swap makes this easy.
- **Networking:** VM needs to accept requests from Cloud Run audio worker. Use internal networking or IAP + auth token (same pattern as encoding worker).
## Migration Steps (High Level)
1. **Spike:** Run audio-separator API server locally with GPU, verify it produces identical output to Modal
2. **Infrastructure:** Create Pulumi resources for GPU VM (instance, disk, firewall, service account) - copy encoding worker pattern
3. **Packer image:** Build image with CUDA, Python, audio-separator, pre-downloaded models
4. **Deploy & test:** Stand up VM, point a test job at it, compare output quality and timing
5. **Cutover:** Update `AUDIO_SEPARATOR_API_URL` in Cloud Run config to point to GCE VM
6. **Monitor:** Watch for errors/quality issues for a week
7. **Decommission:** Remove Modal deployment, remove `audio-separator-api-url` secret, clean up Modal account
## Files to Modify (When Implementing)
- `infrastructure/compute/` - New GPU VM module (similar to `encoding_worker_vm.py`)
- `infrastructure/packer/` - New Packer template for GPU image
- `infrastructure/modules/cloud_run.py` - Update `AUDIO_SEPARATOR_API_URL` value
- `infrastructure/__main__.py` - Wire up new resources
- `infrastructure/config.py` - Add GPU VM machine type/zone constants
- No backend code changes needed if we keep the HTTP API interface
@@ -0,0 +1,102 @@
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import ReviewChangesModal from '../modals/ReviewChangesModal'
import type { CorrectionData } from '@/lib/lyrics-review/types'
// Mock PreviewVideoSection since it has complex dependencies
jest.mock('../PreviewVideoSection', () => ({
__esModule: true,
default: () => <div data-testid="preview-video">Preview</div>,
}))
function makeData(overrides: Partial<CorrectionData> = {}): CorrectionData {
return {
original_segments: [],
reference_lyrics: {},
anchor_sequences: [],
gap_sequences: [],
resized_segments: [],
corrections_made: 0,
confidence: 1,
corrections: [],
corrected_segments: [],
metadata: {},
...overrides,
} as CorrectionData
}
describe('ReviewChangesModal', () => {
const defaultProps = {
open: true,
onClose: jest.fn(),
onSubmit: jest.fn(),
}
it('disables submit button when there are no lyrics (0 segments)', () => {
render(
<ReviewChangesModal
{...defaultProps}
data={makeData({ corrected_segments: [] })}
/>
)
const submitButton = screen.getByRole('button', { name: /proceed to instrumental/i })
expect(submitButton).toBeDisabled()
})
it('shows warning message when there are no lyrics', () => {
render(
<ReviewChangesModal
{...defaultProps}
data={makeData({ corrected_segments: [] })}
/>
)
expect(screen.getByText('No lyrics detected')).toBeInTheDocument()
expect(screen.getByText(/no lyrics were found in the audio/i)).toBeInTheDocument()
expect(screen.getByText(/replace all/i)).toBeInTheDocument()
})
it('enables submit button when segments exist', () => {
const segments = [{ text: 'Hello', words: [], start_time: 0, end_time: 1 }]
render(
<ReviewChangesModal
{...defaultProps}
data={makeData({ corrected_segments: segments as any })}
/>
)
const submitButton = screen.getByRole('button', { name: /proceed to instrumental/i })
expect(submitButton).not.toBeDisabled()
})
it('does not show warning when segments exist', () => {
const segments = [{ text: 'Hello', words: [], start_time: 0, end_time: 1 }]
render(
<ReviewChangesModal
{...defaultProps}
data={makeData({ corrected_segments: segments as any })}
/>
)
expect(screen.queryByText('No lyrics detected')).not.toBeInTheDocument()
expect(screen.getByText(/total segments: 1/i)).toBeInTheDocument()
})
it('calls onSubmit when button clicked with valid segments', async () => {
const user = userEvent.setup()
const onSubmit = jest.fn()
const segments = [{ text: 'Hello', words: [], start_time: 0, end_time: 1 }]
render(
<ReviewChangesModal
{...defaultProps}
onSubmit={onSubmit}
data={makeData({ corrected_segments: segments as any })}
/>
)
await user.click(screen.getByRole('button', { name: /proceed to instrumental/i }))
expect(onSubmit).toHaveBeenCalledTimes(1)
})
})
@@ -3,7 +3,7 @@
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog'
import { Button } from '@/components/ui/button'
import { CorrectionData } from '@/lib/lyrics-review/types'
import { ArrowLeft, ArrowRight, Loader2 } from 'lucide-react'
import { AlertTriangle, ArrowLeft, ArrowRight, Loader2 } from 'lucide-react'
import PreviewVideoSection from '../PreviewVideoSection'
interface ApiClient {
@@ -36,6 +36,7 @@ export default function ReviewChangesModal({
}: ReviewChangesModalProps) {
const corrections = data.corrections || []
const totalSegments = data.corrected_segments?.length || 0
const hasNoLyrics = totalSegments === 0
// Check if there are manual corrections (user-made changes)
const hasManualCorrections = corrections.some(c => c.handler === 'ManualCorrector' || c.handler === 'UserEdit')
@@ -60,15 +61,35 @@ export default function ReviewChangesModal({
timingOffsetMs={timingOffsetMs}
/>
{/* No lyrics warning */}
{hasNoLyrics && (
<div className="flex items-start gap-3 rounded-lg border border-yellow-500/50 bg-yellow-500/10 p-4">
<AlertTriangle className="h-5 w-5 text-yellow-500 mt-0.5 shrink-0" />
<div className="text-sm space-y-2">
<p className="font-medium text-yellow-500">No lyrics detected</p>
<p className="text-muted-foreground">
No lyrics were found in the audio. This usually happens when the input audio has no vocals
(e.g. a karaoke track or instrumental).
</p>
<p className="text-muted-foreground">
You can go back and paste lyrics manually using <strong>Replace All</strong>,
or cancel this job and try again with an audio file that contains vocals.
</p>
</div>
</div>
)}
{/* Info text */}
<div className="text-sm text-muted-foreground space-y-1">
{hasManualCorrections ? (
<p>Manual corrections detected. Review the preview to ensure the lyrics are synchronized correctly.</p>
) : (
<p>No manual corrections detected. If everything looks good in the preview, proceed to select your instrumental.</p>
)}
<p>Total segments: {totalSegments}</p>
</div>
{!hasNoLyrics && (
<div className="text-sm text-muted-foreground space-y-1">
{hasManualCorrections ? (
<p>Manual corrections detected. Review the preview to ensure the lyrics are synchronized correctly.</p>
) : (
<p>No manual corrections detected. If everything looks good in the preview, proceed to select your instrumental.</p>
)}
<p>Total segments: {totalSegments}</p>
</div>
)}
<DialogFooter className="border-t pt-4">
<Button variant="ghost" onClick={onClose} disabled={isSubmitting} className="text-primary">
@@ -77,7 +98,7 @@ export default function ReviewChangesModal({
</Button>
<Button
onClick={handleSubmit}
disabled={isSubmitting}
disabled={isSubmitting || hasNoLyrics}
className="bg-green-600 hover:bg-green-700"
>
{isSubmitting ? (
+31 -4
View File
@@ -395,6 +395,23 @@ class AudioProcessor:
backing_vocals_result = self._organize_stage2_remote_results(
stage2_result["downloaded_files"], artist_title, stems_dir
)
# Validate stage 2 outputs — each model should produce both lead_vocals and backing_vocals
for bv_model, paths in backing_vocals_result.items():
missing = []
if not paths.get("lead_vocals"):
missing.append("lead_vocals")
if not paths.get("backing_vocals"):
missing.append("backing_vocals")
if missing:
error_msg = (
f"Stage 2 completed but model {bv_model} is missing: {', '.join(missing)}. "
f"Got keys: {list(paths.keys())}. Downloaded {len(stage2_result['downloaded_files'])} files. "
"This may indicate an intermittent issue with the remote separation API."
)
self.logger.error(error_msg)
raise Exception(error_msg)
result["backing_vocals"] = backing_vocals_result
else:
error_msg = f"Stage 2 backing vocals separation failed: {stage2_result.get('error', 'Unknown error')}"
@@ -418,7 +435,7 @@ class AudioProcessor:
return result
except Exception as e:
self.logger.error(f"Error during remote audio separation: {str(e)}")
self.logger.error(f"Error during remote audio separation: {str(e)}", exc_info=True)
raise e
def _organize_stage1_remote_results(self, downloaded_files, artist_title, track_output_dir, stems_dir):
@@ -644,7 +661,10 @@ class AudioProcessor:
self.logger.info("Generating combined instrumental tracks with backing vocals")
result = {}
for model, paths in backing_vocals_result.items():
backing_vocals_path = paths["backing_vocals"]
backing_vocals_path = paths.get("backing_vocals")
if not backing_vocals_path:
self.logger.error(f"Skipping combined instrumental for model {model}: backing_vocals path is missing")
continue
combined_path = os.path.join(track_output_dir, f"{artist_title} (Instrumental +BV {model}).{self.lossless_output_format}")
if not self._file_exists(combined_path):
@@ -663,9 +683,16 @@ class AudioProcessor:
def _normalize_audio_files(self, separation_result, artist_title, track_output_dir):
self.logger.info("Normalizing clean instrumental and combined instrumentals")
clean_instrumental_path = separation_result.get("clean_instrumental", {}).get("instrumental")
if not clean_instrumental_path:
raise ValueError(
"Cannot normalize audio: clean instrumental path is missing. "
"This indicates the audio separation did not produce the expected instrumental stem."
)
files_to_normalize = [
("clean_instrumental", separation_result["clean_instrumental"]["instrumental"]),
] + [("combined_instrumentals", path) for path in separation_result["combined_instrumentals"].values()]
("clean_instrumental", clean_instrumental_path),
] + [("combined_instrumentals", path) for path in separation_result.get("combined_instrumentals", {}).values()]
for key, file_path in files_to_normalize:
if self._file_exists(file_path):
@@ -1,10 +1,19 @@
from dataclasses import dataclass
import logging
import requests
import time
import os
import tempfile
from typing import Dict, Optional, Any, Union, Tuple
from pathlib import Path
from tenacity import (
retry,
stop_after_attempt,
wait_exponential,
retry_if_exception,
before_sleep_log,
)
from pydub import AudioSegment
from karaoke_gen.lyrics_transcriber.types import TranscriptionData, LyricsSegment, Word
from karaoke_gen.lyrics_transcriber.transcribers.base_transcriber import BaseTranscriber, TranscriptionError
@@ -107,6 +116,30 @@ class AudioShakeConfig:
timeout_minutes: int = 20 # Added timeout configuration
def _is_retryable_error(exception: BaseException) -> bool:
"""Check if an exception is retryable (server errors, rate limits, network issues)."""
if isinstance(exception, requests.exceptions.ConnectionError):
return True
if isinstance(exception, requests.exceptions.Timeout):
return True
if isinstance(exception, requests.exceptions.HTTPError):
status = exception.response.status_code
return status >= 500 or status == 429
return False
# Retry decorator for AudioShake API calls.
# 5 attempts with exponential backoff: ~60s, ~180s, ~540s, ~1620s (capped at 1800s)
# Total spread: ~40 minutes
_audioshake_retry = retry(
stop=stop_after_attempt(5),
wait=wait_exponential(multiplier=60, max=1800),
retry=retry_if_exception(_is_retryable_error),
before_sleep=before_sleep_log(logging.getLogger(__name__), logging.WARNING),
reraise=True,
)
class AudioShakeAPI:
"""Handles direct API interactions with AudioShake."""
@@ -124,18 +157,31 @@ class AudioShakeAPI:
self._validate_config() # Validate before making any API calls
return {"x-api-key": self.config.api_token, "Content-Type": "application/json"}
@_audioshake_retry
def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response:
"""Make an HTTP request with retry on transient failures."""
response = requests.request(method, url, **kwargs)
response.raise_for_status()
return response
@_audioshake_retry
def _upload_file_with_retry(self, url: str, filepath: str) -> requests.Response:
"""Upload a file with retry, re-opening the file on each attempt to avoid stale handles."""
with open(filepath, "rb") as file:
files = {"file": (os.path.basename(filepath), file)}
response = requests.post(url, headers={"x-api-key": self.config.api_token}, files=files)
response.raise_for_status()
return response
def upload_file(self, filepath: str) -> str:
"""Upload audio file via Assets API and return asset ID."""
self.logger.info(f"Uploading {filepath} to AudioShake")
self._validate_config() # Validate before making API call
url = f"{self.config.base_url}/assets"
with open(filepath, "rb") as file:
files = {"file": (os.path.basename(filepath), file)}
response = requests.post(url, headers={"x-api-key": self.config.api_token}, files=files)
response = self._upload_file_with_retry(url, filepath)
self.logger.debug(f"Upload response: {response.status_code} - {response.text}")
response.raise_for_status()
return response.json()["id"]
def create_task(self, asset_id: str) -> str:
@@ -153,8 +199,7 @@ class AudioShakeAPI:
}
],
}
response = requests.post(url, headers=self._get_headers(), json=data)
response.raise_for_status()
response = self._request_with_retry("POST", url, headers=self._get_headers(), json=data)
return response.json()["id"]
def wait_for_task_result(self, task_id: str) -> Dict[str, Any]:
@@ -186,17 +231,16 @@ class AudioShakeAPI:
last_status_log = current_time
try:
response = requests.get(url, headers=self._get_headers())
response.raise_for_status()
response = self._request_with_retry("GET", url, headers=self._get_headers())
tasks_list = response.json()
# Find our specific task in the list
task_data = None
for task in tasks_list:
if task.get("id") == task_id:
task_data = task
break
if not task_data:
# Task not found in list yet
if initial_retry_count < max_initial_retries:
@@ -206,7 +250,7 @@ class AudioShakeAPI:
continue
else:
raise TranscriptionError(f"Task {task_id} not found in task list after {max_initial_retries} retries")
# Log the full response for debugging
self.logger.debug(f"Task status response: {task_data}")
@@ -214,29 +258,29 @@ class AudioShakeAPI:
targets = task_data.get("targets", [])
if not targets:
raise TranscriptionError("No targets found in task response")
# Check if all targets are completed or if any failed
all_completed = True
for target in targets:
target_status = target.get("status")
target_model = target.get("model")
self.logger.debug(f"Target {target_model} status: {target_status}")
if target_status == "failed":
error_msg = target.get("error", "Unknown error")
raise TranscriptionError(f"Target {target_model} failed: {error_msg}")
elif target_status != "completed":
all_completed = False
if all_completed:
self.logger.info("All targets completed successfully")
return task_data
# Reset retry count on successful response
initial_retry_count = 0
except requests.exceptions.HTTPError as e:
raise
except (TranscriptionError, requests.exceptions.HTTPError):
raise # Non-retryable or already exhausted retries — propagate
time.sleep(30) # Wait before next poll
@@ -332,8 +376,7 @@ class AudioShakeTranscriber(BaseTranscriber):
raise TranscriptionError("Output link not found in alignment target")
# Fetch transcription data
response = requests.get(output_url)
response.raise_for_status()
response = self.api._request_with_retry("GET", output_url)
# Return combined raw data
raw_data = {"task_data": task_data, "transcription": response.json()}
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.146.0"
version = "0.147.0"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+23 -12
View File
@@ -43,21 +43,32 @@ def mock_email_service():
@pytest.fixture
def client(admin_auth_result, mock_user_service, mock_email_service):
"""Test client with mocked dependencies."""
from fastapi import FastAPI
from backend.api.routes.users import router
from backend.api.dependencies import require_admin
from backend.services.user_service import get_user_service
from backend.services.email_service import get_email_service
"""Test client with mocked dependencies.
app = FastAPI()
app.include_router(router, prefix="/api")
GCP services (Firestore, Storage) are mocked because importing the users
router triggers module-level imports (file_upload.py -> JobManager -> FirestoreService)
that require GCP credentials not available in CI.
"""
mock_creds = MagicMock()
mock_creds.universe_domain = "googleapis.com"
app.dependency_overrides[require_admin] = lambda: admin_auth_result
app.dependency_overrides[get_user_service] = lambda: mock_user_service
app.dependency_overrides[get_email_service] = lambda: mock_email_service
with patch("backend.services.firestore_service.firestore"), \
patch("backend.services.storage_service.storage"), \
patch("google.auth.default", return_value=(mock_creds, "test-project")):
from fastapi import FastAPI
from backend.api.routes.users import router
from backend.api.dependencies import require_admin
from backend.services.user_service import get_user_service
from backend.services.email_service import get_email_service
return TestClient(app)
app = FastAPI()
app.include_router(router, prefix="/api")
app.dependency_overrides[require_admin] = lambda: admin_auth_result
app.dependency_overrides[get_user_service] = lambda: mock_user_service
app.dependency_overrides[get_email_service] = lambda: mock_email_service
yield TestClient(app)
class TestAddCredits: