mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Fix three production job failure modes with robust error handling (#565)
@coderabbitai ignore ## Summary Fixes three distinct production job failures identified from investigating jobs 503a4ad3, 7be8d1e5, and 1868f323: - **0-segment lyrics guard**: Block review submission when transcription produces no lyrics (e.g. karaoke input with no vocals). Backend validation + frontend warning with guidance. CDG orchestrator gracefully skips when LRC has no content. - **Audio processor stage 2 validation**: Validate backing vocals separation produced all expected stems (mirrors existing stage 1 validation). Defensive null checks prevent confusing NoneType crashes. Full tracebacks in error logs. - **AudioShake retry with exponential backoff**: 5 attempts via tenacity, ~40 min spread (60s base, 3x multiplier), retries on 5xx/429/connection errors. Applied to all HTTP call points. Fixed file handle exhaustion bug on upload retry. Also includes: - Troubleshooting runbooks for all 3 failure modes - Lessons learned entry on pipeline robustness patterns - Seed planning doc for Modal-to-GCP audio separation migration - Version bump to 0.147.0 ## Test plan - [x] 2696 backend tests passing (54 new/modified) - [x] 712 frontend tests passing (5 new for ReviewChangesModal) - [x] HTTP endpoint test verifies 400 response for 0-segment review submission - [x] AudioShake retry tests verify 502 retry, 400 no-retry, connection error retry - [x] Audio processor tests verify stage 2 validation, null safety, combined instrumentals - [x] CDG orchestrator tests verify graceful skip on empty LRC, fail on valid LRC errors - [x] LRC timestamp regex tests for millisecond, centisecond, and no-decimal formats 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
de02570514
commit
25aa2ef8ef
16 files changed
+970
-60
No files matched your search
@@ -1284,6 +1284,26 @@ async def complete_review(
|
||||
detail=f"Job not in review state (current status: {job.status})"
|
||||
)
|
||||
|
||||
# Check for 0-segment lyrics — fail early with user-friendly message
|
||||
# Only check when lyrics pipeline has run (lyrics_metadata exists in state_data)
|
||||
state_data = job.state_data or {}
|
||||
lyrics_metadata = state_data.get('lyrics_metadata')
|
||||
if lyrics_metadata is not None:
|
||||
segment_count = lyrics_metadata.get('segment_count', 0)
|
||||
corrected_lyrics = state_data.get('corrected_lyrics', {})
|
||||
corrected_segments = corrected_lyrics.get('corrected_segments', [])
|
||||
|
||||
if segment_count == 0 and len(corrected_segments) == 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"Cannot complete review: no lyrics were detected in the audio. "
|
||||
"This can happen when the input audio has no vocals (e.g. a karaoke track or instrumental). "
|
||||
"You can go back and paste lyrics manually using 'Replace All', "
|
||||
"or cancel this job and try again with an audio file that contains vocals."
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
# Store instrumental selection if provided (combined review flow)
|
||||
instrumental_selection = body.instrumental_selection if body else None
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
Tests for audio processor robustness fixes.
|
||||
|
||||
Covers:
|
||||
- Stage 2 validation catches missing stems
|
||||
- _normalize_audio_files handles missing clean instrumental path
|
||||
- _generate_combined_instrumentals handles missing backing vocals path
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from karaoke_gen.audio_processor import AudioProcessor
|
||||
|
||||
|
||||
def _make_processor():
|
||||
"""Create an AudioProcessor with mocked dependencies."""
|
||||
processor = AudioProcessor.__new__(AudioProcessor)
|
||||
processor.logger = MagicMock()
|
||||
processor.clean_instrumental_model = "model_bs_roformer"
|
||||
processor.backing_vocals_models = ["mel_band_roformer_karaoke"]
|
||||
processor.other_stems_models = []
|
||||
processor.lossless_output_format = "FLAC"
|
||||
processor.ffmpeg_base_command = "ffmpeg -y"
|
||||
return processor
|
||||
|
||||
|
||||
class TestNormalizeAudioFilesNullSafety:
|
||||
"""Tests for _normalize_audio_files defensive checks."""
|
||||
|
||||
def test_raises_on_missing_clean_instrumental(self):
|
||||
processor = _make_processor()
|
||||
separation_result = {
|
||||
"clean_instrumental": {}, # missing "instrumental" key
|
||||
"combined_instrumentals": {},
|
||||
}
|
||||
with pytest.raises(ValueError, match="clean instrumental path is missing"):
|
||||
processor._normalize_audio_files(separation_result, "Artist - Title", "/tmp/output")
|
||||
|
||||
def test_raises_on_none_clean_instrumental(self):
|
||||
processor = _make_processor()
|
||||
separation_result = {
|
||||
"clean_instrumental": {"instrumental": None},
|
||||
"combined_instrumentals": {},
|
||||
}
|
||||
with pytest.raises(ValueError, match="clean instrumental path is missing"):
|
||||
processor._normalize_audio_files(separation_result, "Artist - Title", "/tmp/output")
|
||||
|
||||
def test_works_with_valid_path(self, tmp_path):
|
||||
processor = _make_processor()
|
||||
# Create a real file so _file_exists returns True
|
||||
audio_file = tmp_path / "instrumental.flac"
|
||||
audio_file.write_bytes(b"fake audio data")
|
||||
|
||||
separation_result = {
|
||||
"clean_instrumental": {"instrumental": str(audio_file), "vocals": str(tmp_path / "vocals.flac")},
|
||||
"combined_instrumentals": {},
|
||||
"other_stems": {},
|
||||
"backing_vocals": {},
|
||||
}
|
||||
# Mock _normalize_audio to avoid actually running ffmpeg
|
||||
with patch.object(processor, "_normalize_audio"):
|
||||
processor._normalize_audio_files(separation_result, "Artist - Title", str(tmp_path))
|
||||
|
||||
# Should have logged success (no exception)
|
||||
assert processor.logger.info.called
|
||||
|
||||
def test_normalizes_combined_instrumentals(self, tmp_path):
|
||||
processor = _make_processor()
|
||||
instrumental = tmp_path / "instrumental.flac"
|
||||
instrumental.write_bytes(b"fake instrumental")
|
||||
combined = tmp_path / "combined.flac"
|
||||
combined.write_bytes(b"fake combined")
|
||||
|
||||
separation_result = {
|
||||
"clean_instrumental": {"instrumental": str(instrumental), "vocals": str(tmp_path / "v.flac")},
|
||||
"combined_instrumentals": {"mel_band_roformer_karaoke": str(combined)},
|
||||
}
|
||||
with patch.object(processor, "_normalize_audio") as mock_normalize:
|
||||
processor._normalize_audio_files(separation_result, "Artist - Title", str(tmp_path))
|
||||
|
||||
# Both clean instrumental and combined instrumental should be normalized
|
||||
assert mock_normalize.call_count == 2
|
||||
normalized_paths = [call.args[0] for call in mock_normalize.call_args_list]
|
||||
assert str(instrumental) in normalized_paths
|
||||
assert str(combined) in normalized_paths
|
||||
|
||||
|
||||
class TestGenerateCombinedInstrumentalsNullSafety:
|
||||
"""Tests for _generate_combined_instrumentals defensive checks."""
|
||||
|
||||
def test_skips_model_with_missing_backing_vocals(self, tmp_path):
|
||||
processor = _make_processor()
|
||||
instrumental_path = str(tmp_path / "instrumental.flac")
|
||||
|
||||
# Model has lead_vocals but not backing_vocals
|
||||
backing_vocals_result = {
|
||||
"mel_band_roformer_karaoke": {"lead_vocals": "/tmp/lead.flac"}
|
||||
# "backing_vocals" key is missing
|
||||
}
|
||||
result = processor._generate_combined_instrumentals(
|
||||
instrumental_path, backing_vocals_result, "Artist - Title", str(tmp_path)
|
||||
)
|
||||
assert result == {}
|
||||
processor.logger.error.assert_called_once()
|
||||
assert "backing_vocals path is missing" in processor.logger.error.call_args[0][0]
|
||||
|
||||
|
||||
class TestStage2Validation:
|
||||
"""Tests for stage 2 remote results validation in _process_audio_separation_remote."""
|
||||
|
||||
def test_organize_stage2_missing_backing_vocals_stem(self):
|
||||
"""Test that _organize_stage2_remote_results correctly handles partial results."""
|
||||
processor = _make_processor()
|
||||
|
||||
# Simulate only getting the Vocals file, not the Instrumental file
|
||||
# This is what happened in the production failure
|
||||
import tempfile, os
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
stems_dir = os.path.join(tmpdir, "stems")
|
||||
os.makedirs(stems_dir)
|
||||
|
||||
# Create only the vocals file (missing the instrumental/backing vocals file)
|
||||
vocals_file = os.path.join(tmpdir, "audio_(Vocals)_mel_band_roformer_karaoke.flac")
|
||||
with open(vocals_file, "w") as f:
|
||||
f.write("fake")
|
||||
|
||||
result = processor._organize_stage2_remote_results(
|
||||
[vocals_file], "Artist - Title", stems_dir
|
||||
)
|
||||
|
||||
# Should have the model with only lead_vocals, no backing_vocals
|
||||
assert "mel_band_roformer_karaoke" in result
|
||||
assert "lead_vocals" in result["mel_band_roformer_karaoke"]
|
||||
assert "backing_vocals" not in result["mel_band_roformer_karaoke"]
|
||||
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
Tests for AudioShake API retry logic.
|
||||
|
||||
Covers:
|
||||
- Retry on 5xx server errors
|
||||
- Retry on 429 rate limit
|
||||
- Retry on connection errors
|
||||
- No retry on 4xx client errors (except 429)
|
||||
- Retry exhaustion raises the original error
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch, call
|
||||
import requests
|
||||
from tenacity import wait_none
|
||||
|
||||
from karaoke_gen.lyrics_transcriber.transcribers.audioshake import (
|
||||
AudioShakeAPI,
|
||||
AudioShakeConfig,
|
||||
_is_retryable_error,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_retry_wait():
|
||||
"""Disable retry wait times in all tests to avoid sleeping."""
|
||||
# Temporarily replace the wait strategy with no-wait for both retry-decorated methods
|
||||
methods = [AudioShakeAPI._request_with_retry, AudioShakeAPI._upload_file_with_retry]
|
||||
original_waits = [m.retry.wait for m in methods]
|
||||
for m in methods:
|
||||
m.retry.wait = wait_none()
|
||||
yield
|
||||
for m, w in zip(methods, original_waits):
|
||||
m.retry.wait = w
|
||||
|
||||
|
||||
class TestIsRetryableError:
|
||||
"""Tests for the _is_retryable_error predicate."""
|
||||
|
||||
def test_connection_error_is_retryable(self):
|
||||
assert _is_retryable_error(requests.exceptions.ConnectionError()) is True
|
||||
|
||||
def test_timeout_is_retryable(self):
|
||||
assert _is_retryable_error(requests.exceptions.Timeout()) is True
|
||||
|
||||
def test_502_is_retryable(self):
|
||||
response = MagicMock()
|
||||
response.status_code = 502
|
||||
error = requests.exceptions.HTTPError(response=response)
|
||||
assert _is_retryable_error(error) is True
|
||||
|
||||
def test_503_is_retryable(self):
|
||||
response = MagicMock()
|
||||
response.status_code = 503
|
||||
error = requests.exceptions.HTTPError(response=response)
|
||||
assert _is_retryable_error(error) is True
|
||||
|
||||
def test_429_is_retryable(self):
|
||||
response = MagicMock()
|
||||
response.status_code = 429
|
||||
error = requests.exceptions.HTTPError(response=response)
|
||||
assert _is_retryable_error(error) is True
|
||||
|
||||
def test_400_is_not_retryable(self):
|
||||
response = MagicMock()
|
||||
response.status_code = 400
|
||||
error = requests.exceptions.HTTPError(response=response)
|
||||
assert _is_retryable_error(error) is False
|
||||
|
||||
def test_401_is_not_retryable(self):
|
||||
response = MagicMock()
|
||||
response.status_code = 401
|
||||
error = requests.exceptions.HTTPError(response=response)
|
||||
assert _is_retryable_error(error) is False
|
||||
|
||||
def test_404_is_not_retryable(self):
|
||||
response = MagicMock()
|
||||
response.status_code = 404
|
||||
error = requests.exceptions.HTTPError(response=response)
|
||||
assert _is_retryable_error(error) is False
|
||||
|
||||
def test_value_error_is_not_retryable(self):
|
||||
assert _is_retryable_error(ValueError("bad")) is False
|
||||
|
||||
|
||||
class TestAudioShakeAPIRetry:
|
||||
"""Tests for AudioShakeAPI._request_with_retry."""
|
||||
|
||||
def _make_api(self):
|
||||
config = AudioShakeConfig(api_token="test-token")
|
||||
logger = MagicMock()
|
||||
return AudioShakeAPI(config, logger)
|
||||
|
||||
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
|
||||
def test_succeeds_on_first_try(self, mock_request):
|
||||
api = self._make_api()
|
||||
mock_response = MagicMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_request.return_value = mock_response
|
||||
|
||||
result = api._request_with_retry("GET", "https://api.example.com/test")
|
||||
assert result == mock_response
|
||||
mock_request.assert_called_once()
|
||||
|
||||
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
|
||||
def test_retries_on_502_then_succeeds(self, mock_request):
|
||||
api = self._make_api()
|
||||
|
||||
# First call: 502 error
|
||||
bad_response = MagicMock()
|
||||
bad_response.status_code = 502
|
||||
bad_response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=bad_response)
|
||||
|
||||
# Second call: success
|
||||
good_response = MagicMock()
|
||||
good_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_request.side_effect = [bad_response, good_response]
|
||||
|
||||
result = api._request_with_retry("GET", "https://api.example.com/test")
|
||||
|
||||
assert result == good_response
|
||||
assert mock_request.call_count == 2
|
||||
|
||||
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
|
||||
def test_does_not_retry_on_400(self, mock_request):
|
||||
api = self._make_api()
|
||||
|
||||
bad_response = MagicMock()
|
||||
bad_response.status_code = 400
|
||||
bad_response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=bad_response)
|
||||
mock_request.return_value = bad_response
|
||||
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
api._request_with_retry("GET", "https://api.example.com/test")
|
||||
|
||||
# Should NOT retry — only 1 call
|
||||
mock_request.assert_called_once()
|
||||
|
||||
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.request")
|
||||
def test_retries_on_connection_error(self, mock_request):
|
||||
api = self._make_api()
|
||||
|
||||
good_response = MagicMock()
|
||||
good_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_request.side_effect = [
|
||||
requests.exceptions.ConnectionError("Connection refused"),
|
||||
good_response,
|
||||
]
|
||||
|
||||
result = api._request_with_retry("GET", "https://api.example.com/test")
|
||||
|
||||
assert result == good_response
|
||||
assert mock_request.call_count == 2
|
||||
|
||||
|
||||
class TestAudioShakeUploadRetry:
|
||||
"""Tests for upload_file using retry."""
|
||||
|
||||
@patch("karaoke_gen.lyrics_transcriber.transcribers.audioshake.requests.post")
|
||||
def test_upload_file_retries_on_502(self, mock_post, tmp_path):
|
||||
config = AudioShakeConfig(api_token="test-token")
|
||||
logger = MagicMock()
|
||||
api = AudioShakeAPI(config, logger)
|
||||
|
||||
# Create a test file with known content
|
||||
test_file = tmp_path / "test.flac"
|
||||
test_file.write_bytes(b"fake audio data")
|
||||
|
||||
# First call: 502
|
||||
bad_response = MagicMock()
|
||||
bad_response.status_code = 502
|
||||
bad_response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=bad_response)
|
||||
|
||||
# Second call: success
|
||||
good_response = MagicMock()
|
||||
good_response.raise_for_status = MagicMock()
|
||||
good_response.json.return_value = {"id": "asset-123"}
|
||||
good_response.status_code = 200
|
||||
good_response.text = '{"id": "asset-123"}'
|
||||
|
||||
mock_post.side_effect = [bad_response, good_response]
|
||||
|
||||
asset_id = api.upload_file(str(test_file))
|
||||
|
||||
assert asset_id == "asset-123"
|
||||
assert mock_post.call_count == 2
|
||||
|
||||
# Verify file was re-opened on each attempt (both calls got file data)
|
||||
for call_args in mock_post.call_args_list:
|
||||
files_arg = call_args[1].get("files") or call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("files")
|
||||
assert files_arg is not None, "Each retry should pass files"
|
||||
@@ -6,6 +6,8 @@ Tests that require full backend imports are in the emulator integration tests.
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
|
||||
class TestJobStatusTransitionsForReview:
|
||||
@@ -277,6 +279,93 @@ class TestReviewResubmissionClearsWorkerProgress:
|
||||
assert callable(getattr(jm, 'delete_state_data_keys'))
|
||||
|
||||
|
||||
class TestCompleteReviewZeroSegmentValidation:
|
||||
"""Tests for the 0-segment lyrics guard in complete_review endpoint.
|
||||
|
||||
These test the actual HTTP endpoint, not just the validation logic.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def _patched_client(self):
|
||||
"""Create TestClient with mocked services for complete_review."""
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.transition_to_state.return_value = True
|
||||
mock_worker_service = MagicMock()
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = "googleapis.com"
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager", mock_job_manager), \
|
||||
patch("backend.api.routes.jobs.worker_service", mock_worker_service), \
|
||||
patch("backend.services.firestore_service.firestore"), \
|
||||
patch("backend.services.storage_service.storage"), \
|
||||
patch("google.auth.default", return_value=(mock_creds, "test-project")):
|
||||
from backend.main import app
|
||||
from fastapi.testclient import TestClient
|
||||
yield TestClient(app), mock_job_manager
|
||||
|
||||
def _make_job(self, state_data=None):
|
||||
from backend.models.job import Job, JobStatus
|
||||
return Job(
|
||||
job_id="job-zero-seg",
|
||||
status=JobStatus.AWAITING_REVIEW,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
state_data=state_data,
|
||||
)
|
||||
|
||||
def test_endpoint_rejects_zero_segment_lyrics(self, _patched_client):
|
||||
"""POST complete-review returns 400 when lyrics pipeline produced 0 segments."""
|
||||
client, mock_jm = _patched_client
|
||||
job = self._make_job(state_data={
|
||||
'lyrics_metadata': {'segment_count': 0, 'ready_for_review': True},
|
||||
'corrected_lyrics': {'corrected_segments': [], 'original_segments': []},
|
||||
})
|
||||
mock_jm.get_job.return_value = job
|
||||
|
||||
response = client.post(
|
||||
"/api/jobs/job-zero-seg/complete-review",
|
||||
headers={"Authorization": "Bearer test-admin-token"},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "no lyrics were detected" in response.json()["detail"]
|
||||
mock_jm.transition_to_state.assert_not_called()
|
||||
|
||||
def test_endpoint_allows_nonzero_segment_lyrics(self, _patched_client):
|
||||
"""POST complete-review succeeds when lyrics have segments."""
|
||||
client, mock_jm = _patched_client
|
||||
job = self._make_job(state_data={
|
||||
'lyrics_metadata': {'segment_count': 42},
|
||||
'corrected_lyrics': {'corrected_segments': [{'text': 'hello'}] * 42},
|
||||
})
|
||||
mock_jm.get_job.return_value = job
|
||||
|
||||
response = client.post(
|
||||
"/api/jobs/job-zero-seg/complete-review",
|
||||
headers={"Authorization": "Bearer test-admin-token"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_jm.transition_to_state.assert_called_once()
|
||||
|
||||
def test_endpoint_allows_job_without_lyrics_metadata(self, _patched_client):
|
||||
"""POST complete-review succeeds for jobs without lyrics_metadata (e.g. finalise-only)."""
|
||||
client, mock_jm = _patched_client
|
||||
job = self._make_job(state_data={}) # No lyrics pipeline ran
|
||||
mock_jm.get_job.return_value = job
|
||||
|
||||
response = client.post(
|
||||
"/api/jobs/job-zero-seg/complete-review",
|
||||
headers={"Authorization": "Bearer test-admin-token"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestPreviewStyleLoading:
|
||||
"""Tests for the unified style loader used in preview video generation.
|
||||
|
||||
|
||||
@@ -399,8 +399,8 @@ class TestPackagingValidation:
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cdg_validation_fails_when_enabled_but_not_produced(self):
|
||||
"""Test that orchestrator raises when CDG enabled but not produced."""
|
||||
async def test_cdg_validation_fails_when_enabled_but_not_produced_and_lrc_has_content(self):
|
||||
"""Test that orchestrator raises when CDG enabled but not produced and LRC has lyrics."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
@@ -417,6 +417,7 @@ class TestPackagingValidation:
|
||||
pass # CDG generation "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=True), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
|
||||
|
||||
result = await orchestrator.run()
|
||||
@@ -426,8 +427,37 @@ class TestPackagingValidation:
|
||||
mock_encoding.assert_not_called() # Should not reach encoding
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_txt_validation_fails_when_enabled_but_not_produced(self):
|
||||
"""Test that orchestrator raises when TXT enabled but not produced."""
|
||||
async def test_cdg_skipped_gracefully_when_lrc_has_no_content(self):
|
||||
"""Test that CDG failure is a warning (not error) when LRC has no lyrics content."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=True,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
async def mock_packaging():
|
||||
pass # CDG generation "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=False), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \
|
||||
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock):
|
||||
|
||||
result = await orchestrator.run()
|
||||
|
||||
assert result.success is True # Pipeline continues
|
||||
mock_encoding.assert_called_once() # Encoding still runs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_txt_validation_fails_when_enabled_but_not_produced_and_lrc_has_content(self):
|
||||
"""Test that orchestrator raises when TXT enabled but not produced and LRC has lyrics."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
@@ -443,6 +473,7 @@ class TestPackagingValidation:
|
||||
pass # TXT generation "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=True), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
|
||||
|
||||
result = await orchestrator.run()
|
||||
@@ -506,7 +537,7 @@ class TestPackagingValidation:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_both_cdg_and_txt_validation_fail_together(self):
|
||||
"""Test that orchestrator catches CDG failure when both CDG and TXT enabled."""
|
||||
"""Test that orchestrator catches CDG failure when both CDG and TXT enabled and LRC has content."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
@@ -523,6 +554,7 @@ class TestPackagingValidation:
|
||||
pass # Both CDG and TXT "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_lrc_has_lyrics_content", return_value=True), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
|
||||
|
||||
result = await orchestrator.run()
|
||||
@@ -1709,3 +1741,57 @@ class TestDistributionWarnings:
|
||||
|
||||
assert len(orchestrator.result.distribution_warnings) == 1
|
||||
assert "Dropbox upload failed" in orchestrator.result.distribution_warnings[0]
|
||||
|
||||
|
||||
class TestLrcHasLyricsContent:
|
||||
"""Tests for _lrc_has_lyrics_content helper."""
|
||||
|
||||
def _make_orchestrator(self, lrc_file_path=None):
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
lrc_file_path=lrc_file_path,
|
||||
)
|
||||
return VideoWorkerOrchestrator(config)
|
||||
|
||||
def test_returns_false_when_no_lrc_path(self):
|
||||
orchestrator = self._make_orchestrator(lrc_file_path=None)
|
||||
assert orchestrator._lrc_has_lyrics_content() is False
|
||||
|
||||
def test_returns_false_when_lrc_file_missing(self):
|
||||
orchestrator = self._make_orchestrator(lrc_file_path="/nonexistent/path.lrc")
|
||||
assert orchestrator._lrc_has_lyrics_content() is False
|
||||
|
||||
def test_returns_false_for_empty_lrc(self, tmp_path):
|
||||
lrc_file = tmp_path / "empty.lrc"
|
||||
lrc_file.write_text("[re:MidiCo]\n")
|
||||
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
|
||||
assert orchestrator._lrc_has_lyrics_content() is False
|
||||
|
||||
def test_returns_true_for_lrc_with_lyrics(self, tmp_path):
|
||||
lrc_file = tmp_path / "valid.lrc"
|
||||
lrc_file.write_text("[re:MidiCo]\n[00:12.345]Hello world\n[00:15.678]Second line\n")
|
||||
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
|
||||
assert orchestrator._lrc_has_lyrics_content() is True
|
||||
|
||||
def test_returns_true_for_centisecond_timestamps(self, tmp_path):
|
||||
lrc_file = tmp_path / "centisecond.lrc"
|
||||
lrc_file.write_text("[01:02.34]Lyric line\n")
|
||||
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
|
||||
assert orchestrator._lrc_has_lyrics_content() is True
|
||||
|
||||
def test_returns_true_for_no_decimal_timestamps(self, tmp_path):
|
||||
lrc_file = tmp_path / "nodecimal.lrc"
|
||||
lrc_file.write_text("[01:30]Lyric line\n")
|
||||
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
|
||||
assert orchestrator._lrc_has_lyrics_content() is True
|
||||
|
||||
def test_returns_false_for_metadata_only_lrc(self, tmp_path):
|
||||
lrc_file = tmp_path / "meta.lrc"
|
||||
lrc_file.write_text("[ti:Song Title]\n[ar:Artist Name]\n[re:MidiCo]\n")
|
||||
orchestrator = self._make_orchestrator(lrc_file_path=str(lrc_file))
|
||||
assert orchestrator._lrc_has_lyrics_content() is False
|
||||
@@ -439,7 +439,7 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}")
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}", exc_info=True)
|
||||
job_manager.mark_job_failed(
|
||||
job_id=job_id,
|
||||
error_message=f"Audio separation failed: {str(e)}",
|
||||
|
||||
@@ -18,6 +18,7 @@ Pipeline stages:
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, Dict, Any, List
|
||||
@@ -234,16 +235,31 @@ class VideoWorkerOrchestrator:
|
||||
await self._run_packaging()
|
||||
|
||||
# Validate required packaging outputs before expensive encoding
|
||||
# If LRC has no lyrics content (e.g. input audio had no vocals),
|
||||
# skip CDG/TXT gracefully instead of failing the entire job
|
||||
lrc_has_content = self._lrc_has_lyrics_content()
|
||||
if self.config.enable_cdg and not self.result.final_karaoke_cdg_zip:
|
||||
raise RuntimeError(
|
||||
"CDG generation was enabled but failed. "
|
||||
"Check worker logs for CDG-related errors."
|
||||
)
|
||||
if lrc_has_content:
|
||||
raise RuntimeError(
|
||||
"CDG generation was enabled but failed. "
|
||||
"Check worker logs for CDG-related errors."
|
||||
)
|
||||
else:
|
||||
self.job_log.warning(
|
||||
"CDG generation skipped: LRC file has no lyrics content. "
|
||||
"Video will be generated without CDG package."
|
||||
)
|
||||
if self.config.enable_txt and not self.result.final_karaoke_txt_zip:
|
||||
raise RuntimeError(
|
||||
"TXT generation was enabled but failed. "
|
||||
"Check worker logs for TXT-related errors."
|
||||
)
|
||||
if lrc_has_content:
|
||||
raise RuntimeError(
|
||||
"TXT generation was enabled but failed. "
|
||||
"Check worker logs for TXT-related errors."
|
||||
)
|
||||
else:
|
||||
self.job_log.warning(
|
||||
"TXT generation skipped: LRC file has no lyrics content. "
|
||||
"Video will be generated without TXT package."
|
||||
)
|
||||
|
||||
# Stage 2: Encoding
|
||||
await self._run_encoding()
|
||||
@@ -276,6 +292,18 @@ class VideoWorkerOrchestrator:
|
||||
|
||||
return self.result
|
||||
|
||||
def _lrc_has_lyrics_content(self) -> bool:
|
||||
"""Check if the LRC file has actual lyrics content (timestamp lines), not just metadata."""
|
||||
if not self.config.lrc_file_path or not os.path.isfile(self.config.lrc_file_path):
|
||||
return False
|
||||
try:
|
||||
with open(self.config.lrc_file_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
# Match LRC timestamp lines: [MM:SS.xx] or [MM:SS.xxx] or [M:SS.xx]
|
||||
return bool(re.search(r"\[\d{1,2}:\d{2}(?:\.\d{1,3})?\]", content))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def _run_packaging(self):
|
||||
"""Run the packaging stage (CDG/TXT generation)."""
|
||||
self.job_log.info("Starting packaging stage (CDG/TXT)")
|
||||
|
||||
@@ -20,6 +20,12 @@ Autocomplete on the artist/title input fields confused users — it wasn't clear
|
||||
### Fuzzy Search: Artist Tiebreaker for Title-Based Fallback (Mar 2026)
|
||||
When fuzzy-matching by title returns multiple artists with the same song name (e.g. "Bruises" by Lewis Capaldi vs Fox Stevenson), use the user's garbled artist input as a tiebreaker — `stringSimilarity(userInput, track.artist_name) * 0.2` as a bonus score. Also: Spotify's `q` parameter searches track names, not artist names, so `searchCatalogTracks(artist, artist, 20)` (artist as both query and filter) returns nothing. Use `searchCatalogTracks(artist, undefined, 20)` to get the artist's tracks via general search.
|
||||
|
||||
### Pipeline Robustness Patterns (Mar 2026)
|
||||
Three patterns from investigating production job failures where errors cascaded far from their root cause:
|
||||
1. **Validate every stage's outputs, not just some** — stage 1 of audio separation had validation for expected stems, stage 2 didn't. The missing validation let a transient Modal API failure propagate through 3 more functions before crashing with a confusing NoneType error.
|
||||
2. **Block impossible downstream work early** — AudioShake returning 0 lyrics segments was treated as "success", letting the user submit an empty review that only failed at CDG generation. Validate at the boundary: if transcription returns 0 segments, fail with a user-friendly error, not 4 steps later.
|
||||
3. **File handles in retry loops** — when wrapping HTTP calls with retry (e.g. tenacity), ensure file handles are re-opened on each attempt. A `with open()` wrapping a retried `requests.post(files=...)` exhausts the file handle on the first attempt; subsequent retries send empty data.
|
||||
|
||||
---
|
||||
|
||||
## Infrastructure
|
||||
|
||||
@@ -136,3 +136,33 @@ python scripts/backfill_gdrive_uploads.py --all-missing
|
||||
# Specific jobs
|
||||
python scripts/backfill_gdrive_uploads.py --job-ids JOB1,JOB2
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Job failed: "CDG generation was enabled but failed"
|
||||
|
||||
**Cause:** The LRC file has no lyrics content (just metadata like `[re:MidiCo]`). This happens when AudioShake returns 0 segments — typically because the input audio has no vocals (e.g. user uploaded a karaoke track).
|
||||
|
||||
**Fix (v0.135+):** The video orchestrator now skips CDG/TXT gracefully when the LRC has no content. The `complete_review` endpoint also blocks 0-segment submissions with a user-friendly error, and the frontend shows a warning with guidance.
|
||||
|
||||
**For older jobs:** Retry won't help unless the user provides different audio or pastes lyrics manually via "Replace All".
|
||||
|
||||
---
|
||||
|
||||
## Job failed: "Audio separation failed: expected str, bytes or os.PathLike object, not NoneType"
|
||||
|
||||
**Cause:** Modal API intermittently returned fewer stems than expected from stage 2 (backing vocals) separation. Missing stems caused NoneType crashes in downstream processing.
|
||||
|
||||
**Fix (v0.135+):** Stage 2 now validates that all expected stems are present (matching existing stage 1 validation). Defensive null checks prevent confusing TypeError messages. Full tracebacks are logged with `exc_info=True`.
|
||||
|
||||
**Recovery:** Retry the job — the failure is transient (Modal API intermittent). Long-term fix: migrate audio separation from Modal to GCP (see `docs/archive/2026-03-21-modal-to-gcp-migration-plan.md`).
|
||||
|
||||
---
|
||||
|
||||
## Job failed: "Lyrics transcription failed: 502 Server Error: Bad Gateway for url: https://api.audioshake.ai/tasks"
|
||||
|
||||
**Cause:** AudioShake API returned a transient 5xx error.
|
||||
|
||||
**Fix (v0.135+):** AudioShake API calls now retry up to 5 times with exponential backoff (60s base, 3x multiplier, ~40 minutes total spread). Retries on 5xx, 429, connection errors, and timeouts.
|
||||
|
||||
**For older jobs:** Simply retry the job via admin dashboard.
|
||||
@@ -0,0 +1,121 @@
|
||||
# Modal to GCP Audio Separation Migration Plan
|
||||
|
||||
**Date:** 2026-03-21
|
||||
**Status:** Planning seed (not yet started)
|
||||
**Priority:** Medium - reduces external dependency risk and cost
|
||||
|
||||
## Why Migrate
|
||||
|
||||
1. **Reliability** - Modal API intermittently fails to return stem files ("no files were downloaded" errors), causing job failures that require manual retries.
|
||||
2. **Latency** - Round-trip to Modal (upload audio, wait for GPU separation, download stems) adds network overhead. Local GCP processing eliminates cross-provider data transfer.
|
||||
3. **Cost** - Modal charges for GPU compute on their platform. GCP GPU VMs or spot instances may be cheaper, especially with sustained use discounts.
|
||||
4. **Dependency reduction** - Modal is the only third-party compute dependency. Removing it means the entire pipeline runs within GCP.
|
||||
|
||||
## Current State
|
||||
|
||||
### How Modal Is Used
|
||||
|
||||
The audio worker (`backend/workers/audio_worker.py`) runs on Cloud Run and calls Modal via the `audio-separator[remote]` Python package:
|
||||
|
||||
1. Cloud Run audio worker downloads the source audio from GCS
|
||||
2. `AudioProcessor.process_audio_separation()` checks `AUDIO_SEPARATOR_API_URL` env var
|
||||
3. Calls `AudioSeparatorAPIClient.separate_audio()` which POSTs the audio file to Modal's HTTP API
|
||||
4. Modal runs GPU separation and returns stem files
|
||||
5. Worker downloads stems, post-processes, and uploads to GCS
|
||||
|
||||
**API endpoint:** `https://nomadkaraoke--audio-separator-api.modal.run`
|
||||
**Configured in:** `infrastructure/modules/cloud_run.py` (env var on Cloud Run service)
|
||||
|
||||
### Models Used
|
||||
|
||||
- **Stage 1 (Clean instrumental):** `model_bs_roformer_ep_317_sdr_12.9755.ckpt` (BS-Roformer)
|
||||
- **Stage 1 (Other stems):** `htdemucs_6s.yaml` (Demucs 6-stem)
|
||||
- **Stage 2 (Backing vocals):** `mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt` (Mel-Band Roformer)
|
||||
|
||||
### Processing Time
|
||||
|
||||
- Stage 1: ~3-5 minutes
|
||||
- Stage 2: ~2-3 minutes
|
||||
- Total audio worker timeout: 30 minutes (includes download/upload overhead)
|
||||
|
||||
### Existing GCP Infrastructure
|
||||
|
||||
- **Encoding worker VM** (`c4d-highcpu-32`, `us-central1-c`) - CPU-only, handles video encoding
|
||||
- Packer-based image build, GCS-based deployment (wheel + startup.sh)
|
||||
- Pulumi IaC in `infrastructure/`
|
||||
- The `audio-separator` package already supports local processing (just needs GPU + models)
|
||||
|
||||
## Target State
|
||||
|
||||
Run audio separation on a GCP GPU VM instead of Modal. The `audio-separator` package already supports both local and remote modes - we just need to point it at our own GPU endpoint (or run locally on the GPU VM).
|
||||
|
||||
## Options
|
||||
|
||||
### Option A: Dedicated GCE GPU VM (Recommended)
|
||||
|
||||
Similar pattern to the encoding worker. A persistent VM with GPU that runs the audio-separator API.
|
||||
|
||||
- **Pros:** Simple, mirrors existing encoding worker pattern, models stay loaded in memory, fast cold start after first load
|
||||
- **Cons:** Pays for GPU even when idle, need to handle scaling
|
||||
- **GPU:** NVIDIA T4 (~$0.35/hr) or L4 (~$0.70/hr) sufficient for these models
|
||||
- **Cost estimate:** T4 spot instance ~$100-150/month if running 24/7 (vs Modal pay-per-use)
|
||||
|
||||
### Option B: GCE Spot GPU VM (Start/Stop)
|
||||
|
||||
Same as A but stop the VM when not processing, start on demand.
|
||||
|
||||
- **Pros:** Only pay when processing, cheapest option
|
||||
- **Cons:** Cold start (VM boot + model loading ~2-5 min), spot can be preempted, more complex orchestration
|
||||
- **Best if:** Job volume is low/bursty
|
||||
|
||||
### Option C: Cloud Run Jobs with GPU
|
||||
|
||||
Cloud Run now supports GPU (L4) for jobs.
|
||||
|
||||
- **Pros:** Serverless, auto-scaling, no idle cost, Cloud Run already used for other workers
|
||||
- **Cons:** GPU availability can be limited, cold start on each job, 1-hour max timeout, model download on each cold start unless cached
|
||||
- **Cost estimate:** Pay only for job duration, but model loading adds per-job overhead
|
||||
|
||||
### Option D: Run Locally on Audio Worker (Cloud Run)
|
||||
|
||||
Run separation directly in the Cloud Run audio worker with a GPU.
|
||||
|
||||
- **Pros:** Simplest code change (just remove AUDIO_SEPARATOR_API_URL, models load locally)
|
||||
- **Cons:** Cloud Run GPU instances are expensive if kept warm, model loading on cold start
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Option A (dedicated GCE GPU VM)** for initial migration, with a path to Option B if cost is a concern.
|
||||
|
||||
Rationale:
|
||||
- Mirrors the proven encoding worker pattern (Packer image, GCS deploy, Pulumi)
|
||||
- Models stay loaded = fast separation with no cold start penalty
|
||||
- Simplest to implement and debug
|
||||
- Can add start/stop automation later (Option B) if idle cost is too high
|
||||
|
||||
## Key Considerations
|
||||
|
||||
- **Model storage:** ~2-3 GB total for the three models. Bake into Packer image or download on first start to a persistent disk.
|
||||
- **GPU memory:** These models fit comfortably in a T4 (16 GB VRAM). L4 (24 GB) provides headroom.
|
||||
- **API compatibility:** Can reuse the existing `AudioSeparatorAPIClient` by running the audio-separator API server on the GCE VM and updating `AUDIO_SEPARATOR_API_URL` to point to it.
|
||||
- **Fallback:** Keep Modal config as a fallback during migration. The `AUDIO_SEPARATOR_API_URL` env var swap makes this easy.
|
||||
- **Networking:** VM needs to accept requests from Cloud Run audio worker. Use internal networking or IAP + auth token (same pattern as encoding worker).
|
||||
|
||||
## Migration Steps (High Level)
|
||||
|
||||
1. **Spike:** Run audio-separator API server locally with GPU, verify it produces identical output to Modal
|
||||
2. **Infrastructure:** Create Pulumi resources for GPU VM (instance, disk, firewall, service account) - copy encoding worker pattern
|
||||
3. **Packer image:** Build image with CUDA, Python, audio-separator, pre-downloaded models
|
||||
4. **Deploy & test:** Stand up VM, point a test job at it, compare output quality and timing
|
||||
5. **Cutover:** Update `AUDIO_SEPARATOR_API_URL` in Cloud Run config to point to GCE VM
|
||||
6. **Monitor:** Watch for errors/quality issues for a week
|
||||
7. **Decommission:** Remove Modal deployment, remove `audio-separator-api-url` secret, clean up Modal account
|
||||
|
||||
## Files to Modify (When Implementing)
|
||||
|
||||
- `infrastructure/compute/` - New GPU VM module (similar to `encoding_worker_vm.py`)
|
||||
- `infrastructure/packer/` - New Packer template for GPU image
|
||||
- `infrastructure/modules/cloud_run.py` - Update `AUDIO_SEPARATOR_API_URL` value
|
||||
- `infrastructure/__main__.py` - Wire up new resources
|
||||
- `infrastructure/config.py` - Add GPU VM machine type/zone constants
|
||||
- No backend code changes needed if we keep the HTTP API interface
|
||||
@@ -0,0 +1,102 @@
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import ReviewChangesModal from '../modals/ReviewChangesModal'
|
||||
import type { CorrectionData } from '@/lib/lyrics-review/types'
|
||||
|
||||
// Mock PreviewVideoSection since it has complex dependencies
|
||||
jest.mock('../PreviewVideoSection', () => ({
|
||||
__esModule: true,
|
||||
default: () => <div data-testid="preview-video">Preview</div>,
|
||||
}))
|
||||
|
||||
function makeData(overrides: Partial<CorrectionData> = {}): CorrectionData {
|
||||
return {
|
||||
original_segments: [],
|
||||
reference_lyrics: {},
|
||||
anchor_sequences: [],
|
||||
gap_sequences: [],
|
||||
resized_segments: [],
|
||||
corrections_made: 0,
|
||||
confidence: 1,
|
||||
corrections: [],
|
||||
corrected_segments: [],
|
||||
metadata: {},
|
||||
...overrides,
|
||||
} as CorrectionData
|
||||
}
|
||||
|
||||
describe('ReviewChangesModal', () => {
|
||||
const defaultProps = {
|
||||
open: true,
|
||||
onClose: jest.fn(),
|
||||
onSubmit: jest.fn(),
|
||||
}
|
||||
|
||||
it('disables submit button when there are no lyrics (0 segments)', () => {
|
||||
render(
|
||||
<ReviewChangesModal
|
||||
{...defaultProps}
|
||||
data={makeData({ corrected_segments: [] })}
|
||||
/>
|
||||
)
|
||||
|
||||
const submitButton = screen.getByRole('button', { name: /proceed to instrumental/i })
|
||||
expect(submitButton).toBeDisabled()
|
||||
})
|
||||
|
||||
it('shows warning message when there are no lyrics', () => {
|
||||
render(
|
||||
<ReviewChangesModal
|
||||
{...defaultProps}
|
||||
data={makeData({ corrected_segments: [] })}
|
||||
/>
|
||||
)
|
||||
|
||||
expect(screen.getByText('No lyrics detected')).toBeInTheDocument()
|
||||
expect(screen.getByText(/no lyrics were found in the audio/i)).toBeInTheDocument()
|
||||
expect(screen.getByText(/replace all/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('enables submit button when segments exist', () => {
|
||||
const segments = [{ text: 'Hello', words: [], start_time: 0, end_time: 1 }]
|
||||
render(
|
||||
<ReviewChangesModal
|
||||
{...defaultProps}
|
||||
data={makeData({ corrected_segments: segments as any })}
|
||||
/>
|
||||
)
|
||||
|
||||
const submitButton = screen.getByRole('button', { name: /proceed to instrumental/i })
|
||||
expect(submitButton).not.toBeDisabled()
|
||||
})
|
||||
|
||||
it('does not show warning when segments exist', () => {
|
||||
const segments = [{ text: 'Hello', words: [], start_time: 0, end_time: 1 }]
|
||||
render(
|
||||
<ReviewChangesModal
|
||||
{...defaultProps}
|
||||
data={makeData({ corrected_segments: segments as any })}
|
||||
/>
|
||||
)
|
||||
|
||||
expect(screen.queryByText('No lyrics detected')).not.toBeInTheDocument()
|
||||
expect(screen.getByText(/total segments: 1/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('calls onSubmit when button clicked with valid segments', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSubmit = jest.fn()
|
||||
const segments = [{ text: 'Hello', words: [], start_time: 0, end_time: 1 }]
|
||||
|
||||
render(
|
||||
<ReviewChangesModal
|
||||
{...defaultProps}
|
||||
onSubmit={onSubmit}
|
||||
data={makeData({ corrected_segments: segments as any })}
|
||||
/>
|
||||
)
|
||||
|
||||
await user.click(screen.getByRole('button', { name: /proceed to instrumental/i }))
|
||||
expect(onSubmit).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -3,7 +3,7 @@
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { CorrectionData } from '@/lib/lyrics-review/types'
|
||||
import { ArrowLeft, ArrowRight, Loader2 } from 'lucide-react'
|
||||
import { AlertTriangle, ArrowLeft, ArrowRight, Loader2 } from 'lucide-react'
|
||||
import PreviewVideoSection from '../PreviewVideoSection'
|
||||
|
||||
interface ApiClient {
|
||||
@@ -36,6 +36,7 @@ export default function ReviewChangesModal({
|
||||
}: ReviewChangesModalProps) {
|
||||
const corrections = data.corrections || []
|
||||
const totalSegments = data.corrected_segments?.length || 0
|
||||
const hasNoLyrics = totalSegments === 0
|
||||
|
||||
// Check if there are manual corrections (user-made changes)
|
||||
const hasManualCorrections = corrections.some(c => c.handler === 'ManualCorrector' || c.handler === 'UserEdit')
|
||||
@@ -60,15 +61,35 @@ export default function ReviewChangesModal({
|
||||
timingOffsetMs={timingOffsetMs}
|
||||
/>
|
||||
|
||||
{/* No lyrics warning */}
|
||||
{hasNoLyrics && (
|
||||
<div className="flex items-start gap-3 rounded-lg border border-yellow-500/50 bg-yellow-500/10 p-4">
|
||||
<AlertTriangle className="h-5 w-5 text-yellow-500 mt-0.5 shrink-0" />
|
||||
<div className="text-sm space-y-2">
|
||||
<p className="font-medium text-yellow-500">No lyrics detected</p>
|
||||
<p className="text-muted-foreground">
|
||||
No lyrics were found in the audio. This usually happens when the input audio has no vocals
|
||||
(e.g. a karaoke track or instrumental).
|
||||
</p>
|
||||
<p className="text-muted-foreground">
|
||||
You can go back and paste lyrics manually using <strong>Replace All</strong>,
|
||||
or cancel this job and try again with an audio file that contains vocals.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Info text */}
|
||||
<div className="text-sm text-muted-foreground space-y-1">
|
||||
{hasManualCorrections ? (
|
||||
<p>Manual corrections detected. Review the preview to ensure the lyrics are synchronized correctly.</p>
|
||||
) : (
|
||||
<p>No manual corrections detected. If everything looks good in the preview, proceed to select your instrumental.</p>
|
||||
)}
|
||||
<p>Total segments: {totalSegments}</p>
|
||||
</div>
|
||||
{!hasNoLyrics && (
|
||||
<div className="text-sm text-muted-foreground space-y-1">
|
||||
{hasManualCorrections ? (
|
||||
<p>Manual corrections detected. Review the preview to ensure the lyrics are synchronized correctly.</p>
|
||||
) : (
|
||||
<p>No manual corrections detected. If everything looks good in the preview, proceed to select your instrumental.</p>
|
||||
)}
|
||||
<p>Total segments: {totalSegments}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<DialogFooter className="border-t pt-4">
|
||||
<Button variant="ghost" onClick={onClose} disabled={isSubmitting} className="text-primary">
|
||||
@@ -77,7 +98,7 @@ export default function ReviewChangesModal({
|
||||
</Button>
|
||||
<Button
|
||||
onClick={handleSubmit}
|
||||
disabled={isSubmitting}
|
||||
disabled={isSubmitting || hasNoLyrics}
|
||||
className="bg-green-600 hover:bg-green-700"
|
||||
>
|
||||
{isSubmitting ? (
|
||||
|
||||
@@ -395,6 +395,23 @@ class AudioProcessor:
|
||||
backing_vocals_result = self._organize_stage2_remote_results(
|
||||
stage2_result["downloaded_files"], artist_title, stems_dir
|
||||
)
|
||||
|
||||
# Validate stage 2 outputs — each model should produce both lead_vocals and backing_vocals
|
||||
for bv_model, paths in backing_vocals_result.items():
|
||||
missing = []
|
||||
if not paths.get("lead_vocals"):
|
||||
missing.append("lead_vocals")
|
||||
if not paths.get("backing_vocals"):
|
||||
missing.append("backing_vocals")
|
||||
if missing:
|
||||
error_msg = (
|
||||
f"Stage 2 completed but model {bv_model} is missing: {', '.join(missing)}. "
|
||||
f"Got keys: {list(paths.keys())}. Downloaded {len(stage2_result['downloaded_files'])} files. "
|
||||
"This may indicate an intermittent issue with the remote separation API."
|
||||
)
|
||||
self.logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
result["backing_vocals"] = backing_vocals_result
|
||||
else:
|
||||
error_msg = f"Stage 2 backing vocals separation failed: {stage2_result.get('error', 'Unknown error')}"
|
||||
@@ -418,7 +435,7 @@ class AudioProcessor:
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error during remote audio separation: {str(e)}")
|
||||
self.logger.error(f"Error during remote audio separation: {str(e)}", exc_info=True)
|
||||
raise e
|
||||
|
||||
def _organize_stage1_remote_results(self, downloaded_files, artist_title, track_output_dir, stems_dir):
|
||||
@@ -644,7 +661,10 @@ class AudioProcessor:
|
||||
self.logger.info("Generating combined instrumental tracks with backing vocals")
|
||||
result = {}
|
||||
for model, paths in backing_vocals_result.items():
|
||||
backing_vocals_path = paths["backing_vocals"]
|
||||
backing_vocals_path = paths.get("backing_vocals")
|
||||
if not backing_vocals_path:
|
||||
self.logger.error(f"Skipping combined instrumental for model {model}: backing_vocals path is missing")
|
||||
continue
|
||||
combined_path = os.path.join(track_output_dir, f"{artist_title} (Instrumental +BV {model}).{self.lossless_output_format}")
|
||||
|
||||
if not self._file_exists(combined_path):
|
||||
@@ -663,9 +683,16 @@ class AudioProcessor:
|
||||
def _normalize_audio_files(self, separation_result, artist_title, track_output_dir):
|
||||
self.logger.info("Normalizing clean instrumental and combined instrumentals")
|
||||
|
||||
clean_instrumental_path = separation_result.get("clean_instrumental", {}).get("instrumental")
|
||||
if not clean_instrumental_path:
|
||||
raise ValueError(
|
||||
"Cannot normalize audio: clean instrumental path is missing. "
|
||||
"This indicates the audio separation did not produce the expected instrumental stem."
|
||||
)
|
||||
|
||||
files_to_normalize = [
|
||||
("clean_instrumental", separation_result["clean_instrumental"]["instrumental"]),
|
||||
] + [("combined_instrumentals", path) for path in separation_result["combined_instrumentals"].values()]
|
||||
("clean_instrumental", clean_instrumental_path),
|
||||
] + [("combined_instrumentals", path) for path in separation_result.get("combined_instrumentals", {}).values()]
|
||||
|
||||
for key, file_path in files_to_normalize:
|
||||
if self._file_exists(file_path):
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
import requests
|
||||
import time
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Dict, Optional, Any, Union, Tuple
|
||||
from pathlib import Path
|
||||
|
||||
from tenacity import (
|
||||
retry,
|
||||
stop_after_attempt,
|
||||
wait_exponential,
|
||||
retry_if_exception,
|
||||
before_sleep_log,
|
||||
)
|
||||
from pydub import AudioSegment
|
||||
from karaoke_gen.lyrics_transcriber.types import TranscriptionData, LyricsSegment, Word
|
||||
from karaoke_gen.lyrics_transcriber.transcribers.base_transcriber import BaseTranscriber, TranscriptionError
|
||||
@@ -107,6 +116,30 @@ class AudioShakeConfig:
|
||||
timeout_minutes: int = 20 # Added timeout configuration
|
||||
|
||||
|
||||
def _is_retryable_error(exception: BaseException) -> bool:
|
||||
"""Check if an exception is retryable (server errors, rate limits, network issues)."""
|
||||
if isinstance(exception, requests.exceptions.ConnectionError):
|
||||
return True
|
||||
if isinstance(exception, requests.exceptions.Timeout):
|
||||
return True
|
||||
if isinstance(exception, requests.exceptions.HTTPError):
|
||||
status = exception.response.status_code
|
||||
return status >= 500 or status == 429
|
||||
return False
|
||||
|
||||
|
||||
# Retry decorator for AudioShake API calls.
|
||||
# 5 attempts with exponential backoff: ~60s, ~180s, ~540s, ~1620s (capped at 1800s)
|
||||
# Total spread: ~40 minutes
|
||||
_audioshake_retry = retry(
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_exponential(multiplier=60, max=1800),
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
before_sleep=before_sleep_log(logging.getLogger(__name__), logging.WARNING),
|
||||
reraise=True,
|
||||
)
|
||||
|
||||
|
||||
class AudioShakeAPI:
|
||||
"""Handles direct API interactions with AudioShake."""
|
||||
|
||||
@@ -124,18 +157,31 @@ class AudioShakeAPI:
|
||||
self._validate_config() # Validate before making any API calls
|
||||
return {"x-api-key": self.config.api_token, "Content-Type": "application/json"}
|
||||
|
||||
@_audioshake_retry
|
||||
def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response:
|
||||
"""Make an HTTP request with retry on transient failures."""
|
||||
response = requests.request(method, url, **kwargs)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
@_audioshake_retry
|
||||
def _upload_file_with_retry(self, url: str, filepath: str) -> requests.Response:
|
||||
"""Upload a file with retry, re-opening the file on each attempt to avoid stale handles."""
|
||||
with open(filepath, "rb") as file:
|
||||
files = {"file": (os.path.basename(filepath), file)}
|
||||
response = requests.post(url, headers={"x-api-key": self.config.api_token}, files=files)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
def upload_file(self, filepath: str) -> str:
|
||||
"""Upload audio file via Assets API and return asset ID."""
|
||||
self.logger.info(f"Uploading {filepath} to AudioShake")
|
||||
self._validate_config() # Validate before making API call
|
||||
|
||||
url = f"{self.config.base_url}/assets"
|
||||
with open(filepath, "rb") as file:
|
||||
files = {"file": (os.path.basename(filepath), file)}
|
||||
response = requests.post(url, headers={"x-api-key": self.config.api_token}, files=files)
|
||||
response = self._upload_file_with_retry(url, filepath)
|
||||
|
||||
self.logger.debug(f"Upload response: {response.status_code} - {response.text}")
|
||||
response.raise_for_status()
|
||||
return response.json()["id"]
|
||||
|
||||
def create_task(self, asset_id: str) -> str:
|
||||
@@ -153,8 +199,7 @@ class AudioShakeAPI:
|
||||
}
|
||||
],
|
||||
}
|
||||
response = requests.post(url, headers=self._get_headers(), json=data)
|
||||
response.raise_for_status()
|
||||
response = self._request_with_retry("POST", url, headers=self._get_headers(), json=data)
|
||||
return response.json()["id"]
|
||||
|
||||
def wait_for_task_result(self, task_id: str) -> Dict[str, Any]:
|
||||
@@ -186,17 +231,16 @@ class AudioShakeAPI:
|
||||
last_status_log = current_time
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=self._get_headers())
|
||||
response.raise_for_status()
|
||||
response = self._request_with_retry("GET", url, headers=self._get_headers())
|
||||
tasks_list = response.json()
|
||||
|
||||
|
||||
# Find our specific task in the list
|
||||
task_data = None
|
||||
for task in tasks_list:
|
||||
if task.get("id") == task_id:
|
||||
task_data = task
|
||||
break
|
||||
|
||||
|
||||
if not task_data:
|
||||
# Task not found in list yet
|
||||
if initial_retry_count < max_initial_retries:
|
||||
@@ -206,7 +250,7 @@ class AudioShakeAPI:
|
||||
continue
|
||||
else:
|
||||
raise TranscriptionError(f"Task {task_id} not found in task list after {max_initial_retries} retries")
|
||||
|
||||
|
||||
# Log the full response for debugging
|
||||
self.logger.debug(f"Task status response: {task_data}")
|
||||
|
||||
@@ -214,29 +258,29 @@ class AudioShakeAPI:
|
||||
targets = task_data.get("targets", [])
|
||||
if not targets:
|
||||
raise TranscriptionError("No targets found in task response")
|
||||
|
||||
|
||||
# Check if all targets are completed or if any failed
|
||||
all_completed = True
|
||||
for target in targets:
|
||||
target_status = target.get("status")
|
||||
target_model = target.get("model")
|
||||
self.logger.debug(f"Target {target_model} status: {target_status}")
|
||||
|
||||
|
||||
if target_status == "failed":
|
||||
error_msg = target.get("error", "Unknown error")
|
||||
raise TranscriptionError(f"Target {target_model} failed: {error_msg}")
|
||||
elif target_status != "completed":
|
||||
all_completed = False
|
||||
|
||||
|
||||
if all_completed:
|
||||
self.logger.info("All targets completed successfully")
|
||||
return task_data
|
||||
|
||||
# Reset retry count on successful response
|
||||
initial_retry_count = 0
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
raise
|
||||
|
||||
except (TranscriptionError, requests.exceptions.HTTPError):
|
||||
raise # Non-retryable or already exhausted retries — propagate
|
||||
|
||||
time.sleep(30) # Wait before next poll
|
||||
|
||||
@@ -332,8 +376,7 @@ class AudioShakeTranscriber(BaseTranscriber):
|
||||
raise TranscriptionError("Output link not found in alignment target")
|
||||
|
||||
# Fetch transcription data
|
||||
response = requests.get(output_url)
|
||||
response.raise_for_status()
|
||||
response = self.api._request_with_retry("GET", output_url)
|
||||
|
||||
# Return combined raw data
|
||||
raw_data = {"task_data": task_data, "transcription": response.json()}
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.146.0"
|
||||
version = "0.147.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -43,21 +43,32 @@ def mock_email_service():
|
||||
|
||||
@pytest.fixture
|
||||
def client(admin_auth_result, mock_user_service, mock_email_service):
|
||||
"""Test client with mocked dependencies."""
|
||||
from fastapi import FastAPI
|
||||
from backend.api.routes.users import router
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.user_service import get_user_service
|
||||
from backend.services.email_service import get_email_service
|
||||
"""Test client with mocked dependencies.
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api")
|
||||
GCP services (Firestore, Storage) are mocked because importing the users
|
||||
router triggers module-level imports (file_upload.py -> JobManager -> FirestoreService)
|
||||
that require GCP credentials not available in CI.
|
||||
"""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = "googleapis.com"
|
||||
|
||||
app.dependency_overrides[require_admin] = lambda: admin_auth_result
|
||||
app.dependency_overrides[get_user_service] = lambda: mock_user_service
|
||||
app.dependency_overrides[get_email_service] = lambda: mock_email_service
|
||||
with patch("backend.services.firestore_service.firestore"), \
|
||||
patch("backend.services.storage_service.storage"), \
|
||||
patch("google.auth.default", return_value=(mock_creds, "test-project")):
|
||||
from fastapi import FastAPI
|
||||
from backend.api.routes.users import router
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.user_service import get_user_service
|
||||
from backend.services.email_service import get_email_service
|
||||
|
||||
return TestClient(app)
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api")
|
||||
|
||||
app.dependency_overrides[require_admin] = lambda: admin_auth_result
|
||||
app.dependency_overrides[get_user_service] = lambda: mock_user_service
|
||||
app.dependency_overrides[get_email_service] = lambda: mock_email_service
|
||||
|
||||
yield TestClient(app)
|
||||
|
||||
|
||||
class TestAddCredits:
|
||||
|
||||
Reference in new issue
Block a user