mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Fix missing CDG ZIP packages: fail-fast validation + remediation (#416)
## Summary
- **Systemic fix**: Orchestrator now validates CDG/TXT outputs after
packaging, before encoding. If `enable_cdg=True` but no CDG ZIP was
produced, the job fails immediately with a clear error instead of
silently completing without the package.
- **Remediation**: Created `scripts/regenerate_cdg.py` to regenerate and
distribute CDG/TXT packages for affected jobs (5b6aba25, 5161b069).
Packages uploaded to GCS, GDrive, and Dropbox.
- **Root cause**: Before b671b83f, `instrumental_selection: custom` had
no case in the orchestrator config builder — fell through to "Backing"
suffix, CDG generator looked for wrong filename, `FileNotFoundError` was
silently caught.
## Changes
| File | Change |
|------|--------|
| `backend/workers/video_worker_orchestrator.py` | Post-packaging
validation + LRC-missing check update |
| `backend/tests/test_video_worker_orchestrator.py` | 8 new tests in
`TestPackagingValidation` |
| `scripts/regenerate_cdg.py` | One-time remediation script (GCS +
Firestore + GDrive + Dropbox) |
| `docs/LESSONS-LEARNED.md` | Pipeline stage validation pattern |
| `docs/TROUBLESHOOTING.md` | CDG missing recovery runbook |
| `docs/README.md` | Updated recent changes |
| `pyproject.toml` | Version 0.119.6 → 0.119.7 |
## Test plan
- [x] 8 new tests in `TestPackagingValidation` (CDG fails, TXT fails,
combined CDG+TXT fails, CDG succeeds, CDG disabled, LRC missing raises
with CDG, LRC missing raises with TXT, LRC missing skips without CDG)
- [x] All 2062+ backend tests pass
- [x] Remediation script run successfully for both affected jobs
- [x] Verified CDG/TXT ZIPs in GCS, Firestore, GDrive, and Dropbox
> **CodeRabbit note**: 1 finding (unreachable `else` branch in
`_run_packaging` LRC check) — kept as defensive code since the test
`test_lrc_missing_skips_when_cdg_disabled` exercises this path directly.
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
028e9f40d4
commit
23a8b0fb98
7 files changed
+678
-9
No files matched your search
@@ -250,8 +250,28 @@ class TestVideoWorkerOrchestratorPackaging:
|
||||
"""Test packaging stage."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_packaging_no_lrc(self):
|
||||
"""Test packaging stage with no LRC file."""
|
||||
async def test_run_packaging_no_lrc_cdg_disabled(self):
|
||||
"""Test packaging stage with no LRC file and CDG disabled skips silently."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=False,
|
||||
lrc_file_path=None,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
# Should not raise, just skip
|
||||
await orchestrator._run_packaging()
|
||||
|
||||
assert orchestrator.result.final_karaoke_cdg_zip is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_packaging_no_lrc_cdg_enabled_raises(self):
|
||||
"""Test packaging stage raises when CDG enabled but no LRC file."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
@@ -264,10 +284,8 @@ class TestVideoWorkerOrchestratorPackaging:
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
# Should not raise, just skip
|
||||
await orchestrator._run_packaging()
|
||||
|
||||
assert orchestrator.result.final_karaoke_cdg_zip is None
|
||||
with pytest.raises(RuntimeError, match="LRC file not available"):
|
||||
await orchestrator._run_packaging()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_packaging_cdg(self):
|
||||
@@ -368,6 +386,209 @@ class TestVideoWorkerOrchestratorPackaging:
|
||||
"countdown_padding_seconds must be passed for CDG timestamp adjustment"
|
||||
|
||||
|
||||
class TestPackagingValidation:
|
||||
"""Test CDG/TXT packaging validation checks.
|
||||
|
||||
These tests verify that the orchestrator fails fast when CDG/TXT
|
||||
generation is enabled but fails, rather than silently proceeding
|
||||
to expensive encoding. This prevents jobs from completing without
|
||||
their requested CDG/TXT packages.
|
||||
|
||||
Added after jobs 5b6aba25 and 5161b069 completed with enable_cdg=True
|
||||
but no CDG ZIP was produced due to a FileNotFoundError being silently caught.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cdg_validation_fails_when_enabled_but_not_produced(self):
|
||||
"""Test that orchestrator raises when CDG enabled but not produced."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=True,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
# Mock _run_packaging to NOT set result.final_karaoke_cdg_zip (simulates failure)
|
||||
async def mock_packaging():
|
||||
pass # CDG generation "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
|
||||
|
||||
result = await orchestrator.run()
|
||||
|
||||
assert result.success is False
|
||||
assert "CDG generation was enabled but failed" in result.error_message
|
||||
mock_encoding.assert_not_called() # Should not reach encoding
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_txt_validation_fails_when_enabled_but_not_produced(self):
|
||||
"""Test that orchestrator raises when TXT enabled but not produced."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_txt=True,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
async def mock_packaging():
|
||||
pass # TXT generation "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
|
||||
|
||||
result = await orchestrator.run()
|
||||
|
||||
assert result.success is False
|
||||
assert "TXT generation was enabled but failed" in result.error_message
|
||||
mock_encoding.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_succeeds_when_cdg_disabled(self):
|
||||
"""Test that pipeline succeeds normally when CDG is not enabled."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=False,
|
||||
enable_txt=False,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_pkg, \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock):
|
||||
|
||||
result = await orchestrator.run()
|
||||
|
||||
assert result.success is True
|
||||
mock_pkg.assert_not_called() # Packaging skipped entirely
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cdg_validation_passes_when_cdg_produced(self):
|
||||
"""Test that validation passes when CDG is successfully produced."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=True,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
async def mock_packaging_success():
|
||||
orchestrator.result.final_karaoke_cdg_zip = "/output/cdg.zip"
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging_success), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \
|
||||
patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock):
|
||||
|
||||
result = await orchestrator.run()
|
||||
|
||||
assert result.success is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_both_cdg_and_txt_validation_fail_together(self):
|
||||
"""Test that orchestrator catches CDG failure when both CDG and TXT enabled."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=True,
|
||||
enable_txt=True,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
async def mock_packaging():
|
||||
pass # Both CDG and TXT "failed" silently
|
||||
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
|
||||
|
||||
result = await orchestrator.run()
|
||||
|
||||
assert result.success is False
|
||||
# CDG validation is checked first
|
||||
assert "CDG generation was enabled but failed" in result.error_message
|
||||
mock_encoding.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lrc_missing_raises_when_cdg_enabled(self):
|
||||
"""Test that missing LRC file raises when CDG is enabled."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=True,
|
||||
lrc_file_path="/nonexistent/path.lrc",
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
with pytest.raises(RuntimeError, match="LRC file not available"):
|
||||
await orchestrator._run_packaging()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lrc_missing_raises_when_txt_enabled(self):
|
||||
"""Test that missing LRC file raises when TXT is enabled."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_txt=True,
|
||||
lrc_file_path=None,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
with pytest.raises(RuntimeError, match="LRC file not available"):
|
||||
await orchestrator._run_packaging()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lrc_missing_skips_when_cdg_disabled(self):
|
||||
"""Test that missing LRC file just skips packaging when CDG/TXT disabled."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
enable_cdg=False,
|
||||
enable_txt=False,
|
||||
lrc_file_path=None,
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
# Should not raise, just skip
|
||||
await orchestrator._run_packaging()
|
||||
assert orchestrator.result.final_karaoke_cdg_zip is None
|
||||
|
||||
|
||||
class TestVideoWorkerOrchestratorEncoding:
|
||||
"""Test encoding stage."""
|
||||
|
||||
@@ -772,8 +993,12 @@ class TestVideoWorkerOrchestratorFullPipeline:
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
|
||||
# Mock packaging to set CDG result (validation requires it when enable_cdg=True)
|
||||
async def mock_packaging_with_result():
|
||||
orchestrator.result.final_karaoke_cdg_zip = "/output/cdg.zip"
|
||||
|
||||
# Mock all stages
|
||||
with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_packaging, \
|
||||
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging_with_result) as mock_packaging, \
|
||||
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \
|
||||
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock) as mock_org, \
|
||||
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock) as mock_dist, \
|
||||
|
||||
@@ -228,6 +228,18 @@ class VideoWorkerOrchestrator:
|
||||
if self.config.enable_cdg or self.config.enable_txt:
|
||||
await self._run_packaging()
|
||||
|
||||
# Validate required packaging outputs before expensive encoding
|
||||
if self.config.enable_cdg and not self.result.final_karaoke_cdg_zip:
|
||||
raise RuntimeError(
|
||||
"CDG generation was enabled but failed. "
|
||||
"Check worker logs for CDG-related errors."
|
||||
)
|
||||
if self.config.enable_txt and not self.result.final_karaoke_txt_zip:
|
||||
raise RuntimeError(
|
||||
"TXT generation was enabled but failed. "
|
||||
"Check worker logs for TXT-related errors."
|
||||
)
|
||||
|
||||
# Stage 2: Encoding
|
||||
await self._run_encoding()
|
||||
|
||||
@@ -259,6 +271,10 @@ class VideoWorkerOrchestrator:
|
||||
self.job_log.info("Starting packaging stage (CDG/TXT)")
|
||||
|
||||
if not self.config.lrc_file_path or not os.path.isfile(self.config.lrc_file_path):
|
||||
if self.config.enable_cdg or self.config.enable_txt:
|
||||
raise RuntimeError(
|
||||
f"CDG/TXT generation enabled but LRC file not available: {self.config.lrc_file_path}"
|
||||
)
|
||||
self.job_log.warning("No LRC file available, skipping CDG/TXT packaging")
|
||||
return
|
||||
|
||||
|
||||
@@ -58,6 +58,15 @@ When multiple code paths create jobs (file upload, audio search, webhooks), use
|
||||
|
||||
**Pattern**: When a function can fail, prefer exceptions over boolean returns. Silent failures accumulate into production bugs. Use centralized helpers for multi-step operations (state change + side effects) to prevent divergent implementations.
|
||||
|
||||
### Validate Pipeline Stage Outputs Before Proceeding (Feb 2026)
|
||||
**What happened**: Jobs 5b6aba25 and 5161b069 completed with `enable_cdg=True` but no CDG ZIP was produced. The orchestrator silently caught a `FileNotFoundError` during CDG generation and proceeded to 11+ minutes of GPU encoding, distribution, and notifications — all for an incomplete output.
|
||||
|
||||
**Root cause**: CDG generation failed because `instrumental_selection: custom` had no case in the filename construction (fell through to "Backing" suffix). The `FileNotFoundError` was caught by a broad `except Exception` that just logged and continued, since "CDG is optional."
|
||||
|
||||
**Fix**: Add validation gates between pipeline stages. After packaging completes but before expensive encoding begins, verify that all enabled outputs were actually produced. If `enable_cdg=True` but no CDG ZIP exists, raise `RuntimeError` immediately. This fails fast, saves encoding costs, and makes the error visible.
|
||||
|
||||
**Pattern**: When a pipeline stage produces optional outputs that were explicitly requested, validate their existence before proceeding to the next stage. "Optional" means "can be disabled," not "can silently fail when enabled."
|
||||
|
||||
### Fix Both Sides of Dual Code Paths
|
||||
When fixing a bug in a system with multiple code paths (e.g., legacy vs orchestrator, local vs cloud), verify ALL paths are fixed. PR #271 fixed the GCE worker to READ `instrumental_selection` but only checked the legacy path which was already SENDING it. The orchestrator path (production default) wasn't sending it. **Pattern**: If a component receives config from multiple callers, check ALL callers when fixing the receiving side. Write integration tests that cover each path.
|
||||
|
||||
|
||||
+3
-1
@@ -34,7 +34,7 @@
|
||||
|
||||
## Known Issues
|
||||
|
||||
- CDG format generation requires additional style configuration
|
||||
(No known issues)
|
||||
|
||||
## Pending Work
|
||||
|
||||
@@ -42,6 +42,8 @@
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- **CDG Packaging Fail-Fast Validation** (2026-02-27): Fixed silent CDG generation failures where jobs completed with `enable_cdg=True` but no CDG ZIP was produced. **Root cause**: Jobs 5b6aba25 and 5161b069 had `instrumental_selection: custom` but before commit b671b83f, the orchestrator had no `custom` case — it fell through to `"Backing"` suffix. The CDG generator then hit a `FileNotFoundError` looking for the wrong filename, which was silently caught. **Fix**: (1) Orchestrator now validates CDG/TXT outputs after packaging stage and before encoding — if `enable_cdg=True` but no CDG ZIP was produced, the pipeline fails fast instead of silently continuing to expensive encoding, (2) Missing LRC file now raises when CDG/TXT is enabled instead of silently skipping, (3) Added `scripts/regenerate_cdg.py` remediation script for affected jobs. Removed "CDG requires additional style configuration" known issue (the nomad theme has complete CDG styles). Added 7 new tests for validation logic.
|
||||
|
||||
- **GCE Encoding Pipeline Resilience** (2026-02-26): Made encoding pipeline resilient to Cloud Run deployment interruptions. A deployment killed a poller mid-encoding, leaving job a03ad39c permanently stuck at `encoding` status. Three fixes: (1) GCE worker `/encode` endpoint now idempotent — returns cached results for completed jobs, allows retry on failure, returns status for in-progress (previously hard 409 reject), (2) `encoding_service.py` handles cached/in_progress responses and falls back to status check on 409, (3) Health service detects jobs stuck in `encoding` >50 min. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#resilient-remote-worker-interactions-feb-2026).
|
||||
|
||||
- **Private (Non-Published) Tracks** (2026-02-24): Added `is_private` flag on jobs for private/non-published track production. Private jobs upload to Dropbox `Tracks-NonPublished` folder with `NOMADNP` brand prefix, skip YouTube and Google Drive entirely, and send completion emails without the YouTube section. Available as a checkbox on all job creation tabs and as an admin toggle (auto-deletes outputs when toggling a completed job to private). See [archive/2026-02-24-private-tracks-plan.md](archive/2026-02-24-private-tracks-plan.md).
|
||||
|
||||
@@ -33,6 +33,28 @@ curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/video" \
|
||||
|
||||
---
|
||||
|
||||
## CDG/TXT packages missing from completed job
|
||||
|
||||
**Cause:** Before v0.119.7, CDG/TXT generation failures were silently caught. Jobs would complete with `enable_cdg=True` but no CDG ZIP in outputs. Fixed in v0.119.7 with fail-fast validation — new jobs will now fail loudly if CDG/TXT generation fails.
|
||||
|
||||
**Recovery** (for jobs that already completed without CDG/TXT):
|
||||
|
||||
```bash
|
||||
# Regenerate and distribute CDG/TXT packages for specific jobs
|
||||
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
|
||||
python -m scripts.regenerate_cdg JOB_ID [JOB_ID ...]
|
||||
|
||||
# Example:
|
||||
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
|
||||
python -m scripts.regenerate_cdg 5b6aba25 5161b069
|
||||
```
|
||||
|
||||
The script is idempotent — re-running it skips regeneration if the CDG ZIP already exists in GCS and proceeds to any missing distribution steps (GDrive, Dropbox).
|
||||
|
||||
---
|
||||
|
||||
## GCE encoding worker on wrong wheel version
|
||||
|
||||
**Cause:** Worker picks up a new deploy but doesn't restart automatically.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.119.6"
|
||||
version = "0.119.7"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Regenerate CDG ZIP packages for jobs that completed without them.
|
||||
|
||||
This one-time remediation script handles the case where jobs completed with
|
||||
enable_cdg=True but the CDG ZIP wasn't produced (e.g., due to the custom
|
||||
instrumental path bug fixed in b671b83f).
|
||||
|
||||
For each job, the script:
|
||||
1. Fetches job data from Firestore
|
||||
2. Downloads LRC and instrumental audio from GCS
|
||||
3. Loads theme CDG styles
|
||||
4. Generates CDG package using PackagingService
|
||||
5. Uploads CDG ZIP to GCS
|
||||
6. Updates file_urls.packages.cdg_zip in Firestore
|
||||
7. Uploads CDG ZIP to Google Drive (if GDrive was used)
|
||||
8. Updates state_data.gdrive_files in Firestore
|
||||
9. Uploads CDG/TXT ZIPs to Dropbox (if Dropbox was used)
|
||||
|
||||
Usage:
|
||||
python -m scripts.regenerate_cdg JOB_ID [JOB_ID ...]
|
||||
python -m scripts.regenerate_cdg 5b6aba25 5161b069
|
||||
|
||||
Requirements:
|
||||
- GOOGLE_CLOUD_PROJECT=nomadkaraoke (or set via gcloud config)
|
||||
- gcloud auth application-default login (for GCS/Firestore access)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
os.environ.setdefault("GOOGLE_CLOUD_PROJECT", "nomadkaraoke")
|
||||
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.packaging_service import PackagingService
|
||||
from karaoke_gen.style_loader import get_cdg_format
|
||||
from karaoke_gen.utils import sanitize_filename
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("regenerate_cdg")
|
||||
|
||||
|
||||
def get_instrumental_gcs_key(instrumental_selection: str) -> str:
|
||||
"""Get the GCS file_urls.stems key for the instrumental selection."""
|
||||
if instrumental_selection == "custom":
|
||||
return "custom_instrumental"
|
||||
elif instrumental_selection == "clean":
|
||||
return "instrumental_clean"
|
||||
else: # with_backing
|
||||
return "instrumental_with_backing"
|
||||
|
||||
|
||||
def get_instrumental_suffix(instrumental_selection: str) -> str:
|
||||
"""Get the filename suffix for the instrumental selection."""
|
||||
if instrumental_selection == "custom":
|
||||
return "Custom"
|
||||
elif instrumental_selection == "clean":
|
||||
return "Clean"
|
||||
else:
|
||||
return "Backing"
|
||||
|
||||
|
||||
def regenerate_cdg_for_job(job_id: str, storage: StorageService, job_manager: JobManager, dry_run: bool = False):
|
||||
"""Regenerate CDG ZIP for a single job."""
|
||||
logger.info(f"=== Processing job {job_id} ===")
|
||||
|
||||
# 1. Fetch job from Firestore
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found in Firestore")
|
||||
return False
|
||||
|
||||
logger.info(f"Job: {job.artist} - {job.title}")
|
||||
logger.info(f" enable_cdg: {job.enable_cdg}")
|
||||
logger.info(f" status: {job.status}")
|
||||
|
||||
if not job.enable_cdg:
|
||||
logger.warning(f"Job {job_id} does not have enable_cdg=True, skipping")
|
||||
return False
|
||||
|
||||
# Check if CDG already exists in GCS
|
||||
existing_cdg = job.file_urls.get("packages", {}).get("cdg_zip")
|
||||
cdg_already_in_gcs = existing_cdg and storage.file_exists(existing_cdg)
|
||||
if cdg_already_in_gcs:
|
||||
logger.info(f"CDG ZIP already exists in GCS: {existing_cdg}")
|
||||
elif existing_cdg:
|
||||
logger.warning("CDG ZIP in Firestore but NOT in GCS, will regenerate")
|
||||
|
||||
# Get instrumental selection
|
||||
instrumental_selection = job.state_data.get("instrumental_selection", "clean")
|
||||
logger.info(f" instrumental_selection: {instrumental_selection}")
|
||||
|
||||
# Get lyrics metadata for countdown padding
|
||||
lyrics_metadata = job.state_data.get("lyrics_metadata", {})
|
||||
lrc_has_countdown_padding = lyrics_metadata.get("has_countdown_padding", False)
|
||||
countdown_padding_seconds = lyrics_metadata.get("countdown_padding_seconds", 3.0)
|
||||
logger.info(f" countdown_padding: {lrc_has_countdown_padding} ({countdown_padding_seconds}s)")
|
||||
|
||||
# Check required files exist in file_urls
|
||||
lrc_url = job.file_urls.get("lyrics", {}).get("lrc")
|
||||
if not lrc_url:
|
||||
logger.error(f"Job {job_id} has no LRC file URL")
|
||||
return False
|
||||
|
||||
instrumental_key = get_instrumental_gcs_key(instrumental_selection)
|
||||
instrumental_url = job.file_urls.get("stems", {}).get(instrumental_key)
|
||||
if not instrumental_url:
|
||||
logger.error(f"Job {job_id} has no instrumental URL for key '{instrumental_key}'")
|
||||
return False
|
||||
|
||||
# Get style_params for CDG styles
|
||||
style_params_url = job.style_params_gcs_path
|
||||
|
||||
if dry_run:
|
||||
logger.info(f"DRY RUN: Would regenerate CDG for {job.artist} - {job.title}")
|
||||
return True
|
||||
|
||||
# 2. Download or regenerate CDG/TXT packages
|
||||
with tempfile.TemporaryDirectory(prefix=f"cdg-regen-{job_id}-") as temp_dir:
|
||||
safe_artist = sanitize_filename(job.artist) if job.artist else "Unknown"
|
||||
safe_title = sanitize_filename(job.title) if job.title else "Unknown"
|
||||
base_name = f"{safe_artist} - {safe_title}"
|
||||
instrumental_suffix = get_instrumental_suffix(instrumental_selection)
|
||||
brand_code = getattr(job, "brand_code", None) or job.state_data.get("brand_code", "UNKNOWN")
|
||||
|
||||
cdg_zip_path = os.path.join(temp_dir, f"{base_name} (Final Karaoke CDG).zip")
|
||||
txt_zip_path = os.path.join(temp_dir, f"{base_name} (Final Karaoke TXT).zip")
|
||||
|
||||
if cdg_already_in_gcs:
|
||||
# Download existing CDG ZIP from GCS for distribution
|
||||
logger.info("Downloading existing CDG ZIP from GCS for distribution...")
|
||||
storage.download_file(existing_cdg, cdg_zip_path)
|
||||
|
||||
# Also download TXT if it exists
|
||||
existing_txt = job.file_urls.get("packages", {}).get("txt_zip")
|
||||
if existing_txt and storage.file_exists(existing_txt):
|
||||
logger.info("Downloading existing TXT ZIP from GCS...")
|
||||
storage.download_file(existing_txt, txt_zip_path)
|
||||
else:
|
||||
# Regenerate: download source files and generate packages
|
||||
# Download LRC
|
||||
lrc_path = os.path.join(temp_dir, f"{base_name} (Karaoke).lrc")
|
||||
logger.info(f"Downloading LRC: {lrc_url}")
|
||||
storage.download_file(lrc_url, lrc_path)
|
||||
|
||||
# Download instrumental
|
||||
instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac")
|
||||
logger.info(f"Downloading instrumental: {instrumental_url}")
|
||||
storage.download_file(instrumental_url, instrumental_path)
|
||||
|
||||
# Download and parse style_params
|
||||
import json
|
||||
style_params_path = os.path.join(temp_dir, "style_params.json")
|
||||
style_params = None
|
||||
|
||||
if style_params_url:
|
||||
try:
|
||||
logger.info(f"Downloading style params: {style_params_url}")
|
||||
storage.download_file(style_params_url, style_params_path)
|
||||
with open(style_params_path) as f:
|
||||
style_params = json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to download style_params from {style_params_url}: {e}")
|
||||
|
||||
# Fallback: load theme style_params from GCS theme directory
|
||||
if style_params is None:
|
||||
theme_id = getattr(job, "theme_id", "nomad") or "nomad"
|
||||
theme_path = f"themes/{theme_id}/style_params.json"
|
||||
logger.info(f"Trying theme style_params: {theme_path}")
|
||||
try:
|
||||
storage.download_file(theme_path, style_params_path)
|
||||
with open(style_params_path) as f:
|
||||
style_params = json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load theme style_params: {e}")
|
||||
return False
|
||||
|
||||
# Download CDG style assets (font, backgrounds) referenced in style_params
|
||||
cdg_section = style_params.get("cdg", {})
|
||||
style_dir = os.path.join(temp_dir, "style")
|
||||
os.makedirs(style_dir, exist_ok=True)
|
||||
|
||||
# Resolve asset GCS paths. Per-job style_params have full paths like
|
||||
# "themes/nomad/assets/AvenirNext-Bold.ttf". Theme-level style_params
|
||||
# have bare filenames like "AvenirNext-Bold.ttf" that need to be
|
||||
# resolved against the theme's assets directory.
|
||||
theme_id = getattr(job, "theme_id", "nomad") or "nomad"
|
||||
theme_assets_prefix = f"themes/{theme_id}/assets"
|
||||
|
||||
for asset_key in ["font_path", "instrumental_background", "title_screen_background", "outro_background"]:
|
||||
asset_path = cdg_section.get(asset_key)
|
||||
if not asset_path or not isinstance(asset_path, str) or os.path.isabs(asset_path):
|
||||
continue
|
||||
|
||||
# Determine the GCS path — bare filenames need the theme prefix
|
||||
if "/" not in asset_path:
|
||||
gcs_path = f"{theme_assets_prefix}/{asset_path}"
|
||||
else:
|
||||
gcs_path = asset_path
|
||||
|
||||
local_path = os.path.join(style_dir, os.path.basename(asset_path))
|
||||
try:
|
||||
storage.download_file(gcs_path, local_path)
|
||||
cdg_section[asset_key] = local_path
|
||||
logger.info(f"Downloaded CDG asset {asset_key}: {gcs_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to download CDG asset {asset_key} from {gcs_path}: {e}")
|
||||
|
||||
# 3. Extract CDG styles
|
||||
cdg_styles = get_cdg_format(style_params)
|
||||
if not cdg_styles:
|
||||
logger.error(f"No CDG styles found in theme for job {job_id}")
|
||||
return False
|
||||
|
||||
logger.info("CDG styles loaded successfully")
|
||||
|
||||
# 4. Generate CDG package
|
||||
packaging_service = PackagingService(
|
||||
cdg_styles=cdg_styles,
|
||||
dry_run=False,
|
||||
non_interactive=True,
|
||||
)
|
||||
|
||||
mp3_path = os.path.join(temp_dir, f"{base_name} (Karaoke).mp3")
|
||||
cdg_path = os.path.join(temp_dir, f"{base_name} (Karaoke).cdg")
|
||||
|
||||
logger.info("Generating CDG package...")
|
||||
zip_file, mp3_file, cdg_file = packaging_service.create_cdg_package(
|
||||
lrc_file=lrc_path,
|
||||
audio_file=instrumental_path,
|
||||
output_zip_path=cdg_zip_path,
|
||||
artist=job.artist,
|
||||
title=job.title,
|
||||
output_mp3_path=mp3_path,
|
||||
output_cdg_path=cdg_path,
|
||||
lrc_has_countdown_padding=lrc_has_countdown_padding,
|
||||
countdown_padding_seconds=countdown_padding_seconds,
|
||||
)
|
||||
|
||||
if not os.path.isfile(cdg_zip_path):
|
||||
logger.error(f"CDG ZIP not created at expected path: {cdg_zip_path}")
|
||||
return False
|
||||
|
||||
zip_size = os.path.getsize(cdg_zip_path)
|
||||
logger.info(f"CDG ZIP created: {cdg_zip_path} ({zip_size / 1024:.1f} KB)")
|
||||
|
||||
# 5. Upload CDG ZIP to GCS
|
||||
gcs_cdg_path = f"jobs/{job_id}/packages/cdg_zip.zip"
|
||||
logger.info(f"Uploading CDG ZIP to GCS: {gcs_cdg_path}")
|
||||
storage.upload_file(cdg_zip_path, gcs_cdg_path)
|
||||
|
||||
# 6. Update Firestore file_urls
|
||||
logger.info("Updating Firestore file_urls.packages.cdg_zip")
|
||||
job_manager.update_file_url(job_id, "packages", "cdg_zip", gcs_cdg_path)
|
||||
|
||||
# Also generate TXT if enabled
|
||||
if job.enable_txt:
|
||||
logger.info("Job also has enable_txt=True, generating TXT package...")
|
||||
try:
|
||||
txt_zip, txt_file = packaging_service.create_txt_package(
|
||||
lrc_file=lrc_path,
|
||||
mp3_file=mp3_path,
|
||||
output_zip_path=txt_zip_path,
|
||||
)
|
||||
if os.path.isfile(txt_zip_path):
|
||||
gcs_txt_path = f"jobs/{job_id}/packages/txt_zip.zip"
|
||||
storage.upload_file(txt_zip_path, gcs_txt_path)
|
||||
job_manager.update_file_url(job_id, "packages", "txt_zip", gcs_txt_path)
|
||||
logger.info(f"TXT ZIP uploaded: {gcs_txt_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"TXT generation failed: {e}")
|
||||
|
||||
# === Distribution steps (run for both new and existing packages) ===
|
||||
|
||||
# 7. Upload to GDrive if job uses it
|
||||
gdrive_files = job.state_data.get("gdrive_files", {})
|
||||
if gdrive_files and "cdg" not in gdrive_files:
|
||||
logger.info("Uploading CDG ZIP to Google Drive...")
|
||||
try:
|
||||
from backend.services.gdrive_service import get_gdrive_service
|
||||
gdrive = get_gdrive_service()
|
||||
if gdrive.is_configured:
|
||||
gdrive_folder_id = getattr(job, "gdrive_folder_id", None)
|
||||
|
||||
if gdrive_folder_id:
|
||||
safe_base = sanitize_filename(f"{job.artist} - {job.title}")
|
||||
filename_base = f"{brand_code} - {safe_base}"
|
||||
cdg_folder_id = gdrive.get_or_create_folder(gdrive_folder_id, "CDG")
|
||||
file_id = gdrive.upload_file(
|
||||
cdg_zip_path,
|
||||
cdg_folder_id,
|
||||
f"{filename_base}.zip",
|
||||
)
|
||||
logger.info(f"Uploaded CDG to GDrive: {file_id}")
|
||||
|
||||
# 8. Update state_data.gdrive_files
|
||||
gdrive_files["cdg"] = file_id
|
||||
job_manager.update_job(job_id, {"state_data.gdrive_files": gdrive_files})
|
||||
logger.info("Updated Firestore state_data.gdrive_files")
|
||||
else:
|
||||
logger.warning("No gdrive_folder_id on job, skipping GDrive upload")
|
||||
else:
|
||||
logger.warning("GDrive not configured, skipping upload")
|
||||
except Exception as e:
|
||||
logger.error(f"GDrive upload failed (non-fatal): {e}")
|
||||
elif gdrive_files:
|
||||
logger.info("GDrive already has CDG file, skipping GDrive upload")
|
||||
else:
|
||||
logger.info("No existing GDrive files, skipping GDrive upload")
|
||||
|
||||
# 9. Upload to Dropbox if job uses it
|
||||
dropbox_path = job.dropbox_path
|
||||
if dropbox_path:
|
||||
logger.info("Uploading packages to Dropbox...")
|
||||
try:
|
||||
from backend.services.dropbox_service import get_dropbox_service
|
||||
dropbox = get_dropbox_service()
|
||||
if dropbox.is_configured:
|
||||
safe_base = sanitize_filename(f"{job.artist} - {job.title}")
|
||||
folder_name = f"{brand_code} - {safe_base}"
|
||||
remote_folder = f"{dropbox_path}/{folder_name}"
|
||||
|
||||
# Upload CDG ZIP
|
||||
if os.path.isfile(cdg_zip_path):
|
||||
cdg_remote = f"{remote_folder}/{os.path.basename(cdg_zip_path)}"
|
||||
dropbox.upload_file(cdg_zip_path, cdg_remote)
|
||||
logger.info(f"Uploaded CDG to Dropbox: {cdg_remote}")
|
||||
|
||||
# Upload TXT ZIP
|
||||
if os.path.isfile(txt_zip_path):
|
||||
txt_remote = f"{remote_folder}/{os.path.basename(txt_zip_path)}"
|
||||
dropbox.upload_file(txt_zip_path, txt_remote)
|
||||
logger.info(f"Uploaded TXT to Dropbox: {txt_remote}")
|
||||
else:
|
||||
logger.warning("Dropbox not configured, skipping upload")
|
||||
except Exception as e:
|
||||
logger.error(f"Dropbox upload failed (non-fatal): {e}")
|
||||
else:
|
||||
logger.info("No dropbox_path on job, skipping Dropbox upload")
|
||||
|
||||
logger.info(f"=== Job {job_id} CDG regeneration complete ===")
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Regenerate CDG ZIP packages for jobs that completed without them."
|
||||
)
|
||||
parser.add_argument(
|
||||
"job_ids",
|
||||
nargs="+",
|
||||
help="Job IDs to regenerate CDG for",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be done without making changes",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
storage = StorageService()
|
||||
job_manager = JobManager()
|
||||
|
||||
results = {}
|
||||
for job_id in args.job_ids:
|
||||
try:
|
||||
success = regenerate_cdg_for_job(job_id, storage, job_manager, dry_run=args.dry_run)
|
||||
results[job_id] = "OK" if success else "FAILED"
|
||||
except Exception as e:
|
||||
logger.exception(f"Error processing job {job_id}: {e}")
|
||||
results[job_id] = f"ERROR: {e}"
|
||||
|
||||
# Summary
|
||||
logger.info("\n=== Summary ===")
|
||||
for job_id, status in results.items():
|
||||
logger.info(f" {job_id}: {status}")
|
||||
|
||||
# Exit with error if any failed
|
||||
if any(s != "OK" for s in results.values()):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user