From 23a8b0fb98df7085e2bdf2aaa3af08288e78ba58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Fri, 27 Feb 2026 14:13:22 -0500 Subject: [PATCH] Fix missing CDG ZIP packages: fail-fast validation + remediation (#416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - **Systemic fix**: Orchestrator now validates CDG/TXT outputs after packaging, before encoding. If `enable_cdg=True` but no CDG ZIP was produced, the job fails immediately with a clear error instead of silently completing without the package. - **Remediation**: Created `scripts/regenerate_cdg.py` to regenerate and distribute CDG/TXT packages for affected jobs (5b6aba25, 5161b069). Packages uploaded to GCS, GDrive, and Dropbox. - **Root cause**: Before b671b83f, `instrumental_selection: custom` had no case in the orchestrator config builder — fell through to "Backing" suffix, CDG generator looked for wrong filename, `FileNotFoundError` was silently caught. ## Changes | File | Change | |------|--------| | `backend/workers/video_worker_orchestrator.py` | Post-packaging validation + LRC-missing check update | | `backend/tests/test_video_worker_orchestrator.py` | 8 new tests in `TestPackagingValidation` | | `scripts/regenerate_cdg.py` | One-time remediation script (GCS + Firestore + GDrive + Dropbox) | | `docs/LESSONS-LEARNED.md` | Pipeline stage validation pattern | | `docs/TROUBLESHOOTING.md` | CDG missing recovery runbook | | `docs/README.md` | Updated recent changes | | `pyproject.toml` | Version 0.119.6 → 0.119.7 | ## Test plan - [x] 8 new tests in `TestPackagingValidation` (CDG fails, TXT fails, combined CDG+TXT fails, CDG succeeds, CDG disabled, LRC missing raises with CDG, LRC missing raises with TXT, LRC missing skips without CDG) - [x] All 2062+ backend tests pass - [x] Remediation script run successfully for both affected jobs - [x] Verified CDG/TXT ZIPs in GCS, Firestore, GDrive, and Dropbox > **CodeRabbit note**: 1 finding (unreachable `else` branch in `_run_packaging` LRC check) — kept as defensive code since the test `test_lrc_missing_skips_when_cdg_disabled` exercises this path directly. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 --- .../tests/test_video_worker_orchestrator.py | 239 ++++++++++- backend/workers/video_worker_orchestrator.py | 16 + docs/LESSONS-LEARNED.md | 9 + docs/README.md | 4 +- docs/TROUBLESHOOTING.md | 22 + pyproject.toml | 2 +- scripts/regenerate_cdg.py | 395 ++++++++++++++++++ 7 files changed, 678 insertions(+), 9 deletions(-) create mode 100644 scripts/regenerate_cdg.py diff --git a/backend/tests/test_video_worker_orchestrator.py b/backend/tests/test_video_worker_orchestrator.py index 8ddf3327..7acd6365 100644 --- a/backend/tests/test_video_worker_orchestrator.py +++ b/backend/tests/test_video_worker_orchestrator.py @@ -250,8 +250,28 @@ class TestVideoWorkerOrchestratorPackaging: """Test packaging stage.""" @pytest.mark.asyncio - async def test_run_packaging_no_lrc(self): - """Test packaging stage with no LRC file.""" + async def test_run_packaging_no_lrc_cdg_disabled(self): + """Test packaging stage with no LRC file and CDG disabled skips silently.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=False, + lrc_file_path=None, + ) + orchestrator = VideoWorkerOrchestrator(config) + + # Should not raise, just skip + await orchestrator._run_packaging() + + assert orchestrator.result.final_karaoke_cdg_zip is None + + @pytest.mark.asyncio + async def test_run_packaging_no_lrc_cdg_enabled_raises(self): + """Test packaging stage raises when CDG enabled but no LRC file.""" config = OrchestratorConfig( job_id="test-job", artist="Test Artist", @@ -264,10 +284,8 @@ class TestVideoWorkerOrchestratorPackaging: ) orchestrator = VideoWorkerOrchestrator(config) - # Should not raise, just skip - await orchestrator._run_packaging() - - assert orchestrator.result.final_karaoke_cdg_zip is None + with pytest.raises(RuntimeError, match="LRC file not available"): + await orchestrator._run_packaging() @pytest.mark.asyncio async def test_run_packaging_cdg(self): @@ -368,6 +386,209 @@ class TestVideoWorkerOrchestratorPackaging: "countdown_padding_seconds must be passed for CDG timestamp adjustment" +class TestPackagingValidation: + """Test CDG/TXT packaging validation checks. + + These tests verify that the orchestrator fails fast when CDG/TXT + generation is enabled but fails, rather than silently proceeding + to expensive encoding. This prevents jobs from completing without + their requested CDG/TXT packages. + + Added after jobs 5b6aba25 and 5161b069 completed with enable_cdg=True + but no CDG ZIP was produced due to a FileNotFoundError being silently caught. + """ + + @pytest.mark.asyncio + async def test_cdg_validation_fails_when_enabled_but_not_produced(self): + """Test that orchestrator raises when CDG enabled but not produced.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=True, + ) + orchestrator = VideoWorkerOrchestrator(config) + + # Mock _run_packaging to NOT set result.final_karaoke_cdg_zip (simulates failure) + async def mock_packaging(): + pass # CDG generation "failed" silently + + with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding: + + result = await orchestrator.run() + + assert result.success is False + assert "CDG generation was enabled but failed" in result.error_message + mock_encoding.assert_not_called() # Should not reach encoding + + @pytest.mark.asyncio + async def test_txt_validation_fails_when_enabled_but_not_produced(self): + """Test that orchestrator raises when TXT enabled but not produced.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_txt=True, + ) + orchestrator = VideoWorkerOrchestrator(config) + + async def mock_packaging(): + pass # TXT generation "failed" silently + + with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding: + + result = await orchestrator.run() + + assert result.success is False + assert "TXT generation was enabled but failed" in result.error_message + mock_encoding.assert_not_called() + + @pytest.mark.asyncio + async def test_pipeline_succeeds_when_cdg_disabled(self): + """Test that pipeline succeeds normally when CDG is not enabled.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=False, + enable_txt=False, + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_pkg, \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock): + + result = await orchestrator.run() + + assert result.success is True + mock_pkg.assert_not_called() # Packaging skipped entirely + + @pytest.mark.asyncio + async def test_cdg_validation_passes_when_cdg_produced(self): + """Test that validation passes when CDG is successfully produced.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=True, + ) + orchestrator = VideoWorkerOrchestrator(config) + + async def mock_packaging_success(): + orchestrator.result.final_karaoke_cdg_zip = "/output/cdg.zip" + + with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging_success), \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock): + + result = await orchestrator.run() + + assert result.success is True + + @pytest.mark.asyncio + async def test_both_cdg_and_txt_validation_fail_together(self): + """Test that orchestrator catches CDG failure when both CDG and TXT enabled.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=True, + enable_txt=True, + ) + orchestrator = VideoWorkerOrchestrator(config) + + async def mock_packaging(): + pass # Both CDG and TXT "failed" silently + + with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding: + + result = await orchestrator.run() + + assert result.success is False + # CDG validation is checked first + assert "CDG generation was enabled but failed" in result.error_message + mock_encoding.assert_not_called() + + @pytest.mark.asyncio + async def test_lrc_missing_raises_when_cdg_enabled(self): + """Test that missing LRC file raises when CDG is enabled.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=True, + lrc_file_path="/nonexistent/path.lrc", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with pytest.raises(RuntimeError, match="LRC file not available"): + await orchestrator._run_packaging() + + @pytest.mark.asyncio + async def test_lrc_missing_raises_when_txt_enabled(self): + """Test that missing LRC file raises when TXT is enabled.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_txt=True, + lrc_file_path=None, + ) + orchestrator = VideoWorkerOrchestrator(config) + + with pytest.raises(RuntimeError, match="LRC file not available"): + await orchestrator._run_packaging() + + @pytest.mark.asyncio + async def test_lrc_missing_skips_when_cdg_disabled(self): + """Test that missing LRC file just skips packaging when CDG/TXT disabled.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=False, + enable_txt=False, + lrc_file_path=None, + ) + orchestrator = VideoWorkerOrchestrator(config) + + # Should not raise, just skip + await orchestrator._run_packaging() + assert orchestrator.result.final_karaoke_cdg_zip is None + + class TestVideoWorkerOrchestratorEncoding: """Test encoding stage.""" @@ -772,8 +993,12 @@ class TestVideoWorkerOrchestratorFullPipeline: ) orchestrator = VideoWorkerOrchestrator(config) + # Mock packaging to set CDG result (validation requires it when enable_cdg=True) + async def mock_packaging_with_result(): + orchestrator.result.final_karaoke_cdg_zip = "/output/cdg.zip" + # Mock all stages - with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_packaging, \ + with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging_with_result) as mock_packaging, \ patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \ patch.object(orchestrator, "_run_organization", new_callable=AsyncMock) as mock_org, \ patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock) as mock_dist, \ diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index f9f406a2..0c63c5c4 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -228,6 +228,18 @@ class VideoWorkerOrchestrator: if self.config.enable_cdg or self.config.enable_txt: await self._run_packaging() + # Validate required packaging outputs before expensive encoding + if self.config.enable_cdg and not self.result.final_karaoke_cdg_zip: + raise RuntimeError( + "CDG generation was enabled but failed. " + "Check worker logs for CDG-related errors." + ) + if self.config.enable_txt and not self.result.final_karaoke_txt_zip: + raise RuntimeError( + "TXT generation was enabled but failed. " + "Check worker logs for TXT-related errors." + ) + # Stage 2: Encoding await self._run_encoding() @@ -259,6 +271,10 @@ class VideoWorkerOrchestrator: self.job_log.info("Starting packaging stage (CDG/TXT)") if not self.config.lrc_file_path or not os.path.isfile(self.config.lrc_file_path): + if self.config.enable_cdg or self.config.enable_txt: + raise RuntimeError( + f"CDG/TXT generation enabled but LRC file not available: {self.config.lrc_file_path}" + ) self.job_log.warning("No LRC file available, skipping CDG/TXT packaging") return diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index eee489cf..4a24f58a 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -58,6 +58,15 @@ When multiple code paths create jobs (file upload, audio search, webhooks), use **Pattern**: When a function can fail, prefer exceptions over boolean returns. Silent failures accumulate into production bugs. Use centralized helpers for multi-step operations (state change + side effects) to prevent divergent implementations. +### Validate Pipeline Stage Outputs Before Proceeding (Feb 2026) +**What happened**: Jobs 5b6aba25 and 5161b069 completed with `enable_cdg=True` but no CDG ZIP was produced. The orchestrator silently caught a `FileNotFoundError` during CDG generation and proceeded to 11+ minutes of GPU encoding, distribution, and notifications — all for an incomplete output. + +**Root cause**: CDG generation failed because `instrumental_selection: custom` had no case in the filename construction (fell through to "Backing" suffix). The `FileNotFoundError` was caught by a broad `except Exception` that just logged and continued, since "CDG is optional." + +**Fix**: Add validation gates between pipeline stages. After packaging completes but before expensive encoding begins, verify that all enabled outputs were actually produced. If `enable_cdg=True` but no CDG ZIP exists, raise `RuntimeError` immediately. This fails fast, saves encoding costs, and makes the error visible. + +**Pattern**: When a pipeline stage produces optional outputs that were explicitly requested, validate their existence before proceeding to the next stage. "Optional" means "can be disabled," not "can silently fail when enabled." + ### Fix Both Sides of Dual Code Paths When fixing a bug in a system with multiple code paths (e.g., legacy vs orchestrator, local vs cloud), verify ALL paths are fixed. PR #271 fixed the GCE worker to READ `instrumental_selection` but only checked the legacy path which was already SENDING it. The orchestrator path (production default) wasn't sending it. **Pattern**: If a component receives config from multiple callers, check ALL callers when fixing the receiving side. Write integration tests that cover each path. diff --git a/docs/README.md b/docs/README.md index 28d3dd3f..0fe73ce9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,7 +34,7 @@ ## Known Issues -- CDG format generation requires additional style configuration +(No known issues) ## Pending Work @@ -42,6 +42,8 @@ ## Recent Changes +- **CDG Packaging Fail-Fast Validation** (2026-02-27): Fixed silent CDG generation failures where jobs completed with `enable_cdg=True` but no CDG ZIP was produced. **Root cause**: Jobs 5b6aba25 and 5161b069 had `instrumental_selection: custom` but before commit b671b83f, the orchestrator had no `custom` case — it fell through to `"Backing"` suffix. The CDG generator then hit a `FileNotFoundError` looking for the wrong filename, which was silently caught. **Fix**: (1) Orchestrator now validates CDG/TXT outputs after packaging stage and before encoding — if `enable_cdg=True` but no CDG ZIP was produced, the pipeline fails fast instead of silently continuing to expensive encoding, (2) Missing LRC file now raises when CDG/TXT is enabled instead of silently skipping, (3) Added `scripts/regenerate_cdg.py` remediation script for affected jobs. Removed "CDG requires additional style configuration" known issue (the nomad theme has complete CDG styles). Added 7 new tests for validation logic. + - **GCE Encoding Pipeline Resilience** (2026-02-26): Made encoding pipeline resilient to Cloud Run deployment interruptions. A deployment killed a poller mid-encoding, leaving job a03ad39c permanently stuck at `encoding` status. Three fixes: (1) GCE worker `/encode` endpoint now idempotent — returns cached results for completed jobs, allows retry on failure, returns status for in-progress (previously hard 409 reject), (2) `encoding_service.py` handles cached/in_progress responses and falls back to status check on 409, (3) Health service detects jobs stuck in `encoding` >50 min. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#resilient-remote-worker-interactions-feb-2026). - **Private (Non-Published) Tracks** (2026-02-24): Added `is_private` flag on jobs for private/non-published track production. Private jobs upload to Dropbox `Tracks-NonPublished` folder with `NOMADNP` brand prefix, skip YouTube and Google Drive entirely, and send completion emails without the YouTube section. Available as a checkbox on all job creation tabs and as an admin toggle (auto-deletes outputs when toggling a completed job to private). See [archive/2026-02-24-private-tracks-plan.md](archive/2026-02-24-private-tracks-plan.md). diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 5e646c8f..fc71ff1c 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -33,6 +33,28 @@ curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/video" \ --- +## CDG/TXT packages missing from completed job + +**Cause:** Before v0.119.7, CDG/TXT generation failures were silently caught. Jobs would complete with `enable_cdg=True` but no CDG ZIP in outputs. Fixed in v0.119.7 with fail-fast validation — new jobs will now fail loudly if CDG/TXT generation fails. + +**Recovery** (for jobs that already completed without CDG/TXT): + +```bash +# Regenerate and distribute CDG/TXT packages for specific jobs +GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \ +GOOGLE_CLOUD_PROJECT=nomadkaraoke \ +python -m scripts.regenerate_cdg JOB_ID [JOB_ID ...] + +# Example: +GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \ +GOOGLE_CLOUD_PROJECT=nomadkaraoke \ +python -m scripts.regenerate_cdg 5b6aba25 5161b069 +``` + +The script is idempotent — re-running it skips regeneration if the CDG ZIP already exists in GCS and proceeds to any missing distribution steps (GDrive, Dropbox). + +--- + ## GCE encoding worker on wrong wheel version **Cause:** Worker picks up a new deploy but doesn't restart automatically. diff --git a/pyproject.toml b/pyproject.toml index c20f03ec..81a01b6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.119.6" +version = "0.119.7" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/regenerate_cdg.py b/scripts/regenerate_cdg.py new file mode 100644 index 00000000..6572af36 --- /dev/null +++ b/scripts/regenerate_cdg.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +""" +Regenerate CDG ZIP packages for jobs that completed without them. + +This one-time remediation script handles the case where jobs completed with +enable_cdg=True but the CDG ZIP wasn't produced (e.g., due to the custom +instrumental path bug fixed in b671b83f). + +For each job, the script: +1. Fetches job data from Firestore +2. Downloads LRC and instrumental audio from GCS +3. Loads theme CDG styles +4. Generates CDG package using PackagingService +5. Uploads CDG ZIP to GCS +6. Updates file_urls.packages.cdg_zip in Firestore +7. Uploads CDG ZIP to Google Drive (if GDrive was used) +8. Updates state_data.gdrive_files in Firestore +9. Uploads CDG/TXT ZIPs to Dropbox (if Dropbox was used) + +Usage: + python -m scripts.regenerate_cdg JOB_ID [JOB_ID ...] + python -m scripts.regenerate_cdg 5b6aba25 5161b069 + +Requirements: + - GOOGLE_CLOUD_PROJECT=nomadkaraoke (or set via gcloud config) + - gcloud auth application-default login (for GCS/Firestore access) +""" + +import argparse +import logging +import os +import sys +import tempfile +from pathlib import Path + +# Add project root to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +os.environ.setdefault("GOOGLE_CLOUD_PROJECT", "nomadkaraoke") + +from backend.services.storage_service import StorageService +from backend.services.job_manager import JobManager +from backend.services.packaging_service import PackagingService +from karaoke_gen.style_loader import get_cdg_format +from karaoke_gen.utils import sanitize_filename + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +logger = logging.getLogger("regenerate_cdg") + + +def get_instrumental_gcs_key(instrumental_selection: str) -> str: + """Get the GCS file_urls.stems key for the instrumental selection.""" + if instrumental_selection == "custom": + return "custom_instrumental" + elif instrumental_selection == "clean": + return "instrumental_clean" + else: # with_backing + return "instrumental_with_backing" + + +def get_instrumental_suffix(instrumental_selection: str) -> str: + """Get the filename suffix for the instrumental selection.""" + if instrumental_selection == "custom": + return "Custom" + elif instrumental_selection == "clean": + return "Clean" + else: + return "Backing" + + +def regenerate_cdg_for_job(job_id: str, storage: StorageService, job_manager: JobManager, dry_run: bool = False): + """Regenerate CDG ZIP for a single job.""" + logger.info(f"=== Processing job {job_id} ===") + + # 1. Fetch job from Firestore + job = job_manager.get_job(job_id) + if not job: + logger.error(f"Job {job_id} not found in Firestore") + return False + + logger.info(f"Job: {job.artist} - {job.title}") + logger.info(f" enable_cdg: {job.enable_cdg}") + logger.info(f" status: {job.status}") + + if not job.enable_cdg: + logger.warning(f"Job {job_id} does not have enable_cdg=True, skipping") + return False + + # Check if CDG already exists in GCS + existing_cdg = job.file_urls.get("packages", {}).get("cdg_zip") + cdg_already_in_gcs = existing_cdg and storage.file_exists(existing_cdg) + if cdg_already_in_gcs: + logger.info(f"CDG ZIP already exists in GCS: {existing_cdg}") + elif existing_cdg: + logger.warning("CDG ZIP in Firestore but NOT in GCS, will regenerate") + + # Get instrumental selection + instrumental_selection = job.state_data.get("instrumental_selection", "clean") + logger.info(f" instrumental_selection: {instrumental_selection}") + + # Get lyrics metadata for countdown padding + lyrics_metadata = job.state_data.get("lyrics_metadata", {}) + lrc_has_countdown_padding = lyrics_metadata.get("has_countdown_padding", False) + countdown_padding_seconds = lyrics_metadata.get("countdown_padding_seconds", 3.0) + logger.info(f" countdown_padding: {lrc_has_countdown_padding} ({countdown_padding_seconds}s)") + + # Check required files exist in file_urls + lrc_url = job.file_urls.get("lyrics", {}).get("lrc") + if not lrc_url: + logger.error(f"Job {job_id} has no LRC file URL") + return False + + instrumental_key = get_instrumental_gcs_key(instrumental_selection) + instrumental_url = job.file_urls.get("stems", {}).get(instrumental_key) + if not instrumental_url: + logger.error(f"Job {job_id} has no instrumental URL for key '{instrumental_key}'") + return False + + # Get style_params for CDG styles + style_params_url = job.style_params_gcs_path + + if dry_run: + logger.info(f"DRY RUN: Would regenerate CDG for {job.artist} - {job.title}") + return True + + # 2. Download or regenerate CDG/TXT packages + with tempfile.TemporaryDirectory(prefix=f"cdg-regen-{job_id}-") as temp_dir: + safe_artist = sanitize_filename(job.artist) if job.artist else "Unknown" + safe_title = sanitize_filename(job.title) if job.title else "Unknown" + base_name = f"{safe_artist} - {safe_title}" + instrumental_suffix = get_instrumental_suffix(instrumental_selection) + brand_code = getattr(job, "brand_code", None) or job.state_data.get("brand_code", "UNKNOWN") + + cdg_zip_path = os.path.join(temp_dir, f"{base_name} (Final Karaoke CDG).zip") + txt_zip_path = os.path.join(temp_dir, f"{base_name} (Final Karaoke TXT).zip") + + if cdg_already_in_gcs: + # Download existing CDG ZIP from GCS for distribution + logger.info("Downloading existing CDG ZIP from GCS for distribution...") + storage.download_file(existing_cdg, cdg_zip_path) + + # Also download TXT if it exists + existing_txt = job.file_urls.get("packages", {}).get("txt_zip") + if existing_txt and storage.file_exists(existing_txt): + logger.info("Downloading existing TXT ZIP from GCS...") + storage.download_file(existing_txt, txt_zip_path) + else: + # Regenerate: download source files and generate packages + # Download LRC + lrc_path = os.path.join(temp_dir, f"{base_name} (Karaoke).lrc") + logger.info(f"Downloading LRC: {lrc_url}") + storage.download_file(lrc_url, lrc_path) + + # Download instrumental + instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac") + logger.info(f"Downloading instrumental: {instrumental_url}") + storage.download_file(instrumental_url, instrumental_path) + + # Download and parse style_params + import json + style_params_path = os.path.join(temp_dir, "style_params.json") + style_params = None + + if style_params_url: + try: + logger.info(f"Downloading style params: {style_params_url}") + storage.download_file(style_params_url, style_params_path) + with open(style_params_path) as f: + style_params = json.load(f) + except Exception as e: + logger.warning(f"Failed to download style_params from {style_params_url}: {e}") + + # Fallback: load theme style_params from GCS theme directory + if style_params is None: + theme_id = getattr(job, "theme_id", "nomad") or "nomad" + theme_path = f"themes/{theme_id}/style_params.json" + logger.info(f"Trying theme style_params: {theme_path}") + try: + storage.download_file(theme_path, style_params_path) + with open(style_params_path) as f: + style_params = json.load(f) + except Exception as e: + logger.error(f"Failed to load theme style_params: {e}") + return False + + # Download CDG style assets (font, backgrounds) referenced in style_params + cdg_section = style_params.get("cdg", {}) + style_dir = os.path.join(temp_dir, "style") + os.makedirs(style_dir, exist_ok=True) + + # Resolve asset GCS paths. Per-job style_params have full paths like + # "themes/nomad/assets/AvenirNext-Bold.ttf". Theme-level style_params + # have bare filenames like "AvenirNext-Bold.ttf" that need to be + # resolved against the theme's assets directory. + theme_id = getattr(job, "theme_id", "nomad") or "nomad" + theme_assets_prefix = f"themes/{theme_id}/assets" + + for asset_key in ["font_path", "instrumental_background", "title_screen_background", "outro_background"]: + asset_path = cdg_section.get(asset_key) + if not asset_path or not isinstance(asset_path, str) or os.path.isabs(asset_path): + continue + + # Determine the GCS path — bare filenames need the theme prefix + if "/" not in asset_path: + gcs_path = f"{theme_assets_prefix}/{asset_path}" + else: + gcs_path = asset_path + + local_path = os.path.join(style_dir, os.path.basename(asset_path)) + try: + storage.download_file(gcs_path, local_path) + cdg_section[asset_key] = local_path + logger.info(f"Downloaded CDG asset {asset_key}: {gcs_path}") + except Exception as e: + logger.warning(f"Failed to download CDG asset {asset_key} from {gcs_path}: {e}") + + # 3. Extract CDG styles + cdg_styles = get_cdg_format(style_params) + if not cdg_styles: + logger.error(f"No CDG styles found in theme for job {job_id}") + return False + + logger.info("CDG styles loaded successfully") + + # 4. Generate CDG package + packaging_service = PackagingService( + cdg_styles=cdg_styles, + dry_run=False, + non_interactive=True, + ) + + mp3_path = os.path.join(temp_dir, f"{base_name} (Karaoke).mp3") + cdg_path = os.path.join(temp_dir, f"{base_name} (Karaoke).cdg") + + logger.info("Generating CDG package...") + zip_file, mp3_file, cdg_file = packaging_service.create_cdg_package( + lrc_file=lrc_path, + audio_file=instrumental_path, + output_zip_path=cdg_zip_path, + artist=job.artist, + title=job.title, + output_mp3_path=mp3_path, + output_cdg_path=cdg_path, + lrc_has_countdown_padding=lrc_has_countdown_padding, + countdown_padding_seconds=countdown_padding_seconds, + ) + + if not os.path.isfile(cdg_zip_path): + logger.error(f"CDG ZIP not created at expected path: {cdg_zip_path}") + return False + + zip_size = os.path.getsize(cdg_zip_path) + logger.info(f"CDG ZIP created: {cdg_zip_path} ({zip_size / 1024:.1f} KB)") + + # 5. Upload CDG ZIP to GCS + gcs_cdg_path = f"jobs/{job_id}/packages/cdg_zip.zip" + logger.info(f"Uploading CDG ZIP to GCS: {gcs_cdg_path}") + storage.upload_file(cdg_zip_path, gcs_cdg_path) + + # 6. Update Firestore file_urls + logger.info("Updating Firestore file_urls.packages.cdg_zip") + job_manager.update_file_url(job_id, "packages", "cdg_zip", gcs_cdg_path) + + # Also generate TXT if enabled + if job.enable_txt: + logger.info("Job also has enable_txt=True, generating TXT package...") + try: + txt_zip, txt_file = packaging_service.create_txt_package( + lrc_file=lrc_path, + mp3_file=mp3_path, + output_zip_path=txt_zip_path, + ) + if os.path.isfile(txt_zip_path): + gcs_txt_path = f"jobs/{job_id}/packages/txt_zip.zip" + storage.upload_file(txt_zip_path, gcs_txt_path) + job_manager.update_file_url(job_id, "packages", "txt_zip", gcs_txt_path) + logger.info(f"TXT ZIP uploaded: {gcs_txt_path}") + except Exception as e: + logger.error(f"TXT generation failed: {e}") + + # === Distribution steps (run for both new and existing packages) === + + # 7. Upload to GDrive if job uses it + gdrive_files = job.state_data.get("gdrive_files", {}) + if gdrive_files and "cdg" not in gdrive_files: + logger.info("Uploading CDG ZIP to Google Drive...") + try: + from backend.services.gdrive_service import get_gdrive_service + gdrive = get_gdrive_service() + if gdrive.is_configured: + gdrive_folder_id = getattr(job, "gdrive_folder_id", None) + + if gdrive_folder_id: + safe_base = sanitize_filename(f"{job.artist} - {job.title}") + filename_base = f"{brand_code} - {safe_base}" + cdg_folder_id = gdrive.get_or_create_folder(gdrive_folder_id, "CDG") + file_id = gdrive.upload_file( + cdg_zip_path, + cdg_folder_id, + f"{filename_base}.zip", + ) + logger.info(f"Uploaded CDG to GDrive: {file_id}") + + # 8. Update state_data.gdrive_files + gdrive_files["cdg"] = file_id + job_manager.update_job(job_id, {"state_data.gdrive_files": gdrive_files}) + logger.info("Updated Firestore state_data.gdrive_files") + else: + logger.warning("No gdrive_folder_id on job, skipping GDrive upload") + else: + logger.warning("GDrive not configured, skipping upload") + except Exception as e: + logger.error(f"GDrive upload failed (non-fatal): {e}") + elif gdrive_files: + logger.info("GDrive already has CDG file, skipping GDrive upload") + else: + logger.info("No existing GDrive files, skipping GDrive upload") + + # 9. Upload to Dropbox if job uses it + dropbox_path = job.dropbox_path + if dropbox_path: + logger.info("Uploading packages to Dropbox...") + try: + from backend.services.dropbox_service import get_dropbox_service + dropbox = get_dropbox_service() + if dropbox.is_configured: + safe_base = sanitize_filename(f"{job.artist} - {job.title}") + folder_name = f"{brand_code} - {safe_base}" + remote_folder = f"{dropbox_path}/{folder_name}" + + # Upload CDG ZIP + if os.path.isfile(cdg_zip_path): + cdg_remote = f"{remote_folder}/{os.path.basename(cdg_zip_path)}" + dropbox.upload_file(cdg_zip_path, cdg_remote) + logger.info(f"Uploaded CDG to Dropbox: {cdg_remote}") + + # Upload TXT ZIP + if os.path.isfile(txt_zip_path): + txt_remote = f"{remote_folder}/{os.path.basename(txt_zip_path)}" + dropbox.upload_file(txt_zip_path, txt_remote) + logger.info(f"Uploaded TXT to Dropbox: {txt_remote}") + else: + logger.warning("Dropbox not configured, skipping upload") + except Exception as e: + logger.error(f"Dropbox upload failed (non-fatal): {e}") + else: + logger.info("No dropbox_path on job, skipping Dropbox upload") + + logger.info(f"=== Job {job_id} CDG regeneration complete ===") + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Regenerate CDG ZIP packages for jobs that completed without them." + ) + parser.add_argument( + "job_ids", + nargs="+", + help="Job IDs to regenerate CDG for", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without making changes", + ) + args = parser.parse_args() + + storage = StorageService() + job_manager = JobManager() + + results = {} + for job_id in args.job_ids: + try: + success = regenerate_cdg_for_job(job_id, storage, job_manager, dry_run=args.dry_run) + results[job_id] = "OK" if success else "FAILED" + except Exception as e: + logger.exception(f"Error processing job {job_id}: {e}") + results[job_id] = f"ERROR: {e}" + + # Summary + logger.info("\n=== Summary ===") + for job_id, status in results.items(): + logger.info(f" {job_id}: {status}") + + # Exit with error if any failed + if any(s != "OK" for s in results.values()): + sys.exit(1) + + +if __name__ == "__main__": + main()