Fix missing CDG ZIP packages: fail-fast validation + remediation (#416)

## Summary

- **Systemic fix**: Orchestrator now validates CDG/TXT outputs after
packaging, before encoding. If `enable_cdg=True` but no CDG ZIP was
produced, the job fails immediately with a clear error instead of
silently completing without the package.
- **Remediation**: Created `scripts/regenerate_cdg.py` to regenerate and
distribute CDG/TXT packages for affected jobs (5b6aba25, 5161b069).
Packages uploaded to GCS, GDrive, and Dropbox.
- **Root cause**: Before b671b83f, `instrumental_selection: custom` had
no case in the orchestrator config builder — fell through to "Backing"
suffix, CDG generator looked for wrong filename, `FileNotFoundError` was
silently caught.

## Changes

| File | Change |
|------|--------|
| `backend/workers/video_worker_orchestrator.py` | Post-packaging
validation + LRC-missing check update |
| `backend/tests/test_video_worker_orchestrator.py` | 8 new tests in
`TestPackagingValidation` |
| `scripts/regenerate_cdg.py` | One-time remediation script (GCS +
Firestore + GDrive + Dropbox) |
| `docs/LESSONS-LEARNED.md` | Pipeline stage validation pattern |
| `docs/TROUBLESHOOTING.md` | CDG missing recovery runbook |
| `docs/README.md` | Updated recent changes |
| `pyproject.toml` | Version 0.119.6 → 0.119.7 |

## Test plan

- [x] 8 new tests in `TestPackagingValidation` (CDG fails, TXT fails,
combined CDG+TXT fails, CDG succeeds, CDG disabled, LRC missing raises
with CDG, LRC missing raises with TXT, LRC missing skips without CDG)
- [x] All 2062+ backend tests pass
- [x] Remediation script run successfully for both affected jobs
- [x] Verified CDG/TXT ZIPs in GCS, Firestore, GDrive, and Dropbox

> **CodeRabbit note**: 1 finding (unreachable `else` branch in
`_run_packaging` LRC check) — kept as defensive code since the test
`test_lrc_missing_skips_when_cdg_disabled` exercises this path directly.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.6 authored and GitHub committed 2026-02-27 19:13:22 +00:00
1 parent 028e9f40d4
commit 23a8b0fb98
7 files changed
+678 -9

No files matched your search

+232 -7
View File
@@ -250,8 +250,28 @@ class TestVideoWorkerOrchestratorPackaging:
"""Test packaging stage."""
@pytest.mark.asyncio
async def test_run_packaging_no_lrc(self):
"""Test packaging stage with no LRC file."""
async def test_run_packaging_no_lrc_cdg_disabled(self):
"""Test packaging stage with no LRC file and CDG disabled skips silently."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=False,
lrc_file_path=None,
)
orchestrator = VideoWorkerOrchestrator(config)
# Should not raise, just skip
await orchestrator._run_packaging()
assert orchestrator.result.final_karaoke_cdg_zip is None
@pytest.mark.asyncio
async def test_run_packaging_no_lrc_cdg_enabled_raises(self):
"""Test packaging stage raises when CDG enabled but no LRC file."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
@@ -264,10 +284,8 @@ class TestVideoWorkerOrchestratorPackaging:
)
orchestrator = VideoWorkerOrchestrator(config)
# Should not raise, just skip
await orchestrator._run_packaging()
assert orchestrator.result.final_karaoke_cdg_zip is None
with pytest.raises(RuntimeError, match="LRC file not available"):
await orchestrator._run_packaging()
@pytest.mark.asyncio
async def test_run_packaging_cdg(self):
@@ -368,6 +386,209 @@ class TestVideoWorkerOrchestratorPackaging:
"countdown_padding_seconds must be passed for CDG timestamp adjustment"
class TestPackagingValidation:
"""Test CDG/TXT packaging validation checks.
These tests verify that the orchestrator fails fast when CDG/TXT
generation is enabled but fails, rather than silently proceeding
to expensive encoding. This prevents jobs from completing without
their requested CDG/TXT packages.
Added after jobs 5b6aba25 and 5161b069 completed with enable_cdg=True
but no CDG ZIP was produced due to a FileNotFoundError being silently caught.
"""
@pytest.mark.asyncio
async def test_cdg_validation_fails_when_enabled_but_not_produced(self):
"""Test that orchestrator raises when CDG enabled but not produced."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=True,
)
orchestrator = VideoWorkerOrchestrator(config)
# Mock _run_packaging to NOT set result.final_karaoke_cdg_zip (simulates failure)
async def mock_packaging():
pass # CDG generation "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
result = await orchestrator.run()
assert result.success is False
assert "CDG generation was enabled but failed" in result.error_message
mock_encoding.assert_not_called() # Should not reach encoding
@pytest.mark.asyncio
async def test_txt_validation_fails_when_enabled_but_not_produced(self):
"""Test that orchestrator raises when TXT enabled but not produced."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_txt=True,
)
orchestrator = VideoWorkerOrchestrator(config)
async def mock_packaging():
pass # TXT generation "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
result = await orchestrator.run()
assert result.success is False
assert "TXT generation was enabled but failed" in result.error_message
mock_encoding.assert_not_called()
@pytest.mark.asyncio
async def test_pipeline_succeeds_when_cdg_disabled(self):
"""Test that pipeline succeeds normally when CDG is not enabled."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=False,
enable_txt=False,
)
orchestrator = VideoWorkerOrchestrator(config)
with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_pkg, \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock):
result = await orchestrator.run()
assert result.success is True
mock_pkg.assert_not_called() # Packaging skipped entirely
@pytest.mark.asyncio
async def test_cdg_validation_passes_when_cdg_produced(self):
"""Test that validation passes when CDG is successfully produced."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=True,
)
orchestrator = VideoWorkerOrchestrator(config)
async def mock_packaging_success():
orchestrator.result.final_karaoke_cdg_zip = "/output/cdg.zip"
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging_success), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \
patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock):
result = await orchestrator.run()
assert result.success is True
@pytest.mark.asyncio
async def test_both_cdg_and_txt_validation_fail_together(self):
"""Test that orchestrator catches CDG failure when both CDG and TXT enabled."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=True,
enable_txt=True,
)
orchestrator = VideoWorkerOrchestrator(config)
async def mock_packaging():
pass # Both CDG and TXT "failed" silently
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging), \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding:
result = await orchestrator.run()
assert result.success is False
# CDG validation is checked first
assert "CDG generation was enabled but failed" in result.error_message
mock_encoding.assert_not_called()
@pytest.mark.asyncio
async def test_lrc_missing_raises_when_cdg_enabled(self):
"""Test that missing LRC file raises when CDG is enabled."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=True,
lrc_file_path="/nonexistent/path.lrc",
)
orchestrator = VideoWorkerOrchestrator(config)
with pytest.raises(RuntimeError, match="LRC file not available"):
await orchestrator._run_packaging()
@pytest.mark.asyncio
async def test_lrc_missing_raises_when_txt_enabled(self):
"""Test that missing LRC file raises when TXT is enabled."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_txt=True,
lrc_file_path=None,
)
orchestrator = VideoWorkerOrchestrator(config)
with pytest.raises(RuntimeError, match="LRC file not available"):
await orchestrator._run_packaging()
@pytest.mark.asyncio
async def test_lrc_missing_skips_when_cdg_disabled(self):
"""Test that missing LRC file just skips packaging when CDG/TXT disabled."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
enable_cdg=False,
enable_txt=False,
lrc_file_path=None,
)
orchestrator = VideoWorkerOrchestrator(config)
# Should not raise, just skip
await orchestrator._run_packaging()
assert orchestrator.result.final_karaoke_cdg_zip is None
class TestVideoWorkerOrchestratorEncoding:
"""Test encoding stage."""
@@ -772,8 +993,12 @@ class TestVideoWorkerOrchestratorFullPipeline:
)
orchestrator = VideoWorkerOrchestrator(config)
# Mock packaging to set CDG result (validation requires it when enable_cdg=True)
async def mock_packaging_with_result():
orchestrator.result.final_karaoke_cdg_zip = "/output/cdg.zip"
# Mock all stages
with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_packaging, \
with patch.object(orchestrator, "_run_packaging", side_effect=mock_packaging_with_result) as mock_packaging, \
patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \
patch.object(orchestrator, "_run_organization", new_callable=AsyncMock) as mock_org, \
patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock) as mock_dist, \
@@ -228,6 +228,18 @@ class VideoWorkerOrchestrator:
if self.config.enable_cdg or self.config.enable_txt:
await self._run_packaging()
# Validate required packaging outputs before expensive encoding
if self.config.enable_cdg and not self.result.final_karaoke_cdg_zip:
raise RuntimeError(
"CDG generation was enabled but failed. "
"Check worker logs for CDG-related errors."
)
if self.config.enable_txt and not self.result.final_karaoke_txt_zip:
raise RuntimeError(
"TXT generation was enabled but failed. "
"Check worker logs for TXT-related errors."
)
# Stage 2: Encoding
await self._run_encoding()
@@ -259,6 +271,10 @@ class VideoWorkerOrchestrator:
self.job_log.info("Starting packaging stage (CDG/TXT)")
if not self.config.lrc_file_path or not os.path.isfile(self.config.lrc_file_path):
if self.config.enable_cdg or self.config.enable_txt:
raise RuntimeError(
f"CDG/TXT generation enabled but LRC file not available: {self.config.lrc_file_path}"
)
self.job_log.warning("No LRC file available, skipping CDG/TXT packaging")
return
+9
View File
@@ -58,6 +58,15 @@ When multiple code paths create jobs (file upload, audio search, webhooks), use
**Pattern**: When a function can fail, prefer exceptions over boolean returns. Silent failures accumulate into production bugs. Use centralized helpers for multi-step operations (state change + side effects) to prevent divergent implementations.
### Validate Pipeline Stage Outputs Before Proceeding (Feb 2026)
**What happened**: Jobs 5b6aba25 and 5161b069 completed with `enable_cdg=True` but no CDG ZIP was produced. The orchestrator silently caught a `FileNotFoundError` during CDG generation and proceeded to 11+ minutes of GPU encoding, distribution, and notifications — all for an incomplete output.
**Root cause**: CDG generation failed because `instrumental_selection: custom` had no case in the filename construction (fell through to "Backing" suffix). The `FileNotFoundError` was caught by a broad `except Exception` that just logged and continued, since "CDG is optional."
**Fix**: Add validation gates between pipeline stages. After packaging completes but before expensive encoding begins, verify that all enabled outputs were actually produced. If `enable_cdg=True` but no CDG ZIP exists, raise `RuntimeError` immediately. This fails fast, saves encoding costs, and makes the error visible.
**Pattern**: When a pipeline stage produces optional outputs that were explicitly requested, validate their existence before proceeding to the next stage. "Optional" means "can be disabled," not "can silently fail when enabled."
### Fix Both Sides of Dual Code Paths
When fixing a bug in a system with multiple code paths (e.g., legacy vs orchestrator, local vs cloud), verify ALL paths are fixed. PR #271 fixed the GCE worker to READ `instrumental_selection` but only checked the legacy path which was already SENDING it. The orchestrator path (production default) wasn't sending it. **Pattern**: If a component receives config from multiple callers, check ALL callers when fixing the receiving side. Write integration tests that cover each path.
+3 -1
View File
@@ -34,7 +34,7 @@
## Known Issues
- CDG format generation requires additional style configuration
(No known issues)
## Pending Work
@@ -42,6 +42,8 @@
## Recent Changes
- **CDG Packaging Fail-Fast Validation** (2026-02-27): Fixed silent CDG generation failures where jobs completed with `enable_cdg=True` but no CDG ZIP was produced. **Root cause**: Jobs 5b6aba25 and 5161b069 had `instrumental_selection: custom` but before commit b671b83f, the orchestrator had no `custom` case — it fell through to `"Backing"` suffix. The CDG generator then hit a `FileNotFoundError` looking for the wrong filename, which was silently caught. **Fix**: (1) Orchestrator now validates CDG/TXT outputs after packaging stage and before encoding — if `enable_cdg=True` but no CDG ZIP was produced, the pipeline fails fast instead of silently continuing to expensive encoding, (2) Missing LRC file now raises when CDG/TXT is enabled instead of silently skipping, (3) Added `scripts/regenerate_cdg.py` remediation script for affected jobs. Removed "CDG requires additional style configuration" known issue (the nomad theme has complete CDG styles). Added 7 new tests for validation logic.
- **GCE Encoding Pipeline Resilience** (2026-02-26): Made encoding pipeline resilient to Cloud Run deployment interruptions. A deployment killed a poller mid-encoding, leaving job a03ad39c permanently stuck at `encoding` status. Three fixes: (1) GCE worker `/encode` endpoint now idempotent — returns cached results for completed jobs, allows retry on failure, returns status for in-progress (previously hard 409 reject), (2) `encoding_service.py` handles cached/in_progress responses and falls back to status check on 409, (3) Health service detects jobs stuck in `encoding` >50 min. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#resilient-remote-worker-interactions-feb-2026).
- **Private (Non-Published) Tracks** (2026-02-24): Added `is_private` flag on jobs for private/non-published track production. Private jobs upload to Dropbox `Tracks-NonPublished` folder with `NOMADNP` brand prefix, skip YouTube and Google Drive entirely, and send completion emails without the YouTube section. Available as a checkbox on all job creation tabs and as an admin toggle (auto-deletes outputs when toggling a completed job to private). See [archive/2026-02-24-private-tracks-plan.md](archive/2026-02-24-private-tracks-plan.md).
+22
View File
@@ -33,6 +33,28 @@ curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/video" \
---
## CDG/TXT packages missing from completed job
**Cause:** Before v0.119.7, CDG/TXT generation failures were silently caught. Jobs would complete with `enable_cdg=True` but no CDG ZIP in outputs. Fixed in v0.119.7 with fail-fast validation — new jobs will now fail loudly if CDG/TXT generation fails.
**Recovery** (for jobs that already completed without CDG/TXT):
```bash
# Regenerate and distribute CDG/TXT packages for specific jobs
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
python -m scripts.regenerate_cdg JOB_ID [JOB_ID ...]
# Example:
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
python -m scripts.regenerate_cdg 5b6aba25 5161b069
```
The script is idempotent — re-running it skips regeneration if the CDG ZIP already exists in GCS and proceeds to any missing distribution steps (GDrive, Dropbox).
---
## GCE encoding worker on wrong wheel version
**Cause:** Worker picks up a new deploy but doesn't restart automatically.
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.119.6"
version = "0.119.7"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+395
View File
@@ -0,0 +1,395 @@
#!/usr/bin/env python3
"""
Regenerate CDG ZIP packages for jobs that completed without them.
This one-time remediation script handles the case where jobs completed with
enable_cdg=True but the CDG ZIP wasn't produced (e.g., due to the custom
instrumental path bug fixed in b671b83f).
For each job, the script:
1. Fetches job data from Firestore
2. Downloads LRC and instrumental audio from GCS
3. Loads theme CDG styles
4. Generates CDG package using PackagingService
5. Uploads CDG ZIP to GCS
6. Updates file_urls.packages.cdg_zip in Firestore
7. Uploads CDG ZIP to Google Drive (if GDrive was used)
8. Updates state_data.gdrive_files in Firestore
9. Uploads CDG/TXT ZIPs to Dropbox (if Dropbox was used)
Usage:
python -m scripts.regenerate_cdg JOB_ID [JOB_ID ...]
python -m scripts.regenerate_cdg 5b6aba25 5161b069
Requirements:
- GOOGLE_CLOUD_PROJECT=nomadkaraoke (or set via gcloud config)
- gcloud auth application-default login (for GCS/Firestore access)
"""
import argparse
import logging
import os
import sys
import tempfile
from pathlib import Path
# Add project root to path
sys.path.insert(0, str(Path(__file__).parent.parent))
os.environ.setdefault("GOOGLE_CLOUD_PROJECT", "nomadkaraoke")
from backend.services.storage_service import StorageService
from backend.services.job_manager import JobManager
from backend.services.packaging_service import PackagingService
from karaoke_gen.style_loader import get_cdg_format
from karaoke_gen.utils import sanitize_filename
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
logger = logging.getLogger("regenerate_cdg")
def get_instrumental_gcs_key(instrumental_selection: str) -> str:
"""Get the GCS file_urls.stems key for the instrumental selection."""
if instrumental_selection == "custom":
return "custom_instrumental"
elif instrumental_selection == "clean":
return "instrumental_clean"
else: # with_backing
return "instrumental_with_backing"
def get_instrumental_suffix(instrumental_selection: str) -> str:
"""Get the filename suffix for the instrumental selection."""
if instrumental_selection == "custom":
return "Custom"
elif instrumental_selection == "clean":
return "Clean"
else:
return "Backing"
def regenerate_cdg_for_job(job_id: str, storage: StorageService, job_manager: JobManager, dry_run: bool = False):
"""Regenerate CDG ZIP for a single job."""
logger.info(f"=== Processing job {job_id} ===")
# 1. Fetch job from Firestore
job = job_manager.get_job(job_id)
if not job:
logger.error(f"Job {job_id} not found in Firestore")
return False
logger.info(f"Job: {job.artist} - {job.title}")
logger.info(f" enable_cdg: {job.enable_cdg}")
logger.info(f" status: {job.status}")
if not job.enable_cdg:
logger.warning(f"Job {job_id} does not have enable_cdg=True, skipping")
return False
# Check if CDG already exists in GCS
existing_cdg = job.file_urls.get("packages", {}).get("cdg_zip")
cdg_already_in_gcs = existing_cdg and storage.file_exists(existing_cdg)
if cdg_already_in_gcs:
logger.info(f"CDG ZIP already exists in GCS: {existing_cdg}")
elif existing_cdg:
logger.warning("CDG ZIP in Firestore but NOT in GCS, will regenerate")
# Get instrumental selection
instrumental_selection = job.state_data.get("instrumental_selection", "clean")
logger.info(f" instrumental_selection: {instrumental_selection}")
# Get lyrics metadata for countdown padding
lyrics_metadata = job.state_data.get("lyrics_metadata", {})
lrc_has_countdown_padding = lyrics_metadata.get("has_countdown_padding", False)
countdown_padding_seconds = lyrics_metadata.get("countdown_padding_seconds", 3.0)
logger.info(f" countdown_padding: {lrc_has_countdown_padding} ({countdown_padding_seconds}s)")
# Check required files exist in file_urls
lrc_url = job.file_urls.get("lyrics", {}).get("lrc")
if not lrc_url:
logger.error(f"Job {job_id} has no LRC file URL")
return False
instrumental_key = get_instrumental_gcs_key(instrumental_selection)
instrumental_url = job.file_urls.get("stems", {}).get(instrumental_key)
if not instrumental_url:
logger.error(f"Job {job_id} has no instrumental URL for key '{instrumental_key}'")
return False
# Get style_params for CDG styles
style_params_url = job.style_params_gcs_path
if dry_run:
logger.info(f"DRY RUN: Would regenerate CDG for {job.artist} - {job.title}")
return True
# 2. Download or regenerate CDG/TXT packages
with tempfile.TemporaryDirectory(prefix=f"cdg-regen-{job_id}-") as temp_dir:
safe_artist = sanitize_filename(job.artist) if job.artist else "Unknown"
safe_title = sanitize_filename(job.title) if job.title else "Unknown"
base_name = f"{safe_artist} - {safe_title}"
instrumental_suffix = get_instrumental_suffix(instrumental_selection)
brand_code = getattr(job, "brand_code", None) or job.state_data.get("brand_code", "UNKNOWN")
cdg_zip_path = os.path.join(temp_dir, f"{base_name} (Final Karaoke CDG).zip")
txt_zip_path = os.path.join(temp_dir, f"{base_name} (Final Karaoke TXT).zip")
if cdg_already_in_gcs:
# Download existing CDG ZIP from GCS for distribution
logger.info("Downloading existing CDG ZIP from GCS for distribution...")
storage.download_file(existing_cdg, cdg_zip_path)
# Also download TXT if it exists
existing_txt = job.file_urls.get("packages", {}).get("txt_zip")
if existing_txt and storage.file_exists(existing_txt):
logger.info("Downloading existing TXT ZIP from GCS...")
storage.download_file(existing_txt, txt_zip_path)
else:
# Regenerate: download source files and generate packages
# Download LRC
lrc_path = os.path.join(temp_dir, f"{base_name} (Karaoke).lrc")
logger.info(f"Downloading LRC: {lrc_url}")
storage.download_file(lrc_url, lrc_path)
# Download instrumental
instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac")
logger.info(f"Downloading instrumental: {instrumental_url}")
storage.download_file(instrumental_url, instrumental_path)
# Download and parse style_params
import json
style_params_path = os.path.join(temp_dir, "style_params.json")
style_params = None
if style_params_url:
try:
logger.info(f"Downloading style params: {style_params_url}")
storage.download_file(style_params_url, style_params_path)
with open(style_params_path) as f:
style_params = json.load(f)
except Exception as e:
logger.warning(f"Failed to download style_params from {style_params_url}: {e}")
# Fallback: load theme style_params from GCS theme directory
if style_params is None:
theme_id = getattr(job, "theme_id", "nomad") or "nomad"
theme_path = f"themes/{theme_id}/style_params.json"
logger.info(f"Trying theme style_params: {theme_path}")
try:
storage.download_file(theme_path, style_params_path)
with open(style_params_path) as f:
style_params = json.load(f)
except Exception as e:
logger.error(f"Failed to load theme style_params: {e}")
return False
# Download CDG style assets (font, backgrounds) referenced in style_params
cdg_section = style_params.get("cdg", {})
style_dir = os.path.join(temp_dir, "style")
os.makedirs(style_dir, exist_ok=True)
# Resolve asset GCS paths. Per-job style_params have full paths like
# "themes/nomad/assets/AvenirNext-Bold.ttf". Theme-level style_params
# have bare filenames like "AvenirNext-Bold.ttf" that need to be
# resolved against the theme's assets directory.
theme_id = getattr(job, "theme_id", "nomad") or "nomad"
theme_assets_prefix = f"themes/{theme_id}/assets"
for asset_key in ["font_path", "instrumental_background", "title_screen_background", "outro_background"]:
asset_path = cdg_section.get(asset_key)
if not asset_path or not isinstance(asset_path, str) or os.path.isabs(asset_path):
continue
# Determine the GCS path — bare filenames need the theme prefix
if "/" not in asset_path:
gcs_path = f"{theme_assets_prefix}/{asset_path}"
else:
gcs_path = asset_path
local_path = os.path.join(style_dir, os.path.basename(asset_path))
try:
storage.download_file(gcs_path, local_path)
cdg_section[asset_key] = local_path
logger.info(f"Downloaded CDG asset {asset_key}: {gcs_path}")
except Exception as e:
logger.warning(f"Failed to download CDG asset {asset_key} from {gcs_path}: {e}")
# 3. Extract CDG styles
cdg_styles = get_cdg_format(style_params)
if not cdg_styles:
logger.error(f"No CDG styles found in theme for job {job_id}")
return False
logger.info("CDG styles loaded successfully")
# 4. Generate CDG package
packaging_service = PackagingService(
cdg_styles=cdg_styles,
dry_run=False,
non_interactive=True,
)
mp3_path = os.path.join(temp_dir, f"{base_name} (Karaoke).mp3")
cdg_path = os.path.join(temp_dir, f"{base_name} (Karaoke).cdg")
logger.info("Generating CDG package...")
zip_file, mp3_file, cdg_file = packaging_service.create_cdg_package(
lrc_file=lrc_path,
audio_file=instrumental_path,
output_zip_path=cdg_zip_path,
artist=job.artist,
title=job.title,
output_mp3_path=mp3_path,
output_cdg_path=cdg_path,
lrc_has_countdown_padding=lrc_has_countdown_padding,
countdown_padding_seconds=countdown_padding_seconds,
)
if not os.path.isfile(cdg_zip_path):
logger.error(f"CDG ZIP not created at expected path: {cdg_zip_path}")
return False
zip_size = os.path.getsize(cdg_zip_path)
logger.info(f"CDG ZIP created: {cdg_zip_path} ({zip_size / 1024:.1f} KB)")
# 5. Upload CDG ZIP to GCS
gcs_cdg_path = f"jobs/{job_id}/packages/cdg_zip.zip"
logger.info(f"Uploading CDG ZIP to GCS: {gcs_cdg_path}")
storage.upload_file(cdg_zip_path, gcs_cdg_path)
# 6. Update Firestore file_urls
logger.info("Updating Firestore file_urls.packages.cdg_zip")
job_manager.update_file_url(job_id, "packages", "cdg_zip", gcs_cdg_path)
# Also generate TXT if enabled
if job.enable_txt:
logger.info("Job also has enable_txt=True, generating TXT package...")
try:
txt_zip, txt_file = packaging_service.create_txt_package(
lrc_file=lrc_path,
mp3_file=mp3_path,
output_zip_path=txt_zip_path,
)
if os.path.isfile(txt_zip_path):
gcs_txt_path = f"jobs/{job_id}/packages/txt_zip.zip"
storage.upload_file(txt_zip_path, gcs_txt_path)
job_manager.update_file_url(job_id, "packages", "txt_zip", gcs_txt_path)
logger.info(f"TXT ZIP uploaded: {gcs_txt_path}")
except Exception as e:
logger.error(f"TXT generation failed: {e}")
# === Distribution steps (run for both new and existing packages) ===
# 7. Upload to GDrive if job uses it
gdrive_files = job.state_data.get("gdrive_files", {})
if gdrive_files and "cdg" not in gdrive_files:
logger.info("Uploading CDG ZIP to Google Drive...")
try:
from backend.services.gdrive_service import get_gdrive_service
gdrive = get_gdrive_service()
if gdrive.is_configured:
gdrive_folder_id = getattr(job, "gdrive_folder_id", None)
if gdrive_folder_id:
safe_base = sanitize_filename(f"{job.artist} - {job.title}")
filename_base = f"{brand_code} - {safe_base}"
cdg_folder_id = gdrive.get_or_create_folder(gdrive_folder_id, "CDG")
file_id = gdrive.upload_file(
cdg_zip_path,
cdg_folder_id,
f"{filename_base}.zip",
)
logger.info(f"Uploaded CDG to GDrive: {file_id}")
# 8. Update state_data.gdrive_files
gdrive_files["cdg"] = file_id
job_manager.update_job(job_id, {"state_data.gdrive_files": gdrive_files})
logger.info("Updated Firestore state_data.gdrive_files")
else:
logger.warning("No gdrive_folder_id on job, skipping GDrive upload")
else:
logger.warning("GDrive not configured, skipping upload")
except Exception as e:
logger.error(f"GDrive upload failed (non-fatal): {e}")
elif gdrive_files:
logger.info("GDrive already has CDG file, skipping GDrive upload")
else:
logger.info("No existing GDrive files, skipping GDrive upload")
# 9. Upload to Dropbox if job uses it
dropbox_path = job.dropbox_path
if dropbox_path:
logger.info("Uploading packages to Dropbox...")
try:
from backend.services.dropbox_service import get_dropbox_service
dropbox = get_dropbox_service()
if dropbox.is_configured:
safe_base = sanitize_filename(f"{job.artist} - {job.title}")
folder_name = f"{brand_code} - {safe_base}"
remote_folder = f"{dropbox_path}/{folder_name}"
# Upload CDG ZIP
if os.path.isfile(cdg_zip_path):
cdg_remote = f"{remote_folder}/{os.path.basename(cdg_zip_path)}"
dropbox.upload_file(cdg_zip_path, cdg_remote)
logger.info(f"Uploaded CDG to Dropbox: {cdg_remote}")
# Upload TXT ZIP
if os.path.isfile(txt_zip_path):
txt_remote = f"{remote_folder}/{os.path.basename(txt_zip_path)}"
dropbox.upload_file(txt_zip_path, txt_remote)
logger.info(f"Uploaded TXT to Dropbox: {txt_remote}")
else:
logger.warning("Dropbox not configured, skipping upload")
except Exception as e:
logger.error(f"Dropbox upload failed (non-fatal): {e}")
else:
logger.info("No dropbox_path on job, skipping Dropbox upload")
logger.info(f"=== Job {job_id} CDG regeneration complete ===")
return True
def main():
parser = argparse.ArgumentParser(
description="Regenerate CDG ZIP packages for jobs that completed without them."
)
parser.add_argument(
"job_ids",
nargs="+",
help="Job IDs to regenerate CDG for",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be done without making changes",
)
args = parser.parse_args()
storage = StorageService()
job_manager = JobManager()
results = {}
for job_id in args.job_ids:
try:
success = regenerate_cdg_for_job(job_id, storage, job_manager, dry_run=args.dry_run)
results[job_id] = "OK" if success else "FAILED"
except Exception as e:
logger.exception(f"Error processing job {job_id}: {e}")
results[job_id] = f"ERROR: {e}"
# Summary
logger.info("\n=== Summary ===")
for job_id, status in results.items():
logger.info(f" {job_id}: {status}")
# Exit with error if any failed
if any(s != "OK" for s in results.values()):
sys.exit(1)
if __name__ == "__main__":
main()