From c1588992ceffa2656d2cb41301127daa305c1c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Sun, 4 Jan 2026 00:25:23 -0500 Subject: [PATCH] refactor: Unify video worker pipeline with new orchestrator (#182) Resolves GCE vs KaraokeFinalise code path divergence that caused features like YouTube upload, Discord notifications, and CDG/TXT packaging to be missing from the GCE encoding path. - New services: youtube_upload_service, discord_service, packaging_service, local_encoding_service, encoding_interface - New VideoWorkerOrchestrator coordinates all stages (packaging, encoding, distribution, notifications) - Works with any encoding backend (GCE or local) - 139 new tests, 976 total tests passing - Feature flag USE_NEW_ORCHESTRATOR (default: true) with rollback capability --- .../plans/video-worker-stage-refactor.plan.md | 401 ++++++++++ backend/services/discord_service.py | 172 +++++ backend/services/encoding_interface.py | 418 +++++++++++ backend/services/local_encoding_service.py | 590 +++++++++++++++ backend/services/packaging_service.py | 287 +++++++ backend/services/youtube_upload_service.py | 445 +++++++++++ backend/tests/test_discord_service.py | 244 ++++++ backend/tests/test_encoding_interface.py | 351 +++++++++ backend/tests/test_local_encoding_service.py | 423 +++++++++++ backend/tests/test_packaging_service.py | 382 ++++++++++ .../tests/test_video_worker_orchestrator.py | 698 ++++++++++++++++++ backend/tests/test_youtube_upload_service.py | 568 ++++++++++++++ backend/workers/video_worker.py | 276 ++++++- backend/workers/video_worker_orchestrator.py | 628 ++++++++++++++++ pyproject.toml | 2 +- 15 files changed, 5865 insertions(+), 20 deletions(-) create mode 100644 .claude/plans/video-worker-stage-refactor.plan.md create mode 100644 backend/services/discord_service.py create mode 100644 backend/services/encoding_interface.py create mode 100644 backend/services/local_encoding_service.py create mode 100644 backend/services/packaging_service.py create mode 100644 backend/services/youtube_upload_service.py create mode 100644 backend/tests/test_discord_service.py create mode 100644 backend/tests/test_encoding_interface.py create mode 100644 backend/tests/test_local_encoding_service.py create mode 100644 backend/tests/test_packaging_service.py create mode 100644 backend/tests/test_video_worker_orchestrator.py create mode 100644 backend/tests/test_youtube_upload_service.py create mode 100644 backend/workers/video_worker_orchestrator.py diff --git a/.claude/plans/video-worker-stage-refactor.plan.md b/.claude/plans/video-worker-stage-refactor.plan.md new file mode 100644 index 00000000..dfc0ea3c --- /dev/null +++ b/.claude/plans/video-worker-stage-refactor.plan.md @@ -0,0 +1,401 @@ +# Plan: Video Worker Stage Refactor + +**Created:** 2026-01-03 +**Branch:** feat/sess-20260103-2250-investigate-youtube-upload +**Status:** COMPLETE - Ready for PR +**Last Updated:** 2026-01-04 + +## Overview + +Refactor `video_worker.py` and `KaraokeFinalise` into separate, composable stages following SOLID principles. This eliminates the GCE vs KaraokeFinalise code path divergence that causes features like YouTube upload to be missing from the GCE encoding path. + +## Three Execution Modes + +Understanding the three ways this system runs is critical to the refactor: + +### 1. Local CLI (`karaoke-gen`) +- Runs entirely on user's machine +- KaraokeFinalise.process() orchestrates everything +- Writes directly to local Dropbox sync folder (e.g., `~/Dropbox/Karaoke/Tracks-Organized/`) +- **Produces full output structure** with stems/, lyrics/, visualizations/, etc. +- **This is the source of truth for output format** + +### 2. Remote CLI (`karaoke-gen-remote`) +- User's CLI talks to cloud backend (api.nomadkaraoke.com) +- Cloud does ALL processing (audio, lyrics, video encoding) +- CLI just monitors job status and downloads final outputs +- With Dropbox configured: cloud uploads full structure to user's Dropbox +- Without Dropbox: user downloads individual files + +### 3. Web Frontend (`gen.nomadkaraoke.com`) +- Same cloud backend as remote CLI +- User sees: + - YouTube video link + - Dropbox folder link + - Direct download: final karaoke video (720p) + - Direct download: 4K video + - Direct download: "with vocals" video + - Direct download: CDG zip (if enabled) + +## Problem Statement + +Currently, `video_worker.py` has two completely divergent code paths: + +```python +if use_gce_encoding: + # GCE PATH: Only encoding, missing YouTube/Discord/CDG/TXT + result = await _encode_via_gce(...) + # result DOES NOT include youtube_url! +else: + # STANDARD PATH: KaraokeFinalise does everything + finalise = KaraokeFinalise(...) + result = finalise.process() + # result includes youtube_url from upload +``` + +### What GCE Encoding Broke + +| Feature | KaraokeFinalise Path | GCE Path | +|---------|---------------------|----------| +| Video encoding | ✅ 4 formats | ✅ 4 formats | +| YouTube upload | ✅ | ❌ **BROKEN** | +| Discord notification | ✅ | ❌ **BROKEN** | +| CDG/TXT packaging | ✅ | ❌ **BROKEN** | +| Brand code generation | ✅ | ✅ (via Dropbox service) | +| Dropbox upload | ✅ (rclone) | ✅ (native SDK) | +| Google Drive upload | ❌ | ✅ (native SDK) | + +### Root Cause + +`KaraokeFinalise.process()` is a monolith that handles: +1. CDG/TXT packaging +2. Video encoding (the CPU-intensive part) +3. YouTube upload +4. Discord notifications +5. Folder organization + +The GCE encoding was added as a bypass around the entire class, rather than replacing just the encoding step. + +## Requirements + +### Functional Requirements +- [ ] YouTube upload works regardless of encoding backend (GCE or local) +- [ ] Discord notifications work regardless of encoding backend +- [ ] CDG/TXT packaging works regardless of encoding backend +- [ ] Brand code generation works consistently +- [ ] Dropbox/Google Drive upload works (already works in both paths) +- [ ] All existing CLI functionality continues to work +- [ ] Output structure matches local CLI exactly + +### Output Structure Parity + +Cloud backend must produce identical output to local CLI: + +``` +NOMAD-XXXX - Artist - Title/ +├── Artist - Title (Title).mov +├── Artist - Title (Title).jpg +├── Artist - Title (End).mov +├── Artist - Title (End).jpg +├── Artist - Title (Final Karaoke Lossless 4k).mp4 +├── Artist - Title (Final Karaoke Lossless 4k).mkv +├── Artist - Title (Final Karaoke Lossy 4k).mp4 +├── Artist - Title (Final Karaoke Lossy 720p).mp4 +├── Artist - Title (Final Karaoke CDG).zip +├── Artist - Title (Final Karaoke TXT).zip +├── Artist - Title (Instrumental {model}).flac +├── Artist - Title (Instrumental +BV {model}).flac +├── Artist - Title (Karaoke).mp4 +├── Artist - Title (Karaoke).cdg +├── Artist - Title (Karaoke).mp3 +├── Artist - Title (Karaoke).lrc +├── Artist - Title (With Vocals).mp4 +├── stems/ +│ ├── Artist - Title (Instrumental {model}).flac +│ ├── Artist - Title (Vocals {model}).flac +│ ├── Artist - Title (Lead Vocals {model}).flac +│ ├── Artist - Title (Backing Vocals {model}).flac +│ └── ... (other stems) +├── lyrics/ +│ ├── Artist - Title (Karaoke).lrc +│ ├── Artist - Title (Karaoke).ass +│ ├── Artist - Title (Karaoke).srt +│ └── ... (corrections JSONs) +├── previews/ +├── visualizations/ +└── styles_assets/ +``` + +### Non-Functional Requirements +- [ ] Single Responsibility: Each stage has one job +- [ ] Open/Closed: Easy to add new encoding backends without modifying stages +- [ ] Testable: Each stage can be unit tested in isolation +- [ ] No duplicate code between paths +- [ ] Backwards compatible with existing KaraokeFinalise CLI usage + +## Technical Approach + +### Architecture: Stage-Based Pipeline + +``` +video_worker.py (VideoWorkerOrchestrator) +│ +├── 1. EncodingStage (interface) +│ ├── LocalEncodingService (wraps KaraokeFinalise encoding methods) +│ └── GCEEncodingService (existing encoding_service.py) +│ +├── 2. PackagingStage +│ ├── CDGPackager (extracts from KaraokeFinalise) +│ └── TXTPackager (extracts from KaraokeFinalise) +│ +├── 3. DistributionStage +│ ├── YouTubeUploadService (extracts from KaraokeFinalise) +│ ├── DropboxService (existing) +│ ├── GoogleDriveService (existing) +│ └── DiscordService (extracts from KaraokeFinalise) +│ +└── 4. OrganizationStage + └── BrandCodeService (extracts from KaraokeFinalise + DropboxService) +``` + +### Key Design Decisions + +1. **Extract, Don't Rewrite**: Pull methods out of KaraokeFinalise into services, don't rewrite from scratch +2. **Keep KaraokeFinalise for CLI**: Local CLI still uses KaraokeFinalise.process() - we extract services that it can also use +3. **Interface-Based Encoding**: EncodingStage interface allows swapping GCE/local without changing orchestration +4. **Existing Service Pattern**: Follow singleton + factory pattern already used in codebase +5. **Local CLI stays sync**: Local karaoke-gen CLI remains synchronous as it is today + +### Data Flow + +``` +JobConfig (from Firestore) + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ VideoWorkerOrchestrator │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ 1. Setup: Download files, prepare directories ││ +│ │ - Download title.mov, end.mov, with_vocals.mkv ││ +│ │ - Download instrumental audio ││ +│ │ - Prepare temp directory ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ 2. Packaging: CDG/TXT (if enabled) - runs BEFORE encoding ││ +│ │ Reason: CDG depends on LRC file which is available ││ +│ │ Input: LRC file, instrumental audio ││ +│ │ Output: CDG.zip, TXT.zip, .cdg, .mp3 ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ 3. Encoding: GCE or Local (via interface) ││ +│ │ Input: title.mov, karaoke.mov, end.mov ││ +│ │ Output: 4K lossless, 4K lossy, 720p, MKV ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ 4. Organization: Brand code, folder structure ││ +│ │ - Generate brand code (NOMAD-XXXX) ││ +│ │ - Prepare full output directory structure ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ 5. Distribution: Upload to destinations ││ +│ │ - YouTube upload (if enabled) ││ +│ │ - Dropbox upload (if configured) ││ +│ │ - Google Drive upload (if configured) ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ 6. Notification: Discord, Email ││ +│ │ Input: YouTube URL, brand code, Dropbox link ││ +│ └─────────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +Result (youtube_url, brand_code, dropbox_link, file_paths, etc.) +``` + +## Implementation Steps + +### Phase 1: Extract Services from KaraokeFinalise (Non-Breaking) ✅ COMPLETE + +These services can be used by both the cloud orchestrator AND KaraokeFinalise. + +1. [x] **Create `backend/services/youtube_upload_service.py`** ✅ + - Extract YouTube upload logic from KaraokeFinalise + - Methods: `upload_video()`, `check_duplicate()`, `delete_video()`, `truncate_title()` + - Supports pre-loaded credentials (server-side) or client secrets file (CLI) + - 23 unit tests with mocked YouTube API + +2. [x] **Create `backend/services/discord_service.py`** ✅ + - Extract Discord notification from KaraokeFinalise + - Methods: `post_message()`, `post_video_notification()`, `is_enabled()` + - Simple HTTP POST to webhook with validation + - 20 unit tests with mocked requests + +3. [x] **Create `backend/services/packaging_service.py`** ✅ + - Extract CDG/TXT generation from KaraokeFinalise + - Methods: `create_cdg_package()`, `create_txt_package()` + - Uses lyrics_converter and CDGGenerator (existing dependencies) + - 18 unit tests with temp files + +4. [x] **Create `backend/services/local_encoding_service.py`** ✅ + - Extract encoding methods from KaraokeFinalise + - Methods: `encode_all_formats()`, individual encoding methods + - Hardware acceleration detection with software fallback + - 28 unit tests with mocked subprocess + +### Phase 2: Create Encoding Interface ✅ COMPLETE + +5. [x] **Create `backend/services/encoding_interface.py`** ✅ + - Define ABC `EncodingBackend` with methods: `encode()`, `is_available()`, `get_status()` + - `EncodingInput` and `EncodingOutput` dataclasses for config/results + - Factory function `get_encoding_backend()` with "local", "gce", "auto" options + +6. [x] **`LocalEncodingBackend`** ✅ + - Implements `EncodingBackend` interface + - Wraps `LocalEncodingService` with `asyncio.to_thread()` + - 17 unit tests + +7. [x] **`GCEEncodingBackend`** ✅ + - Implements `EncodingBackend` interface + - Wraps existing `EncodingService` (encoding_service.py) + - Requires GCS paths in options for cloud execution + - 6 additional unit tests + +### Phase 3: Create Orchestrator ✅ COMPLETE + +8. [x] **Create `backend/workers/video_worker_orchestrator.py`** ✅ + - New class that coordinates all stages + - Replaces the branching logic in `generate_video()` + - Methods: + - `__init__(config, job_manager, storage, job_logger)` + - `async run() -> OrchestratorResult` + - `_run_packaging()` - CDG/TXT + - `_run_encoding()` - GCE or local via EncodingBackend interface + - `_run_organization()` - Brand code, folder structure + - `_run_distribution()` - YouTube, Dropbox, GDrive + - `_run_notifications()` - Discord + - 27 unit tests (test_video_worker_orchestrator.py) + +9. [x] **Update `backend/workers/video_worker.py`** ✅ + - Added `USE_NEW_ORCHESTRATOR` feature flag (default: true) + - `generate_video()` routes to orchestrator or legacy path + - `generate_video_orchestrated()` - new unified pipeline + - `generate_video_legacy()` - original code for rollback + - Legacy GCE and KaraokeFinalise paths preserved for safety + +### Phase 4: Update KaraokeFinalise to Use Services (OPTIONAL) + +10. [ ] **Update `karaoke_gen/karaoke_finalise/karaoke_finalise.py`** + - Import and use the new services + - Replace inline YouTube/Discord/CDG code with service calls + - Keep process() working for CLI backwards compatibility + - Services become the single source of truth + - **NOTE**: This is optional for MVP - the orchestrator already uses the extracted services directly + +### Phase 5: Testing & Validation ✅ COMPLETE + +11. [x] **Add unit tests for each service** ✅ + - test_youtube_upload_service.py (23 tests) + - test_discord_service.py (20 tests) + - test_packaging_service.py (18 tests) + - test_local_encoding_service.py (28 tests) + - test_encoding_interface.py (23 tests) + - test_video_worker_orchestrator.py (27 tests) + - **Total: 139 new tests** + +12. [x] **Full test suite passes** ✅ + - 976 tests passed (86 skipped - emulator/integration tests) + - No regressions from changes + +13. [ ] **Manual E2E testing** (After deployment) + - [ ] GCE encoding job with YouTube upload enabled + - [ ] Local encoding job with YouTube upload enabled + - [ ] CLI karaoke-gen with all features + - [ ] Job with CDG/TXT enabled + - [ ] Verify output structure matches local CLI + +## Files Created/Modified + +| File | Action | Description | Status | +|------|--------|-------------|--------| +| `backend/services/youtube_upload_service.py` | Create | YouTube upload logic (~446 lines) | ✅ | +| `backend/services/discord_service.py` | Create | Discord notification service (~167 lines) | ✅ | +| `backend/services/packaging_service.py` | Create | CDG/TXT package generation (~266 lines) | ✅ | +| `backend/services/local_encoding_service.py` | Create | Local FFmpeg encoding service (~512 lines) | ✅ | +| `backend/services/encoding_interface.py` | Create | Abstract interface + backends (~418 lines) | ✅ | +| `backend/workers/video_worker_orchestrator.py` | Create | Stage orchestration logic (~550 lines) | ✅ | +| `backend/workers/video_worker.py` | Modify | Added orchestrator path + feature flag | ✅ | +| `backend/tests/test_youtube_upload_service.py` | Create | 23 unit tests | ✅ | +| `backend/tests/test_discord_service.py` | Create | 20 unit tests | ✅ | +| `backend/tests/test_packaging_service.py` | Create | 18 unit tests | ✅ | +| `backend/tests/test_local_encoding_service.py` | Create | 28 unit tests | ✅ | +| `backend/tests/test_encoding_interface.py` | Create | 23 unit tests | ✅ | +| `backend/tests/test_video_worker_orchestrator.py` | Create | 27 unit tests | ✅ | +| `karaoke_gen/karaoke_finalise/karaoke_finalise.py` | Modify | Use extracted services (optional) | Deferred | + +## Testing Strategy + +### Unit Tests +- **youtube_upload_service**: Mock YouTube API, test upload/delete/duplicate check +- **discord_service**: Mock requests, test webhook POST +- **packaging_service**: Use temp files, test CDG/TXT generation +- **local_encoding_service**: Mock subprocess, test FFmpeg command construction + +### Integration Tests +- **video_worker_orchestrator**: Mock all services, test stage sequencing +- **video_worker**: Test with mocked orchestrator + +### E2E Verification +- [ ] GCE encoding job with YouTube upload enabled → YouTube link appears +- [ ] Local encoding job with YouTube upload enabled → Same behavior +- [ ] CLI `karaoke-gen` with all features → No regression +- [ ] Job with CDG/TXT enabled → Packages generated +- [ ] Dropbox output structure → Matches local CLI exactly + +## Open Questions + +- [x] Should we keep KaraokeFinalise.process() or deprecate it? + - **Decision**: Keep it for CLI backwards compatibility. It will use the new services internally. + +- [x] Should local_encoding_service run FFmpeg sync or async? + - **Decision**: Sync implementation, async wrapper via `asyncio.to_thread()` for interface compatibility. + +- [x] Order of packaging vs encoding? + - **Decision**: CDG/TXT runs BEFORE encoding (depends on LRC file, not videos). KaraokeFinalise does it this way too. + +## Rollback Plan + +1. The refactor is additive - new services don't replace old code until orchestrator is wired up +2. Feature flag: `USE_NEW_ORCHESTRATOR=true` to enable new code path +3. If issues found in production: + - Set `USE_NEW_ORCHESTRATOR=false` to revert to old behavior + - Old `generate_video()` code preserved behind feature flag +4. After 1 week stable, remove feature flag and old code + +## Success Criteria + +1. Job `5b78ad46` scenario (GCE + YouTube) would upload to YouTube +2. All 803+ existing tests pass +3. E2E happy path test passes +4. No regression in local CLI functionality +5. Output structure matches local CLI exactly +6. Code coverage maintained or improved + +## Estimated Scope + +- **Phase 1** (Extract Services): 4 new service files + tests (~400-600 lines each) +- **Phase 2** (Interface): 1 new file, 2 modifications (~100 lines) +- **Phase 3** (Orchestrator): 2 files (~300-400 lines) +- **Phase 4** (KaraokeFinalise): 1 modification (~100 lines of changes) +- **Phase 5** (Testing): Integration tests + manual verification + +This is a medium-sized refactor that can be done incrementally. Each phase produces working, tested code. diff --git a/backend/services/discord_service.py b/backend/services/discord_service.py new file mode 100644 index 00000000..404ea69a --- /dev/null +++ b/backend/services/discord_service.py @@ -0,0 +1,172 @@ +""" +Discord Notification Service. + +Provides Discord webhook notification functionality, extracted from KaraokeFinalise +for use by both the cloud backend (video_worker) and local CLI. + +This service handles: +- Posting messages to Discord webhooks +- Video upload notifications +- Validation of webhook URLs +""" + +import logging +import re +from typing import Optional + +import requests + +logger = logging.getLogger(__name__) + + +class DiscordNotificationService: + """ + Service for sending Discord webhook notifications. + + Supports posting messages to Discord channels via webhooks, + commonly used to notify when new karaoke videos are uploaded. + """ + + # Discord webhook URL pattern + WEBHOOK_URL_PATTERN = re.compile( + r"^https://discord\.com/api/webhooks/\d+/[a-zA-Z0-9_-]+$" + ) + + def __init__( + self, + webhook_url: Optional[str] = None, + dry_run: bool = False, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize the Discord notification service. + + Args: + webhook_url: Discord webhook URL for posting notifications + dry_run: If True, log actions without performing them + logger: Optional logger instance + """ + self.webhook_url = webhook_url + self.dry_run = dry_run + self.logger = logger or logging.getLogger(__name__) + + # Validate webhook URL if provided + if webhook_url: + self._validate_webhook_url(webhook_url) + + def _validate_webhook_url(self, url: str) -> None: + """ + Validate that the webhook URL is a valid Discord webhook URL. + + Args: + url: URL to validate + + Raises: + ValueError: If URL is not a valid Discord webhook URL + """ + url = url.strip() + if not url.startswith("https://discord.com/api/webhooks/"): + raise ValueError( + f"Invalid Discord webhook URL: {url}. " + "URL must start with 'https://discord.com/api/webhooks/'" + ) + + def post_message( + self, + message: str, + webhook_url: Optional[str] = None, + ) -> bool: + """ + Post a message to a Discord channel via webhook. + + Args: + message: The message content to post + webhook_url: Optional webhook URL (uses instance URL if not provided) + + Returns: + True if message was posted successfully, False otherwise + + Raises: + ValueError: If no webhook URL is available + requests.HTTPError: If the webhook request fails + """ + url = webhook_url or self.webhook_url + if not url: + raise ValueError("No Discord webhook URL provided") + + url = url.strip() + self._validate_webhook_url(url) + + if self.dry_run: + self.logger.info( + f"DRY RUN: Would post Discord message: '{message}' to {url}" + ) + return True + + self.logger.info(f"Posting message to Discord webhook...") + data = {"content": message} + response = requests.post(url, json=data, timeout=30) + response.raise_for_status() + self.logger.info("Message posted to Discord successfully") + return True + + def post_video_notification( + self, + youtube_url: str, + webhook_url: Optional[str] = None, + ) -> bool: + """ + Post a notification about a new video upload. + + Args: + youtube_url: The YouTube URL of the uploaded video + webhook_url: Optional webhook URL (uses instance URL if not provided) + + Returns: + True if notification was posted successfully, False otherwise + """ + if not youtube_url: + self.logger.info("Skipping Discord notification - no YouTube URL available") + return False + + message = f"New upload: {youtube_url}" + return self.post_message(message, webhook_url) + + def is_enabled(self) -> bool: + """ + Check if Discord notifications are enabled. + + Returns: + True if a valid webhook URL is configured + """ + return bool(self.webhook_url) + + +# Singleton instance and factory function (following existing service pattern) +_discord_notification_service: Optional[DiscordNotificationService] = None + + +def get_discord_notification_service( + webhook_url: Optional[str] = None, + **kwargs +) -> DiscordNotificationService: + """ + Get a Discord notification service instance. + + Args: + webhook_url: Discord webhook URL for notifications + **kwargs: Additional arguments passed to DiscordNotificationService + + Returns: + DiscordNotificationService instance + """ + global _discord_notification_service + + # Create new instance if webhook URL changed or not yet created + if _discord_notification_service is None or webhook_url: + _discord_notification_service = DiscordNotificationService( + webhook_url=webhook_url, + **kwargs + ) + + return _discord_notification_service diff --git a/backend/services/encoding_interface.py b/backend/services/encoding_interface.py new file mode 100644 index 00000000..d9b47e6e --- /dev/null +++ b/backend/services/encoding_interface.py @@ -0,0 +1,418 @@ +""" +Encoding Interface. + +Defines the abstract interface for video encoding backends, allowing +the video worker orchestrator to use either GCE or local encoding +interchangeably. + +This follows the Strategy pattern - different encoding implementations +can be swapped without changing the orchestration logic. +""" + +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Any + +logger = logging.getLogger(__name__) + + +@dataclass +class EncodingInput: + """ + Input configuration for video encoding. + + Contains all the paths and metadata needed to encode a karaoke video. + """ + # Required input files + title_video_path: str # Title card video (MOV) + karaoke_video_path: str # Main karaoke video with vocals (MOV/MKV) + instrumental_audio_path: str # Instrumental audio track (FLAC) + + # Optional input files + end_video_path: Optional[str] = None # End credits video (MOV) + + # Metadata for output naming + artist: str = "" + title: str = "" + brand_code: Optional[str] = None + + # Output directory + output_dir: str = "" + + # Additional options + options: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class EncodingOutput: + """ + Output from video encoding. + + Contains paths to all generated video files and status information. + """ + success: bool + error_message: Optional[str] = None + + # Output file paths (relative to output_dir or absolute) + karaoke_mp4_path: Optional[str] = None # Karaoke video with instrumental audio + with_vocals_mp4_path: Optional[str] = None # With vocals MP4 version + lossless_4k_mp4_path: Optional[str] = None # Final lossless 4K MP4 + lossy_4k_mp4_path: Optional[str] = None # Final lossy 4K MP4 + lossless_mkv_path: Optional[str] = None # MKV with FLAC (for YouTube) + lossy_720p_mp4_path: Optional[str] = None # 720p web version + + # All output files as a dict for convenience + output_files: Dict[str, str] = field(default_factory=dict) + + # Encoding metadata + encoding_time_seconds: Optional[float] = None + encoding_backend: Optional[str] = None # "gce" or "local" + + +class EncodingBackend(ABC): + """ + Abstract base class for video encoding backends. + + Implementations: + - GCEEncodingBackend: Cloud-based encoding using Google Compute Engine + - LocalEncodingBackend: Local FFmpeg-based encoding + """ + + @property + @abstractmethod + def name(self) -> str: + """Return the name of this encoding backend.""" + pass + + @abstractmethod + async def encode(self, input_config: EncodingInput) -> EncodingOutput: + """ + Encode video files according to the input configuration. + + This method should: + 1. Create karaoke video with instrumental audio + 2. Convert with-vocals video to MP4 if needed + 3. Encode lossless 4K MP4 (concatenating title + karaoke + end) + 4. Encode lossy 4K MP4 with AAC audio + 5. Create MKV with FLAC audio for YouTube + 6. Encode 720p version for web + + Args: + input_config: EncodingInput with all required paths and options + + Returns: + EncodingOutput with paths to generated files and status + """ + pass + + @abstractmethod + async def is_available(self) -> bool: + """ + Check if this encoding backend is available and configured. + + Returns: + True if the backend can be used, False otherwise + """ + pass + + @abstractmethod + async def get_status(self) -> Dict[str, Any]: + """ + Get the current status of the encoding backend. + + Returns: + Dict with status information (availability, queue length, etc.) + """ + pass + + +class LocalEncodingBackend(EncodingBackend): + """ + Local FFmpeg-based encoding backend. + + Wraps the LocalEncodingService to implement the EncodingBackend interface. + Uses asyncio.to_thread() to run synchronous FFmpeg operations asynchronously. + """ + + def __init__( + self, + dry_run: bool = False, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize the local encoding backend. + + Args: + dry_run: If True, log operations without executing + logger: Optional logger instance + """ + self.dry_run = dry_run + self.logger = logger or logging.getLogger(__name__) + self._service = None + + @property + def name(self) -> str: + return "local" + + def _get_service(self): + """Lazy-load the local encoding service.""" + if self._service is None: + from backend.services.local_encoding_service import LocalEncodingService + self._service = LocalEncodingService( + dry_run=self.dry_run, + logger=self.logger + ) + return self._service + + async def encode(self, input_config: EncodingInput) -> EncodingOutput: + """ + Encode video using local FFmpeg. + + Args: + input_config: Encoding input configuration + + Returns: + EncodingOutput with results + """ + import asyncio + import time + from backend.services.local_encoding_service import EncodingConfig + + start_time = time.time() + + # Build output file paths + base_name = f"{input_config.artist} - {input_config.title}" + output_dir = input_config.output_dir or "." + + import os + config = EncodingConfig( + title_video=input_config.title_video_path, + karaoke_video=input_config.karaoke_video_path, + instrumental_audio=input_config.instrumental_audio_path, + end_video=input_config.end_video_path, + output_karaoke_mp4=os.path.join(output_dir, f"{base_name} (Karaoke).mp4"), + output_with_vocals_mp4=os.path.join(output_dir, f"{base_name} (With Vocals).mp4"), + output_lossless_4k_mp4=os.path.join(output_dir, f"{base_name} (Final Karaoke Lossless 4k).mp4"), + output_lossy_4k_mp4=os.path.join(output_dir, f"{base_name} (Final Karaoke Lossy 4k).mp4"), + output_lossless_mkv=os.path.join(output_dir, f"{base_name} (Final Karaoke Lossless 4k).mkv"), + output_720p_mp4=os.path.join(output_dir, f"{base_name} (Final Karaoke Lossy 720p).mp4"), + ) + + # Run encoding in thread pool to avoid blocking + service = self._get_service() + result = await asyncio.to_thread(service.encode_all_formats, config) + + encoding_time = time.time() - start_time + + if result.success: + return EncodingOutput( + success=True, + karaoke_mp4_path=config.output_karaoke_mp4, + with_vocals_mp4_path=config.output_with_vocals_mp4, + lossless_4k_mp4_path=config.output_lossless_4k_mp4, + lossy_4k_mp4_path=config.output_lossy_4k_mp4, + lossless_mkv_path=config.output_lossless_mkv, + lossy_720p_mp4_path=config.output_720p_mp4, + output_files=result.output_files, + encoding_time_seconds=encoding_time, + encoding_backend=self.name + ) + else: + return EncodingOutput( + success=False, + error_message=result.error, + output_files=result.output_files, + encoding_time_seconds=encoding_time, + encoding_backend=self.name + ) + + async def is_available(self) -> bool: + """Check if local encoding is available (FFmpeg installed).""" + import subprocess + import asyncio + + try: + result = await asyncio.to_thread( + subprocess.run, + ["ffmpeg", "-version"], + capture_output=True, + timeout=10 + ) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + async def get_status(self) -> Dict[str, Any]: + """Get local encoding status.""" + available = await self.is_available() + service = self._get_service() + + return { + "backend": self.name, + "available": available, + "hwaccel_available": service.hwaccel_available if available else False, + "video_encoder": service.video_encoder if available else None, + } + + +class GCEEncodingBackend(EncodingBackend): + """ + GCE-based encoding backend. + + Wraps the existing EncodingService to implement the EncodingBackend interface. + Submits jobs to a remote GCE worker for high-performance encoding. + """ + + def __init__( + self, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize the GCE encoding backend. + + Args: + logger: Optional logger instance + """ + self.logger = logger or logging.getLogger(__name__) + self._service = None + + @property + def name(self) -> str: + return "gce" + + def _get_service(self): + """Lazy-load the GCE encoding service.""" + if self._service is None: + from backend.services.encoding_service import get_encoding_service + self._service = get_encoding_service() + return self._service + + async def encode(self, input_config: EncodingInput) -> EncodingOutput: + """ + Encode video using GCE worker. + + Note: GCE encoding requires files to be in GCS, so this method + expects input_config.options to contain 'input_gcs_path' and + 'output_gcs_path'. + + Args: + input_config: Encoding input configuration + + Returns: + EncodingOutput with results + """ + import time + + start_time = time.time() + service = self._get_service() + + # GCE requires GCS paths + input_gcs_path = input_config.options.get("input_gcs_path") + output_gcs_path = input_config.options.get("output_gcs_path") + job_id = input_config.options.get("job_id", input_config.brand_code or "unknown") + + if not input_gcs_path or not output_gcs_path: + return EncodingOutput( + success=False, + error_message="GCE encoding requires input_gcs_path and output_gcs_path in options", + encoding_backend=self.name + ) + + try: + # Build encoding config + encoding_config = { + "formats": ["mp4_4k_lossless", "mp4_4k_lossy", "mkv_4k", "mp4_720p"], + "artist": input_config.artist, + "title": input_config.title, + } + + # Submit and wait for completion + result = await service.encode_videos( + job_id=job_id, + input_gcs_path=input_gcs_path, + output_gcs_path=output_gcs_path, + encoding_config=encoding_config, + ) + + encoding_time = time.time() - start_time + + # Extract output file paths from result + output_files = result.get("output_files", {}) + + return EncodingOutput( + success=True, + lossless_4k_mp4_path=output_files.get("mp4_4k_lossless"), + lossy_4k_mp4_path=output_files.get("mp4_4k_lossy"), + lossless_mkv_path=output_files.get("mkv_4k"), + lossy_720p_mp4_path=output_files.get("mp4_720p"), + output_files=output_files, + encoding_time_seconds=encoding_time, + encoding_backend=self.name + ) + + except Exception as e: + encoding_time = time.time() - start_time + self.logger.error(f"GCE encoding failed: {e}") + return EncodingOutput( + success=False, + error_message=str(e), + encoding_time_seconds=encoding_time, + encoding_backend=self.name + ) + + async def is_available(self) -> bool: + """Check if GCE encoding is available and configured.""" + service = self._get_service() + return service.is_enabled + + async def get_status(self) -> Dict[str, Any]: + """Get GCE encoding status.""" + service = self._get_service() + + status = { + "backend": self.name, + "available": service.is_enabled, + "configured": service.is_configured, + } + + if service.is_configured: + try: + health = await service.health_check() + status["health"] = health + except Exception as e: + status["health_error"] = str(e) + + return status + + +# Factory function to get an encoding backend +def get_encoding_backend( + backend_type: str = "auto", + **kwargs +) -> EncodingBackend: + """ + Get an encoding backend instance. + + Args: + backend_type: Type of backend - "local", "gce", or "auto" + **kwargs: Additional arguments for the backend + + Returns: + EncodingBackend instance + + Raises: + ValueError: If backend_type is unknown + """ + if backend_type == "local": + return LocalEncodingBackend(**kwargs) + elif backend_type == "gce": + return GCEEncodingBackend(**kwargs) + elif backend_type == "auto": + # Check if GCE is available, otherwise use local + gce_backend = GCEEncodingBackend(**kwargs) + if gce_backend._get_service().is_enabled: + return gce_backend + return LocalEncodingBackend(**kwargs) + else: + raise ValueError(f"Unknown encoding backend type: {backend_type}") diff --git a/backend/services/local_encoding_service.py b/backend/services/local_encoding_service.py new file mode 100644 index 00000000..56da30ad --- /dev/null +++ b/backend/services/local_encoding_service.py @@ -0,0 +1,590 @@ +""" +Local Encoding Service. + +Provides local FFmpeg-based video encoding functionality, extracted from KaraokeFinalise +for use by both the cloud backend (video_worker as a fallback) and local CLI. + +This service handles: +- Concatenating title, karaoke, and end videos +- Encoding to multiple output formats (4K lossless, 4K lossy, 720p, MKV) +- Hardware acceleration detection and fallback to software encoding +- Remuxing video with instrumental audio +""" + +import logging +import os +import shlex +import subprocess +import sys +from dataclasses import dataclass +from typing import Optional, List, Dict, Any, Tuple + +logger = logging.getLogger(__name__) + + +@dataclass +class EncodingConfig: + """Configuration for video encoding.""" + title_video: str # Path to title video + karaoke_video: str # Path to karaoke video (with vocals) + instrumental_audio: str # Path to instrumental audio + end_video: Optional[str] = None # Optional path to end credits video + + # Output paths + output_karaoke_mp4: Optional[str] = None + output_with_vocals_mp4: Optional[str] = None + output_lossless_4k_mp4: Optional[str] = None + output_lossy_4k_mp4: Optional[str] = None + output_lossless_mkv: Optional[str] = None + output_720p_mp4: Optional[str] = None + + +@dataclass +class EncodingResult: + """Result of video encoding operation.""" + success: bool + output_files: Dict[str, str] + error: Optional[str] = None + + +class LocalEncodingService: + """ + Service for local FFmpeg-based video encoding. + + Supports hardware acceleration (NVENC) with automatic fallback + to software encoding (libx264) when unavailable. + """ + + # MP4 flags for better compatibility + MP4_FLAGS = "-movflags +faststart" + + def __init__( + self, + dry_run: bool = False, + log_level: int = logging.INFO, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize the local encoding service. + + Args: + dry_run: If True, log commands without executing them + log_level: Logging level (affects FFmpeg verbosity) + logger: Optional logger instance + """ + self.dry_run = dry_run + self.log_level = log_level + self.logger = logger or logging.getLogger(__name__) + + # Hardware acceleration settings (detected on first use) + self._hwaccel_available: Optional[bool] = None + self._video_encoder: Optional[str] = None + self._scale_filter: Optional[str] = None + self._hwaccel_decode_flags: Optional[str] = None + self._aac_codec: Optional[str] = None + + # Build FFmpeg base command + self._ffmpeg_base_command = self._build_ffmpeg_base_command() + + def _build_ffmpeg_base_command(self) -> str: + """Build the FFmpeg base command with appropriate flags.""" + # Use bundled FFmpeg for frozen builds + ffmpeg_path = os.path.join(sys._MEIPASS, "ffmpeg.exe") if getattr(sys, "frozen", False) else "ffmpeg" + + base_cmd = f"{ffmpeg_path} -hide_banner -nostats -y" + + if self.log_level == logging.DEBUG: + base_cmd += " -loglevel verbose" + else: + base_cmd += " -loglevel fatal" + + return base_cmd + + def _detect_hardware_acceleration(self) -> Tuple[bool, str, str, str, str]: + """ + Detect available hardware acceleration. + + Returns: + Tuple of (hwaccel_available, video_encoder, scale_filter, + hwaccel_decode_flags, aac_codec) + """ + self.logger.info("Detecting hardware acceleration capabilities...") + + # Try NVENC (NVIDIA) + try: + test_cmd = f"{self._ffmpeg_base_command} -hide_banner -loglevel error " \ + f"-f lavfi -i testsrc=duration=1:size=320x240:rate=1 " \ + f"-c:v h264_nvenc -f null -" + subprocess.run( + test_cmd, shell=True, check=True, + capture_output=True, timeout=30 + ) + self.logger.info("NVIDIA NVENC hardware acceleration available") + return ( + True, + "h264_nvenc", + "scale_cuda", + "-hwaccel cuda -hwaccel_output_format cuda", + "aac" + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + pass + + # No hardware acceleration available + self.logger.info("No hardware acceleration available, using software encoding") + return (False, "libx264", "scale", "", "aac") + + @property + def hwaccel_available(self) -> bool: + """Check if hardware acceleration is available.""" + if self._hwaccel_available is None: + self._detect_and_set_hwaccel() + return self._hwaccel_available + + @property + def video_encoder(self) -> str: + """Get the video encoder to use.""" + if self._video_encoder is None: + self._detect_and_set_hwaccel() + return self._video_encoder + + @property + def scale_filter(self) -> str: + """Get the scale filter to use.""" + if self._scale_filter is None: + self._detect_and_set_hwaccel() + return self._scale_filter + + @property + def hwaccel_decode_flags(self) -> str: + """Get hardware acceleration decode flags.""" + if self._hwaccel_decode_flags is None: + self._detect_and_set_hwaccel() + return self._hwaccel_decode_flags + + @property + def aac_codec(self) -> str: + """Get the AAC codec to use.""" + if self._aac_codec is None: + self._detect_and_set_hwaccel() + return self._aac_codec + + def _detect_and_set_hwaccel(self) -> None: + """Detect and set hardware acceleration settings.""" + ( + self._hwaccel_available, + self._video_encoder, + self._scale_filter, + self._hwaccel_decode_flags, + self._aac_codec + ) = self._detect_hardware_acceleration() + + def _get_nvenc_quality_settings(self, preset: str = "medium") -> str: + """Get NVENC quality settings for different presets.""" + if not self.hwaccel_available: + return "" + + settings = { + "lossless": "-preset p7 -tune hq -rc vbr -cq 0 -qmin 0 -qmax 0", + "high": "-preset p7 -tune hq -rc vbr -cq 19 -b:v 0", + "medium": "-preset p4 -tune hq -rc vbr -cq 23 -b:v 0", + "fast": "-preset p2 -tune ll -rc vbr -cq 28 -b:v 0", + } + return settings.get(preset, settings["medium"]) + + def _execute_command( + self, + command: str, + description: str, + timeout: int = 3600, # 1 hour default + ) -> bool: + """ + Execute an FFmpeg command. + + Args: + command: The FFmpeg command to execute + description: Human-readable description of the operation + timeout: Command timeout in seconds + + Returns: + True if successful, False otherwise + """ + self.logger.info(f"Executing: {description}") + self.logger.debug(f"Command: {command}") + + if self.dry_run: + self.logger.info(f"DRY RUN: Would execute: {command}") + return True + + try: + result = subprocess.run( + command, + shell=True, + check=True, + capture_output=True, + text=True, + timeout=timeout + ) + self.logger.info(f"Completed: {description}") + return True + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed: {description}") + self.logger.error(f"Error: {e.stderr}") + return False + except subprocess.TimeoutExpired: + self.logger.error(f"Timeout: {description}") + return False + + def _execute_command_with_fallback( + self, + gpu_command: str, + cpu_command: str, + description: str, + ) -> bool: + """ + Execute a command with GPU, falling back to CPU if it fails. + + Args: + gpu_command: Hardware-accelerated command + cpu_command: Software fallback command + description: Human-readable description + + Returns: + True if successful, False otherwise + """ + if self.hwaccel_available: + self.logger.info(f"Trying hardware-accelerated encoding for: {description}") + if self._execute_command(gpu_command, description): + return True + self.logger.warning(f"Hardware encoding failed, falling back to software") + + return self._execute_command(cpu_command, description) + + def remux_with_instrumental( + self, + input_video: str, + instrumental_audio: str, + output_file: str, + ) -> bool: + """ + Remux video with instrumental audio track. + + Args: + input_video: Path to input video file + instrumental_audio: Path to instrumental audio file + output_file: Path for output file + + Returns: + True if successful, False otherwise + """ + command = ( + f'{self._ffmpeg_base_command} -i "{input_video}" ' + f'-i "{instrumental_audio}" -map 0:v -map 1:a -c copy ' + f'{self.MP4_FLAGS} "{output_file}"' + ) + return self._execute_command(command, "Remuxing with instrumental audio") + + def convert_mov_to_mp4( + self, + input_file: str, + output_file: str, + ) -> bool: + """ + Convert MOV to MP4 format. + + Args: + input_file: Path to input MOV file + output_file: Path for output MP4 file + + Returns: + True if successful, False otherwise + """ + gpu_command = ( + f'{self._ffmpeg_base_command} {self.hwaccel_decode_flags} -i "{input_file}" ' + f'-c:v {self.video_encoder} -c:a copy {self.MP4_FLAGS} "{output_file}"' + ) + cpu_command = ( + f'{self._ffmpeg_base_command} -i "{input_file}" ' + f'-c:v libx264 -c:a copy {self.MP4_FLAGS} "{output_file}"' + ) + return self._execute_command_with_fallback( + gpu_command, cpu_command, "Converting MOV to MP4" + ) + + def encode_lossless_mp4( + self, + title_video: str, + karaoke_video: str, + output_file: str, + end_video: Optional[str] = None, + ) -> bool: + """ + Create lossless 4K MP4 by concatenating title, karaoke, and optionally end videos. + + Args: + title_video: Path to title video + karaoke_video: Path to karaoke video + output_file: Path for output file + end_video: Optional path to end credits video + + Returns: + True if successful, False otherwise + """ + # Quote file paths + title_quoted = shlex.quote(os.path.abspath(title_video)) + karaoke_quoted = shlex.quote(os.path.abspath(karaoke_video)) + + # Build filter and inputs for concatenation + if end_video and os.path.isfile(end_video): + end_quoted = shlex.quote(os.path.abspath(end_video)) + extra_input = f"-i {end_quoted}" + concat_filter = ( + '-filter_complex "[0:v:0][0:a:0][1:v:0][1:a:0][2:v:0][2:a:0]' + 'concat=n=3:v=1:a=1[outv][outa]"' + ) + else: + extra_input = "" + concat_filter = ( + '-filter_complex "[0:v:0][0:a:0][1:v:0][1:a:0]' + 'concat=n=2:v=1:a=1[outv][outa]"' + ) + + gpu_command = ( + f"{self._ffmpeg_base_command} {self.hwaccel_decode_flags} -i {title_quoted} " + f"{self.hwaccel_decode_flags} -i {karaoke_quoted} {extra_input} " + f'{concat_filter} -map "[outv]" -map "[outa]" -c:v {self.video_encoder} ' + f'{self._get_nvenc_quality_settings("lossless")} -c:a pcm_s16le ' + f'{self.MP4_FLAGS} "{output_file}"' + ) + cpu_command = ( + f"{self._ffmpeg_base_command} -i {title_quoted} -i {karaoke_quoted} {extra_input} " + f'{concat_filter} -map "[outv]" -map "[outa]" -c:v libx264 -c:a pcm_s16le ' + f'{self.MP4_FLAGS} "{output_file}"' + ) + + return self._execute_command_with_fallback( + gpu_command, cpu_command, "Encoding lossless 4K MP4" + ) + + def encode_lossy_mp4( + self, + input_file: str, + output_file: str, + ) -> bool: + """ + Create lossy 4K MP4 with AAC audio. + + Args: + input_file: Path to input file (typically lossless MP4) + output_file: Path for output file + + Returns: + True if successful, False otherwise + """ + command = ( + f'{self._ffmpeg_base_command} -i "{input_file}" ' + f'-c:v copy -c:a {self.aac_codec} -ar 48000 -b:a 320k ' + f'{self.MP4_FLAGS} "{output_file}"' + ) + return self._execute_command(command, "Encoding lossy 4K MP4 with AAC") + + def encode_lossless_mkv( + self, + input_file: str, + output_file: str, + ) -> bool: + """ + Create MKV with FLAC audio (for YouTube upload). + + Args: + input_file: Path to input file + output_file: Path for output file + + Returns: + True if successful, False otherwise + """ + command = ( + f'{self._ffmpeg_base_command} -i "{input_file}" ' + f'-c:v copy -c:a flac "{output_file}"' + ) + return self._execute_command(command, "Creating MKV with FLAC for YouTube") + + def encode_720p( + self, + input_file: str, + output_file: str, + ) -> bool: + """ + Create 720p MP4 with AAC audio. + + Args: + input_file: Path to input file + output_file: Path for output file + + Returns: + True if successful, False otherwise + """ + gpu_command = ( + f'{self._ffmpeg_base_command} {self.hwaccel_decode_flags} -i "{input_file}" ' + f'-c:v {self.video_encoder} -vf "{self.scale_filter}=1280:720" ' + f'{self._get_nvenc_quality_settings("medium")} -b:v 2000k ' + f'-c:a {self.aac_codec} -ar 48000 -b:a 128k ' + f'{self.MP4_FLAGS} "{output_file}"' + ) + cpu_command = ( + f'{self._ffmpeg_base_command} -i "{input_file}" ' + f'-c:v libx264 -vf "scale=1280:720" -b:v 2000k -preset medium -tune animation ' + f'-c:a {self.aac_codec} -ar 48000 -b:a 128k ' + f'{self.MP4_FLAGS} "{output_file}"' + ) + return self._execute_command_with_fallback( + gpu_command, cpu_command, "Encoding 720p MP4" + ) + + def encode_all_formats( + self, + config: EncodingConfig, + ) -> EncodingResult: + """ + Encode video to all output formats. + + This performs the full encoding pipeline: + 1. Remux with instrumental audio + 2. Convert to MP4 if needed + 3. Encode lossless 4K MP4 (concatenated) + 4. Encode lossy 4K MP4 + 5. Encode lossless MKV (for YouTube) + 6. Encode 720p MP4 + + Args: + config: Encoding configuration with input/output paths + + Returns: + EncodingResult with success status and output file paths + """ + output_files = {} + + try: + # Step 1: Remux with instrumental audio + if config.output_karaoke_mp4: + self.logger.info("[Step 1/6] Remuxing video with instrumental audio...") + if not self.remux_with_instrumental( + config.karaoke_video, + config.instrumental_audio, + config.output_karaoke_mp4 + ): + return EncodingResult( + success=False, + output_files=output_files, + error="Failed to remux with instrumental audio" + ) + output_files["karaoke_mp4"] = config.output_karaoke_mp4 + + # Step 2: Convert to MP4 if needed + if config.output_with_vocals_mp4: + if not config.karaoke_video.endswith(".mp4"): + self.logger.info("[Step 2/6] Converting karaoke video to MP4...") + if not self.convert_mov_to_mp4( + config.karaoke_video, + config.output_with_vocals_mp4 + ): + return EncodingResult( + success=False, + output_files=output_files, + error="Failed to convert to MP4" + ) + output_files["with_vocals_mp4"] = config.output_with_vocals_mp4 + else: + self.logger.info("[Step 2/6] Skipped - video already MP4") + + # Step 3: Encode lossless 4K MP4 + if config.output_lossless_4k_mp4: + self.logger.info("[Step 3/6] Encoding lossless 4K MP4...") + karaoke_for_concat = config.output_karaoke_mp4 or config.karaoke_video + if not self.encode_lossless_mp4( + config.title_video, + karaoke_for_concat, + config.output_lossless_4k_mp4, + config.end_video + ): + return EncodingResult( + success=False, + output_files=output_files, + error="Failed to encode lossless 4K MP4" + ) + output_files["lossless_4k_mp4"] = config.output_lossless_4k_mp4 + + # Step 4: Encode lossy 4K MP4 + if config.output_lossy_4k_mp4 and config.output_lossless_4k_mp4: + self.logger.info("[Step 4/6] Encoding lossy 4K MP4...") + if not self.encode_lossy_mp4( + config.output_lossless_4k_mp4, + config.output_lossy_4k_mp4 + ): + return EncodingResult( + success=False, + output_files=output_files, + error="Failed to encode lossy 4K MP4" + ) + output_files["lossy_4k_mp4"] = config.output_lossy_4k_mp4 + + # Step 5: Create MKV with FLAC audio + if config.output_lossless_mkv and config.output_lossless_4k_mp4: + self.logger.info("[Step 5/6] Creating MKV with FLAC audio...") + if not self.encode_lossless_mkv( + config.output_lossless_4k_mp4, + config.output_lossless_mkv + ): + return EncodingResult( + success=False, + output_files=output_files, + error="Failed to create MKV" + ) + output_files["lossless_mkv"] = config.output_lossless_mkv + + # Step 6: Encode 720p version + if config.output_720p_mp4 and config.output_lossless_4k_mp4: + self.logger.info("[Step 6/6] Encoding 720p MP4...") + if not self.encode_720p( + config.output_lossless_4k_mp4, + config.output_720p_mp4 + ): + return EncodingResult( + success=False, + output_files=output_files, + error="Failed to encode 720p" + ) + output_files["720p_mp4"] = config.output_720p_mp4 + + self.logger.info("All encoding steps completed successfully") + return EncodingResult(success=True, output_files=output_files) + + except Exception as e: + self.logger.error(f"Encoding failed with exception: {e}") + return EncodingResult( + success=False, + output_files=output_files, + error=str(e) + ) + + +# Singleton instance and factory function (following existing service pattern) +_local_encoding_service: Optional[LocalEncodingService] = None + + +def get_local_encoding_service(**kwargs) -> LocalEncodingService: + """ + Get a local encoding service instance. + + Args: + **kwargs: Arguments passed to LocalEncodingService + + Returns: + LocalEncodingService instance + """ + global _local_encoding_service + + if _local_encoding_service is None: + _local_encoding_service = LocalEncodingService(**kwargs) + + return _local_encoding_service diff --git a/backend/services/packaging_service.py b/backend/services/packaging_service.py new file mode 100644 index 00000000..7d083aad --- /dev/null +++ b/backend/services/packaging_service.py @@ -0,0 +1,287 @@ +""" +Packaging Service. + +Provides CDG and TXT package generation functionality, extracted from KaraokeFinalise +for use by both the cloud backend (video_worker) and local CLI. + +This service handles: +- CDG (CD+G) file generation from LRC files +- TXT lyric file generation from LRC files +- ZIP packaging of CDG/MP3 and TXT/MP3 pairs +""" + +import logging +import os +import zipfile +from typing import Optional, Dict, Any, Tuple + +logger = logging.getLogger(__name__) + + +class PackagingService: + """ + Service for creating CDG and TXT karaoke packages. + + CDG (CD+Graphics) is a format used by karaoke machines. + TXT packages are used by software karaoke players. + Both formats are packaged as ZIP files with an MP3 audio track. + """ + + def __init__( + self, + cdg_styles: Optional[Dict[str, Any]] = None, + dry_run: bool = False, + non_interactive: bool = False, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize the packaging service. + + Args: + cdg_styles: CDG style configuration for CDG generation + dry_run: If True, log actions without performing them + non_interactive: If True, skip interactive prompts + logger: Optional logger instance + """ + self.cdg_styles = cdg_styles + self.dry_run = dry_run + self.non_interactive = non_interactive + self.logger = logger or logging.getLogger(__name__) + + def create_cdg_package( + self, + lrc_file: str, + audio_file: str, + output_zip_path: str, + artist: str, + title: str, + output_mp3_path: Optional[str] = None, + output_cdg_path: Optional[str] = None, + ) -> Tuple[str, Optional[str], Optional[str]]: + """ + Create a CDG package (ZIP containing CDG and MP3 files). + + Args: + lrc_file: Path to the LRC lyrics file + audio_file: Path to the instrumental audio file + output_zip_path: Path for the output ZIP file + artist: Artist name for metadata + title: Song title for metadata + output_mp3_path: Optional path for the extracted MP3 file + output_cdg_path: Optional path for the extracted CDG file + + Returns: + Tuple of (zip_path, mp3_path, cdg_path) + + Raises: + ValueError: If CDG styles are not configured + FileNotFoundError: If input files are missing + Exception: If CDG generation fails + """ + self.logger.info(f"Creating CDG package for {artist} - {title}") + + # Validate inputs + if not os.path.isfile(lrc_file): + raise FileNotFoundError(f"LRC file not found: {lrc_file}") + if not os.path.isfile(audio_file): + raise FileNotFoundError(f"Audio file not found: {audio_file}") + + # Check if ZIP already exists + if os.path.isfile(output_zip_path): + if self.non_interactive: + self.logger.info( + f"CDG ZIP exists, will be overwritten: {output_zip_path}" + ) + else: + self.logger.info(f"CDG ZIP already exists: {output_zip_path}") + + # Check if individual files exist (allows skipping generation) + if output_mp3_path and output_cdg_path: + if os.path.isfile(output_mp3_path) and os.path.isfile(output_cdg_path): + self.logger.info("Found existing MP3 and CDG files, creating ZIP directly") + if not self.dry_run: + self._create_zip_from_files( + output_zip_path, + [(output_mp3_path, os.path.basename(output_mp3_path)), + (output_cdg_path, os.path.basename(output_cdg_path))] + ) + return output_zip_path, output_mp3_path, output_cdg_path + + if self.dry_run: + self.logger.info( + f"DRY RUN: Would generate CDG package: {output_zip_path}" + ) + return output_zip_path, output_mp3_path, output_cdg_path + + # Generate CDG files + if self.cdg_styles is None: + raise ValueError( + "CDG styles configuration is required for CDG generation" + ) + + self.logger.info("Generating CDG and MP3 files") + from lyrics_transcriber.output.cdg import CDGGenerator + + output_dir = os.path.dirname(output_zip_path) or os.getcwd() + generator = CDGGenerator(output_dir=output_dir, logger=self.logger) + + cdg_file, mp3_file, zip_file = generator.generate_cdg_from_lrc( + lrc_file=lrc_file, + audio_file=audio_file, + title=title, + artist=artist, + cdg_styles=self.cdg_styles, + ) + + # Rename ZIP to expected output path if different + if os.path.isfile(zip_file) and zip_file != output_zip_path: + os.rename(zip_file, output_zip_path) + self.logger.info(f"Renamed CDG ZIP: {zip_file} -> {output_zip_path}") + + if not os.path.isfile(output_zip_path): + raise Exception(f"Failed to create CDG ZIP file: {output_zip_path}") + + # Extract the ZIP to get individual files if paths provided + extracted_mp3 = None + extracted_cdg = None + if output_mp3_path or output_cdg_path: + self.logger.info(f"Extracting CDG ZIP file: {output_zip_path}") + with zipfile.ZipFile(output_zip_path, "r") as zip_ref: + zip_ref.extractall(output_dir) + + # Find extracted files + if output_mp3_path and os.path.isfile(output_mp3_path): + extracted_mp3 = output_mp3_path + self.logger.info(f"Extracted MP3: {extracted_mp3}") + if output_cdg_path and os.path.isfile(output_cdg_path): + extracted_cdg = output_cdg_path + self.logger.info(f"Extracted CDG: {extracted_cdg}") + + self.logger.info(f"CDG package created: {output_zip_path}") + return output_zip_path, extracted_mp3, extracted_cdg + + def create_txt_package( + self, + lrc_file: str, + mp3_file: str, + output_zip_path: str, + output_txt_path: Optional[str] = None, + ) -> Tuple[str, Optional[str]]: + """ + Create a TXT package (ZIP containing TXT lyrics and MP3 files). + + Args: + lrc_file: Path to the LRC lyrics file + mp3_file: Path to the MP3 audio file + output_zip_path: Path for the output ZIP file + output_txt_path: Optional path for the generated TXT file + + Returns: + Tuple of (zip_path, txt_path) + + Raises: + FileNotFoundError: If input files are missing + Exception: If TXT generation fails + """ + self.logger.info(f"Creating TXT package from {lrc_file}") + + # Validate inputs + if not os.path.isfile(lrc_file): + raise FileNotFoundError(f"LRC file not found: {lrc_file}") + if not os.path.isfile(mp3_file): + raise FileNotFoundError(f"MP3 file not found: {mp3_file}") + + # Check if ZIP already exists + if os.path.isfile(output_zip_path): + if self.non_interactive: + self.logger.info( + f"TXT ZIP exists, will be overwritten: {output_zip_path}" + ) + else: + self.logger.info(f"TXT ZIP already exists: {output_zip_path}") + + if self.dry_run: + self.logger.info( + f"DRY RUN: Would create TXT package: {output_zip_path}" + ) + return output_zip_path, output_txt_path + + # Generate TXT from LRC + self.logger.info(f"Converting LRC to TXT format: {lrc_file}") + from lyrics_converter import LyricsConverter + + txt_converter = LyricsConverter(output_format="txt", filepath=lrc_file) + converted_txt = txt_converter.convert_file() + + # Write TXT file + if output_txt_path is None: + # Default to same name as ZIP but with .txt extension + output_txt_path = output_zip_path.replace(".zip", ".txt") + + with open(output_txt_path, "w") as txt_file: + txt_file.write(converted_txt) + self.logger.info(f"TXT file written: {output_txt_path}") + + # Create ZIP containing MP3 and TXT + self.logger.info(f"Creating TXT ZIP: {output_zip_path}") + self._create_zip_from_files( + output_zip_path, + [(mp3_file, os.path.basename(mp3_file)), + (output_txt_path, os.path.basename(output_txt_path))] + ) + + if not os.path.isfile(output_zip_path): + raise Exception(f"Failed to create TXT ZIP file: {output_zip_path}") + + self.logger.info(f"TXT package created: {output_zip_path}") + return output_zip_path, output_txt_path + + def _create_zip_from_files( + self, + zip_path: str, + files: list, + ) -> None: + """ + Create a ZIP file from a list of files. + + Args: + zip_path: Path for the output ZIP file + files: List of (file_path, archive_name) tuples + """ + with zipfile.ZipFile(zip_path, "w") as zipf: + for file_path, archive_name in files: + if os.path.isfile(file_path): + zipf.write(file_path, archive_name) + self.logger.debug(f"Added to ZIP: {archive_name}") + else: + self.logger.warning(f"File not found for ZIP: {file_path}") + + +# Singleton instance and factory function (following existing service pattern) +_packaging_service: Optional[PackagingService] = None + + +def get_packaging_service( + cdg_styles: Optional[Dict[str, Any]] = None, + **kwargs +) -> PackagingService: + """ + Get a packaging service instance. + + Args: + cdg_styles: CDG style configuration + **kwargs: Additional arguments passed to PackagingService + + Returns: + PackagingService instance + """ + global _packaging_service + + # Create new instance if settings changed + if _packaging_service is None or cdg_styles: + _packaging_service = PackagingService( + cdg_styles=cdg_styles, + **kwargs + ) + + return _packaging_service diff --git a/backend/services/youtube_upload_service.py b/backend/services/youtube_upload_service.py new file mode 100644 index 00000000..e709a307 --- /dev/null +++ b/backend/services/youtube_upload_service.py @@ -0,0 +1,445 @@ +""" +YouTube Upload Service. + +Provides video upload functionality for YouTube, extracted from KaraokeFinalise +for use by both the cloud backend (video_worker) and local CLI. + +This service handles: +- Video uploads with metadata (title, description, tags) +- Thumbnail uploads +- Duplicate video detection and replacement +- Authentication via pre-stored credentials or client secrets file +""" + +import logging +import os +from typing import Optional, Dict, Any, Tuple + +from thefuzz import fuzz + +logger = logging.getLogger(__name__) + + +class YouTubeUploadService: + """ + Service for uploading videos to YouTube. + + Supports two authentication modes: + 1. Pre-stored credentials (for server-side/non-interactive use) + 2. Client secrets file (for interactive CLI use with browser OAuth) + """ + + YOUTUBE_URL_PREFIX = "https://www.youtube.com/watch?v=" + SCOPES = ["https://www.googleapis.com/auth/youtube"] + + def __init__( + self, + credentials: Optional[Dict[str, Any]] = None, + client_secrets_file: Optional[str] = None, + non_interactive: bool = False, + server_side_mode: bool = False, + dry_run: bool = False, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize the YouTube upload service. + + Args: + credentials: Pre-stored OAuth credentials dict with keys: + - token: Access token + - refresh_token: Refresh token + - token_uri: Token endpoint URL + - client_id: OAuth client ID + - client_secret: OAuth client secret + - scopes: List of OAuth scopes + client_secrets_file: Path to OAuth client secrets JSON file + (for interactive authentication) + non_interactive: If True, skip interactive prompts + server_side_mode: If True, use exact matching for duplicate detection + dry_run: If True, log actions without performing them + logger: Optional logger instance + """ + self.credentials = credentials + self.client_secrets_file = client_secrets_file + self.non_interactive = non_interactive + self.server_side_mode = server_side_mode + self.dry_run = dry_run + self.logger = logger or logging.getLogger(__name__) + + self._youtube_service = None + self._channel_id = None + + def _authenticate(self): + """ + Authenticate with YouTube API and return service object. + + Returns: + YouTube API service object + + Raises: + Exception: If authentication fails or is required in non-interactive mode + """ + from google.auth.transport.requests import Request + from google.oauth2.credentials import Credentials + from googleapiclient.discovery import build + from google_auth_oauthlib.flow import InstalledAppFlow + import pickle + + # Check if we have pre-stored credentials (for non-interactive mode) + if self.credentials and self.non_interactive: + try: + creds = Credentials( + token=self.credentials.get('token'), + refresh_token=self.credentials.get('refresh_token'), + token_uri=self.credentials.get('token_uri'), + client_id=self.credentials.get('client_id'), + client_secret=self.credentials.get('client_secret'), + scopes=self.credentials.get('scopes', self.SCOPES) + ) + + # Refresh token if needed + if creds.expired and creds.refresh_token: + creds.refresh(Request()) + + youtube = build('youtube', 'v3', credentials=creds) + self.logger.info("Successfully authenticated with YouTube using pre-stored credentials") + return youtube + + except Exception as e: + self.logger.error(f"Failed to authenticate with pre-stored credentials: {str(e)}") + # Fall through to original authentication if pre-stored credentials fail + + # For non-interactive mode without pre-stored credentials, raise error + if self.non_interactive: + raise Exception( + "YouTube authentication required but running in non-interactive mode. " + "Please pre-authenticate or disable YouTube upload." + ) + + # Interactive authentication using client secrets file + if not self.client_secrets_file: + raise Exception("No YouTube credentials or client secrets file provided") + + # Token file stores the user's access and refresh tokens + youtube_token_file = "/tmp/karaoke-finalise-youtube-token.pickle" + + creds = None + + # Check if we have saved credentials + if os.path.exists(youtube_token_file): + with open(youtube_token_file, "rb") as token: + creds = pickle.load(token) + + # If there are no valid credentials, let the user log in + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + self.client_secrets_file, scopes=self.SCOPES + ) + creds = flow.run_local_server(port=0) + + # Save the credentials for the next run + with open(youtube_token_file, "wb") as token: + pickle.dump(creds, token) + + return build("youtube", "v3", credentials=creds) + + @property + def youtube_service(self): + """Lazy-load YouTube service on first access.""" + if self._youtube_service is None: + self._youtube_service = self._authenticate() + return self._youtube_service + + def get_channel_id(self) -> Optional[str]: + """ + Get the authenticated user's YouTube channel ID. + + Returns: + Channel ID string, or None if not found + """ + if self._channel_id: + return self._channel_id + + request = self.youtube_service.channels().list(part="snippet", mine=True) + response = request.execute() + + if "items" in response and len(response["items"]) > 0: + self._channel_id = response["items"][0]["id"] + return self._channel_id + + return None + + def check_duplicate( + self, + title: str, + exact_match: Optional[bool] = None + ) -> Tuple[bool, Optional[str], Optional[str]]: + """ + Check if a video with the given title already exists on the channel. + + Args: + title: Video title to search for + exact_match: If True, require exact title match. If None, uses + server_side_mode setting (exact in server mode, fuzzy in CLI) + + Returns: + Tuple of (exists: bool, video_id: Optional[str], video_url: Optional[str]) + """ + channel_id = self.get_channel_id() + if not channel_id: + self.logger.warning("Could not get channel ID, skipping duplicate check") + return False, None, None + + use_exact_match = exact_match if exact_match is not None else self.server_side_mode + + self.logger.info(f"Searching YouTube channel {channel_id} for title: {title}") + request = self.youtube_service.search().list( + part="snippet", + channelId=channel_id, + q=title, + type="video", + maxResults=10 + ) + response = request.execute() + + if "items" not in response or len(response["items"]) == 0: + self.logger.info(f"No matching video found with title: {title}") + return False, None, None + + for item in response["items"]: + # Verify the video actually belongs to our channel + result_channel_id = item["snippet"]["channelId"] + if result_channel_id != channel_id: + self.logger.debug( + f"Skipping video from different channel: {item['snippet']['title']} " + f"(channel: {result_channel_id})" + ) + continue + + found_title = item["snippet"]["title"] + + # Determine if this is a match + if use_exact_match: + is_match = title.lower() == found_title.lower() + similarity_score = 100 if is_match else 0 + else: + similarity_score = fuzz.ratio(title.lower(), found_title.lower()) + is_match = similarity_score >= 70 + + if is_match: + video_id = item["id"]["videoId"] + video_url = f"{self.YOUTUBE_URL_PREFIX}{video_id}" + self.logger.info( + f"Potential match found on YouTube channel with ID: {video_id} " + f"and title: {found_title} (similarity: {similarity_score}%)" + ) + + # In non-interactive mode, return the match directly + if self.non_interactive: + self.logger.info("Non-interactive mode, found a match.") + return True, video_id, video_url + + # Interactive confirmation + confirmation = input( + f"Is '{found_title}' the video you are finalising? (y/n): " + ).strip().lower() + if confirmation == "y": + return True, video_id, video_url + + self.logger.info(f"No matching video found with title: {title}") + return False, None, None + + def delete_video(self, video_id: str) -> bool: + """ + Delete a YouTube video by its ID. + + Args: + video_id: The YouTube video ID to delete + + Returns: + True if successful, False otherwise + """ + self.logger.info(f"Deleting YouTube video with ID: {video_id}") + + if self.dry_run: + self.logger.info(f"DRY RUN: Would delete YouTube video with ID: {video_id}") + return True + + try: + self.youtube_service.videos().delete(id=video_id).execute() + self.logger.info(f"Successfully deleted YouTube video with ID: {video_id}") + return True + except Exception as e: + self.logger.error(f"Failed to delete YouTube video with ID {video_id}: {e}") + return False + + @staticmethod + def truncate_title(title: str, max_length: int = 95) -> str: + """ + Truncate title to the nearest whole word within max_length. + + Args: + title: Title to truncate + max_length: Maximum length (default 95 for YouTube) + + Returns: + Truncated title with "..." if needed + """ + if len(title) <= max_length: + return title + truncated_title = title[:max_length].rsplit(" ", 1)[0] + if len(truncated_title) < max_length: + truncated_title += " ..." + return truncated_title + + def upload_video( + self, + video_path: str, + title: str, + description: str, + thumbnail_path: Optional[str] = None, + tags: Optional[list] = None, + category_id: str = "10", # Music category + privacy_status: str = "public", + replace_existing: bool = False, + ) -> Tuple[Optional[str], Optional[str]]: + """ + Upload a video to YouTube with metadata and optional thumbnail. + + Args: + video_path: Path to the video file to upload + title: Video title (will be truncated to 95 chars if needed) + description: Video description + thumbnail_path: Optional path to thumbnail image + tags: Optional list of tags/keywords + category_id: YouTube category ID (default "10" for Music) + privacy_status: "public", "private", or "unlisted" + replace_existing: If True, delete existing video with same title + + Returns: + Tuple of (video_id, video_url) or (None, None) if upload failed/skipped + """ + from googleapiclient.http import MediaFileUpload + + # Truncate title if needed + youtube_title = self.truncate_title(title) + + self.logger.info(f"Uploading video to YouTube: {youtube_title}") + + if self.dry_run: + self.logger.info( + f"DRY RUN: Would upload {video_path} to YouTube with title: {youtube_title}" + ) + return "dry_run_video_id", f"{self.YOUTUBE_URL_PREFIX}dry_run_video_id" + + # Check for existing video + should_replace = True if self.server_side_mode else replace_existing + exists, existing_id, existing_url = self.check_duplicate(youtube_title) + + if exists: + if should_replace: + self.logger.info(f"Video already exists on YouTube, deleting before re-upload: {existing_url}") + if self.delete_video(existing_id): + self.logger.info("Successfully deleted existing video, proceeding with upload") + else: + self.logger.error("Failed to delete existing video, aborting upload") + return None, None + else: + self.logger.warning(f"Video already exists on YouTube, skipping upload: {existing_url}") + return existing_id, existing_url + + # Prepare video metadata + body = { + "snippet": { + "title": youtube_title, + "description": description, + "tags": tags or [], + "categoryId": category_id, + }, + "status": {"privacyStatus": privacy_status}, + } + + # Determine MIME type based on file extension + ext = os.path.splitext(video_path)[1].lower() + mime_type = { + ".mkv": "video/x-matroska", + ".mp4": "video/mp4", + ".mov": "video/quicktime", + ".avi": "video/x-msvideo", + }.get(ext, "video/*") + + # Upload video + self.logger.info(f"Authenticating with YouTube...") + media_file = MediaFileUpload(video_path, mimetype=mime_type, resumable=True) + + self.logger.info(f"Uploading video to YouTube...") + request = self.youtube_service.videos().insert( + part="snippet,status", + body=body, + media_body=media_file + ) + response = request.execute() + + video_id = response.get("id") + video_url = f"{self.YOUTUBE_URL_PREFIX}{video_id}" + self.logger.info(f"Uploaded video to YouTube: {video_url}") + + # Upload thumbnail if provided + if thumbnail_path and os.path.isfile(thumbnail_path): + try: + self.logger.info(f"Uploading thumbnail from: {thumbnail_path}") + media_thumbnail = MediaFileUpload(thumbnail_path, mimetype="image/jpeg") + self.youtube_service.thumbnails().set( + videoId=video_id, + media_body=media_thumbnail + ).execute() + self.logger.info(f"Uploaded thumbnail for video ID {video_id}") + except Exception as e: + self.logger.error(f"Failed to upload thumbnail: {e}") + self.logger.warning( + "Video uploaded but thumbnail not set. " + "You may need to set it manually on YouTube." + ) + elif thumbnail_path: + self.logger.warning(f"Thumbnail file not found, skipping: {thumbnail_path}") + + return video_id, video_url + + +# Singleton instance and factory function (following existing service pattern) +_youtube_upload_service: Optional[YouTubeUploadService] = None + + +def get_youtube_upload_service( + credentials: Optional[Dict[str, Any]] = None, + client_secrets_file: Optional[str] = None, + **kwargs +) -> YouTubeUploadService: + """ + Get a YouTube upload service instance. + + For server-side use, pass credentials from YouTubeService. + For CLI use, pass client_secrets_file. + + Args: + credentials: Pre-stored OAuth credentials dict + client_secrets_file: Path to OAuth client secrets JSON file + **kwargs: Additional arguments passed to YouTubeUploadService + + Returns: + YouTubeUploadService instance + """ + global _youtube_upload_service + + # Create new instance if credentials/settings changed + if _youtube_upload_service is None or credentials or client_secrets_file: + _youtube_upload_service = YouTubeUploadService( + credentials=credentials, + client_secrets_file=client_secrets_file, + **kwargs + ) + + return _youtube_upload_service diff --git a/backend/tests/test_discord_service.py b/backend/tests/test_discord_service.py new file mode 100644 index 00000000..74ec4ce0 --- /dev/null +++ b/backend/tests/test_discord_service.py @@ -0,0 +1,244 @@ +""" +Tests for DiscordNotificationService. + +Tests cover: +- Service initialization +- Webhook URL validation +- Message posting +- Video notification posting +- Dry run mode +""" + +import pytest +from unittest.mock import MagicMock, patch + +from backend.services.discord_service import ( + DiscordNotificationService, + get_discord_notification_service, +) + + +class TestDiscordNotificationServiceInit: + """Test service initialization.""" + + def test_init_with_webhook_url(self): + """Test initialization with valid webhook URL.""" + url = "https://discord.com/api/webhooks/123456789/abcdef123456" + service = DiscordNotificationService(webhook_url=url) + assert service.webhook_url == url + assert service.is_enabled() is True + + def test_init_without_webhook_url(self): + """Test initialization without webhook URL.""" + service = DiscordNotificationService() + assert service.webhook_url is None + assert service.is_enabled() is False + + def test_init_with_dry_run(self): + """Test initialization with dry run mode.""" + service = DiscordNotificationService(dry_run=True) + assert service.dry_run is True + + def test_init_with_invalid_webhook_url_raises(self): + """Test that invalid webhook URL raises ValueError.""" + with pytest.raises(ValueError) as exc_info: + DiscordNotificationService(webhook_url="https://example.com/webhook") + assert "Invalid Discord webhook URL" in str(exc_info.value) + + def test_init_with_non_discord_url_raises(self): + """Test that non-Discord URL raises ValueError.""" + with pytest.raises(ValueError): + DiscordNotificationService( + webhook_url="https://slack.com/api/webhooks/123" + ) + + +class TestDiscordNotificationServiceValidation: + """Test webhook URL validation.""" + + def test_validate_valid_webhook_url(self): + """Test that valid webhook URL passes validation.""" + service = DiscordNotificationService() + # Should not raise + service._validate_webhook_url( + "https://discord.com/api/webhooks/123456789/abcdef" + ) + + def test_validate_invalid_webhook_url(self): + """Test that invalid webhook URL raises ValueError.""" + service = DiscordNotificationService() + with pytest.raises(ValueError) as exc_info: + service._validate_webhook_url("https://example.com/webhook") + assert "Invalid Discord webhook URL" in str(exc_info.value) + + def test_validate_strips_whitespace(self): + """Test that whitespace is stripped from URL.""" + service = DiscordNotificationService() + # Should not raise - whitespace is stripped + service._validate_webhook_url( + " https://discord.com/api/webhooks/123456789/abcdef " + ) + + +class TestDiscordNotificationServicePostMessage: + """Test message posting.""" + + @patch("backend.services.discord_service.requests.post") + def test_post_message_success(self, mock_post): + """Test successful message posting.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url) + + result = service.post_message("Test message") + + assert result is True + mock_post.assert_called_once_with( + url, + json={"content": "Test message"}, + timeout=30 + ) + + @patch("backend.services.discord_service.requests.post") + def test_post_message_with_custom_webhook(self, mock_post): + """Test message posting with custom webhook URL.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + + service = DiscordNotificationService() + custom_url = "https://discord.com/api/webhooks/987654321/fedcba" + + result = service.post_message("Test message", webhook_url=custom_url) + + assert result is True + mock_post.assert_called_once_with( + custom_url, + json={"content": "Test message"}, + timeout=30 + ) + + def test_post_message_no_webhook_raises(self): + """Test that posting without webhook URL raises ValueError.""" + service = DiscordNotificationService() + + with pytest.raises(ValueError) as exc_info: + service.post_message("Test message") + assert "No Discord webhook URL provided" in str(exc_info.value) + + def test_post_message_dry_run(self): + """Test message posting in dry run mode.""" + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url, dry_run=True) + + result = service.post_message("Test message") + + assert result is True + # In dry run mode, no actual request should be made + + @patch("backend.services.discord_service.requests.post") + def test_post_message_http_error(self, mock_post): + """Test handling of HTTP errors.""" + import requests as req + mock_response = MagicMock() + mock_response.raise_for_status.side_effect = req.HTTPError("404 Not Found") + mock_post.return_value = mock_response + + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url) + + with pytest.raises(req.HTTPError): + service.post_message("Test message") + + +class TestDiscordNotificationServiceVideoNotification: + """Test video notification posting.""" + + @patch("backend.services.discord_service.requests.post") + def test_post_video_notification_success(self, mock_post): + """Test successful video notification.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url) + + result = service.post_video_notification( + "https://www.youtube.com/watch?v=abc123" + ) + + assert result is True + mock_post.assert_called_once() + call_args = mock_post.call_args + assert "New upload:" in call_args[1]["json"]["content"] + assert "abc123" in call_args[1]["json"]["content"] + + def test_post_video_notification_no_youtube_url(self): + """Test video notification with empty YouTube URL.""" + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url) + + result = service.post_video_notification("") + + assert result is False + + def test_post_video_notification_none_youtube_url(self): + """Test video notification with None YouTube URL.""" + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url) + + result = service.post_video_notification(None) + + assert result is False + + def test_post_video_notification_dry_run(self): + """Test video notification in dry run mode.""" + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url, dry_run=True) + + result = service.post_video_notification( + "https://www.youtube.com/watch?v=abc123" + ) + + assert result is True + + +class TestGetDiscordNotificationService: + """Test factory function.""" + + def test_get_service_creates_instance(self): + """Test that factory function creates a new instance.""" + import backend.services.discord_service as module + module._discord_notification_service = None + + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = get_discord_notification_service(webhook_url=url) + + assert service is not None + assert isinstance(service, DiscordNotificationService) + assert service.webhook_url == url + + def test_get_service_without_webhook(self): + """Test factory function without webhook URL.""" + import backend.services.discord_service as module + module._discord_notification_service = None + + service = get_discord_notification_service() + + assert service is not None + assert service.webhook_url is None + assert service.is_enabled() is False + + def test_get_service_with_dry_run(self): + """Test factory function with dry run mode.""" + import backend.services.discord_service as module + module._discord_notification_service = None + + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = get_discord_notification_service(webhook_url=url, dry_run=True) + + assert service.dry_run is True diff --git a/backend/tests/test_encoding_interface.py b/backend/tests/test_encoding_interface.py new file mode 100644 index 00000000..899daa5b --- /dev/null +++ b/backend/tests/test_encoding_interface.py @@ -0,0 +1,351 @@ +""" +Tests for Encoding Interface. + +Tests cover: +- EncodingInput and EncodingOutput dataclasses +- LocalEncodingBackend implementation +- Backend factory function +""" + +import pytest +from unittest.mock import MagicMock, patch, AsyncMock + +from backend.services.encoding_interface import ( + EncodingInput, + EncodingOutput, + EncodingBackend, + LocalEncodingBackend, + GCEEncodingBackend, + get_encoding_backend, +) + + +class TestEncodingInput: + """Test EncodingInput dataclass.""" + + def test_required_fields(self): + """Test that required fields must be provided.""" + input_config = EncodingInput( + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + assert input_config.title_video_path == "/path/title.mov" + assert input_config.end_video_path is None + + def test_all_fields(self): + """Test all fields including optional ones.""" + input_config = EncodingInput( + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + end_video_path="/path/end.mov", + artist="Test Artist", + title="Test Title", + brand_code="NOMAD-1234", + output_dir="/output", + options={"quality": "high"} + ) + assert input_config.artist == "Test Artist" + assert input_config.brand_code == "NOMAD-1234" + assert input_config.options["quality"] == "high" + + +class TestEncodingOutput: + """Test EncodingOutput dataclass.""" + + def test_success_output(self): + """Test successful output.""" + output = EncodingOutput( + success=True, + lossless_4k_mp4_path="/output/video.mp4", + encoding_backend="local" + ) + assert output.success is True + assert output.error_message is None + + def test_failure_output(self): + """Test failure output.""" + output = EncodingOutput( + success=False, + error_message="Encoding failed", + encoding_backend="gce" + ) + assert output.success is False + assert output.error_message == "Encoding failed" + + def test_output_files_dict(self): + """Test output_files dictionary.""" + output = EncodingOutput( + success=True, + output_files={ + "lossless_4k_mp4": "/output/lossless.mp4", + "720p_mp4": "/output/720p.mp4" + } + ) + assert "lossless_4k_mp4" in output.output_files + assert output.output_files["720p_mp4"] == "/output/720p.mp4" + + +class TestLocalEncodingBackend: + """Test LocalEncodingBackend implementation.""" + + def test_name(self): + """Test backend name.""" + backend = LocalEncodingBackend() + assert backend.name == "local" + + def test_init_with_dry_run(self): + """Test initialization with dry run.""" + backend = LocalEncodingBackend(dry_run=True) + assert backend.dry_run is True + + @pytest.mark.asyncio + @patch("subprocess.run") + async def test_is_available_success(self, mock_run): + """Test availability check when FFmpeg is installed.""" + mock_run.return_value = MagicMock(returncode=0) + + backend = LocalEncodingBackend() + available = await backend.is_available() + + assert available is True + + @pytest.mark.asyncio + @patch("subprocess.run") + async def test_is_available_not_found(self, mock_run): + """Test availability check when FFmpeg is not installed.""" + mock_run.side_effect = FileNotFoundError() + + backend = LocalEncodingBackend() + available = await backend.is_available() + + assert available is False + + @pytest.mark.asyncio + @patch.object(LocalEncodingBackend, "is_available") + @patch.object(LocalEncodingBackend, "_get_service") + async def test_get_status(self, mock_get_service, mock_is_available): + """Test status retrieval.""" + mock_is_available.return_value = True + mock_service = MagicMock() + mock_service.hwaccel_available = True + mock_service.video_encoder = "h264_nvenc" + mock_get_service.return_value = mock_service + + backend = LocalEncodingBackend() + status = await backend.get_status() + + assert status["backend"] == "local" + assert status["available"] is True + + @pytest.mark.asyncio + @patch("asyncio.to_thread") + @patch.object(LocalEncodingBackend, "_get_service") + async def test_encode_success(self, mock_get_service, mock_to_thread): + """Test successful encoding.""" + from backend.services.local_encoding_service import EncodingResult + + mock_result = EncodingResult( + success=True, + output_files={"key": "/path/file.mp4"} + ) + mock_to_thread.return_value = mock_result + + mock_service = MagicMock() + mock_get_service.return_value = mock_service + + backend = LocalEncodingBackend() + input_config = EncodingInput( + title_video_path="/input/title.mov", + karaoke_video_path="/input/karaoke.mov", + instrumental_audio_path="/input/audio.flac", + artist="Test Artist", + title="Test Title", + output_dir="/output" + ) + + output = await backend.encode(input_config) + + assert output.success is True + assert output.encoding_backend == "local" + assert output.encoding_time_seconds is not None + + @pytest.mark.asyncio + @patch("asyncio.to_thread") + @patch.object(LocalEncodingBackend, "_get_service") + async def test_encode_failure(self, mock_get_service, mock_to_thread): + """Test encoding failure.""" + from backend.services.local_encoding_service import EncodingResult + + mock_result = EncodingResult( + success=False, + output_files={}, + error="FFmpeg failed" + ) + mock_to_thread.return_value = mock_result + + mock_service = MagicMock() + mock_get_service.return_value = mock_service + + backend = LocalEncodingBackend() + input_config = EncodingInput( + title_video_path="/input/title.mov", + karaoke_video_path="/input/karaoke.mov", + instrumental_audio_path="/input/audio.flac", + artist="Test", + title="Test", + ) + + output = await backend.encode(input_config) + + assert output.success is False + assert output.error_message == "FFmpeg failed" + + +class TestGCEEncodingBackend: + """Test GCEEncodingBackend implementation.""" + + def test_name(self): + """Test backend name.""" + backend = GCEEncodingBackend() + assert backend.name == "gce" + + @patch.object(GCEEncodingBackend, "_get_service") + @pytest.mark.asyncio + async def test_is_available_enabled(self, mock_get_service): + """Test availability when GCE is enabled.""" + mock_service = MagicMock() + mock_service.is_enabled = True + mock_get_service.return_value = mock_service + + backend = GCEEncodingBackend() + available = await backend.is_available() + + assert available is True + + @patch.object(GCEEncodingBackend, "_get_service") + @pytest.mark.asyncio + async def test_is_available_disabled(self, mock_get_service): + """Test availability when GCE is disabled.""" + mock_service = MagicMock() + mock_service.is_enabled = False + mock_get_service.return_value = mock_service + + backend = GCEEncodingBackend() + available = await backend.is_available() + + assert available is False + + @patch.object(GCEEncodingBackend, "_get_service") + @pytest.mark.asyncio + async def test_encode_missing_gcs_paths(self, mock_get_service): + """Test encoding fails without GCS paths.""" + backend = GCEEncodingBackend() + input_config = EncodingInput( + title_video_path="/input/title.mov", + karaoke_video_path="/input/karaoke.mov", + instrumental_audio_path="/input/audio.flac", + # Missing GCS paths in options + ) + + output = await backend.encode(input_config) + + assert output.success is False + assert "gcs_path" in output.error_message.lower() + + @patch.object(GCEEncodingBackend, "_get_service") + @pytest.mark.asyncio + async def test_encode_success(self, mock_get_service): + """Test successful GCE encoding.""" + mock_service = MagicMock() + mock_service.encode_videos = AsyncMock(return_value={ + "status": "complete", + "output_files": { + "mp4_4k_lossless": "gs://bucket/output/lossless.mp4", + "mp4_720p": "gs://bucket/output/720p.mp4", + } + }) + mock_get_service.return_value = mock_service + + backend = GCEEncodingBackend() + input_config = EncodingInput( + title_video_path="/input/title.mov", + karaoke_video_path="/input/karaoke.mov", + instrumental_audio_path="/input/audio.flac", + artist="Test Artist", + title="Test Title", + options={ + "job_id": "test-job", + "input_gcs_path": "gs://bucket/input/", + "output_gcs_path": "gs://bucket/output/", + } + ) + + output = await backend.encode(input_config) + + assert output.success is True + assert output.encoding_backend == "gce" + mock_service.encode_videos.assert_called_once() + + @patch.object(GCEEncodingBackend, "_get_service") + @pytest.mark.asyncio + async def test_encode_failure(self, mock_get_service): + """Test GCE encoding failure handling.""" + mock_service = MagicMock() + mock_service.encode_videos = AsyncMock(side_effect=Exception("GCE worker error")) + mock_get_service.return_value = mock_service + + backend = GCEEncodingBackend() + input_config = EncodingInput( + title_video_path="/input/title.mov", + karaoke_video_path="/input/karaoke.mov", + instrumental_audio_path="/input/audio.flac", + options={ + "job_id": "test-job", + "input_gcs_path": "gs://bucket/input/", + "output_gcs_path": "gs://bucket/output/", + } + ) + + output = await backend.encode(input_config) + + assert output.success is False + assert "GCE worker error" in output.error_message + + +class TestGetEncodingBackend: + """Test encoding backend factory function.""" + + def test_get_local_backend(self): + """Test getting local backend.""" + backend = get_encoding_backend("local") + assert isinstance(backend, LocalEncodingBackend) + assert backend.name == "local" + + @patch.object(GCEEncodingBackend, "_get_service") + def test_get_auto_backend_gce_disabled(self, mock_get_service): + """Test getting auto backend falls back to local when GCE disabled.""" + mock_service = MagicMock() + mock_service.is_enabled = False + mock_get_service.return_value = mock_service + + backend = get_encoding_backend("auto") + assert isinstance(backend, LocalEncodingBackend) + + def test_get_local_backend_with_options(self): + """Test getting local backend with options.""" + backend = get_encoding_backend("local", dry_run=True) + assert backend.dry_run is True + + def test_get_gce_backend(self): + """Test getting GCE backend.""" + backend = get_encoding_backend("gce") + assert isinstance(backend, GCEEncodingBackend) + assert backend.name == "gce" + + def test_get_unknown_backend_raises(self): + """Test that unknown backend raises ValueError.""" + with pytest.raises(ValueError) as exc_info: + get_encoding_backend("unknown") + assert "Unknown encoding backend type" in str(exc_info.value) diff --git a/backend/tests/test_local_encoding_service.py b/backend/tests/test_local_encoding_service.py new file mode 100644 index 00000000..37fedd04 --- /dev/null +++ b/backend/tests/test_local_encoding_service.py @@ -0,0 +1,423 @@ +""" +Tests for LocalEncodingService. + +Tests cover: +- Service initialization +- Hardware acceleration detection +- FFmpeg command execution +- Individual encoding methods +- Full encoding pipeline +- Dry run mode +""" + +import os +import tempfile +import pytest +from unittest.mock import MagicMock, patch, call + +from backend.services.local_encoding_service import ( + LocalEncodingService, + EncodingConfig, + EncodingResult, + get_local_encoding_service, +) + + +class TestLocalEncodingServiceInit: + """Test service initialization.""" + + def test_init_default_values(self): + """Test default initialization.""" + service = LocalEncodingService() + assert service.dry_run is False + assert "ffmpeg" in service._ffmpeg_base_command + + def test_init_with_dry_run(self): + """Test initialization with dry run mode.""" + service = LocalEncodingService(dry_run=True) + assert service.dry_run is True + + def test_init_with_debug_logging(self): + """Test initialization with debug logging level.""" + import logging + service = LocalEncodingService(log_level=logging.DEBUG) + assert "-loglevel verbose" in service._ffmpeg_base_command + + def test_init_with_info_logging(self): + """Test initialization with info logging level.""" + import logging + service = LocalEncodingService(log_level=logging.INFO) + assert "-loglevel fatal" in service._ffmpeg_base_command + + +class TestLocalEncodingServiceHWAccel: + """Test hardware acceleration detection.""" + + @patch("subprocess.run") + def test_detect_nvenc_available(self, mock_run): + """Test NVENC detection when available.""" + mock_run.return_value = MagicMock(returncode=0) + + service = LocalEncodingService() + # Force detection + service._detect_and_set_hwaccel() + + assert service._hwaccel_available is True + assert service._video_encoder == "h264_nvenc" + assert service._scale_filter == "scale_cuda" + + @patch("subprocess.run") + def test_detect_no_hwaccel(self, mock_run): + """Test fallback when no hardware acceleration available.""" + import subprocess + mock_run.side_effect = subprocess.CalledProcessError(1, "test") + + service = LocalEncodingService() + # Force detection + service._detect_and_set_hwaccel() + + assert service._hwaccel_available is False + assert service._video_encoder == "libx264" + assert service._scale_filter == "scale" + + def test_nvenc_quality_settings(self): + """Test NVENC quality settings for different presets.""" + service = LocalEncodingService() + service._hwaccel_available = True + + lossless = service._get_nvenc_quality_settings("lossless") + assert "cq 0" in lossless + + medium = service._get_nvenc_quality_settings("medium") + assert "p4" in medium + + def test_nvenc_quality_settings_disabled(self): + """Test NVENC quality settings when hwaccel is disabled.""" + service = LocalEncodingService() + service._hwaccel_available = False + + settings = service._get_nvenc_quality_settings("lossless") + assert settings == "" + + +class TestLocalEncodingServiceExecuteCommand: + """Test command execution.""" + + @patch("subprocess.run") + def test_execute_command_success(self, mock_run): + """Test successful command execution.""" + mock_run.return_value = MagicMock(returncode=0) + + service = LocalEncodingService() + result = service._execute_command("echo test", "Test command") + + assert result is True + mock_run.assert_called_once() + + @patch("subprocess.run") + def test_execute_command_failure(self, mock_run): + """Test command execution failure.""" + import subprocess + mock_run.side_effect = subprocess.CalledProcessError( + 1, "test", stderr="Error message" + ) + + service = LocalEncodingService() + result = service._execute_command("echo test", "Test command") + + assert result is False + + @patch("subprocess.run") + def test_execute_command_timeout(self, mock_run): + """Test command execution timeout.""" + import subprocess + mock_run.side_effect = subprocess.TimeoutExpired("test", 30) + + service = LocalEncodingService() + result = service._execute_command("echo test", "Test command", timeout=30) + + assert result is False + + def test_execute_command_dry_run(self): + """Test command execution in dry run mode.""" + service = LocalEncodingService(dry_run=True) + result = service._execute_command("echo test", "Test command") + + assert result is True + # No actual subprocess should be called in dry run mode + + +class TestLocalEncodingServiceEncodingMethods: + """Test individual encoding methods.""" + + @patch.object(LocalEncodingService, "_execute_command") + def test_remux_with_instrumental(self, mock_execute): + """Test remuxing with instrumental audio.""" + mock_execute.return_value = True + + service = LocalEncodingService() + result = service.remux_with_instrumental( + "/input/video.mov", + "/input/audio.flac", + "/output/video.mp4" + ) + + assert result is True + mock_execute.assert_called_once() + call_args = mock_execute.call_args[0][0] + assert "/input/video.mov" in call_args + assert "/input/audio.flac" in call_args + assert "-map 0:v -map 1:a" in call_args + + @patch.object(LocalEncodingService, "_execute_command_with_fallback") + def test_convert_mov_to_mp4(self, mock_execute): + """Test MOV to MP4 conversion.""" + mock_execute.return_value = True + + service = LocalEncodingService() + result = service.convert_mov_to_mp4( + "/input/video.mov", + "/output/video.mp4" + ) + + assert result is True + mock_execute.assert_called_once() + + @patch.object(LocalEncodingService, "_execute_command_with_fallback") + def test_encode_lossless_mp4_without_end(self, mock_execute): + """Test lossless 4K MP4 encoding without end credits.""" + mock_execute.return_value = True + + service = LocalEncodingService() + result = service.encode_lossless_mp4( + "/input/title.mov", + "/input/karaoke.mp4", + "/output/lossless.mp4" + ) + + assert result is True + mock_execute.assert_called_once() + call_args = mock_execute.call_args + assert "concat=n=2" in call_args[0][0] # GPU command + + @patch.object(LocalEncodingService, "_execute_command_with_fallback") + def test_encode_lossless_mp4_with_end(self, mock_execute): + """Test lossless 4K MP4 encoding with end credits.""" + mock_execute.return_value = True + + with tempfile.TemporaryDirectory() as tmpdir: + end_file = os.path.join(tmpdir, "end.mov") + with open(end_file, "w") as f: + f.write("fake video") + + service = LocalEncodingService() + result = service.encode_lossless_mp4( + "/input/title.mov", + "/input/karaoke.mp4", + "/output/lossless.mp4", + end_video=end_file + ) + + assert result is True + call_args = mock_execute.call_args + assert "concat=n=3" in call_args[0][0] # 3 videos + + @patch.object(LocalEncodingService, "_execute_command") + def test_encode_lossy_mp4(self, mock_execute): + """Test lossy 4K MP4 encoding.""" + mock_execute.return_value = True + + service = LocalEncodingService() + result = service.encode_lossy_mp4( + "/input/lossless.mp4", + "/output/lossy.mp4" + ) + + assert result is True + call_args = mock_execute.call_args[0][0] + assert "-c:v copy" in call_args + assert "aac" in call_args.lower() + + @patch.object(LocalEncodingService, "_execute_command") + def test_encode_lossless_mkv(self, mock_execute): + """Test MKV encoding with FLAC audio.""" + mock_execute.return_value = True + + service = LocalEncodingService() + result = service.encode_lossless_mkv( + "/input/lossless.mp4", + "/output/video.mkv" + ) + + assert result is True + call_args = mock_execute.call_args[0][0] + assert "-c:v copy" in call_args + assert "-c:a flac" in call_args + + @patch.object(LocalEncodingService, "_execute_command_with_fallback") + def test_encode_720p(self, mock_execute): + """Test 720p encoding.""" + mock_execute.return_value = True + + service = LocalEncodingService() + result = service.encode_720p( + "/input/lossless.mp4", + "/output/720p.mp4" + ) + + assert result is True + call_args = mock_execute.call_args + # Should contain scale filter + assert "1280:720" in call_args[0][0] or "1280:720" in call_args[0][1] + + +class TestLocalEncodingServiceFullPipeline: + """Test full encoding pipeline.""" + + @patch.object(LocalEncodingService, "remux_with_instrumental") + @patch.object(LocalEncodingService, "convert_mov_to_mp4") + @patch.object(LocalEncodingService, "encode_lossless_mp4") + @patch.object(LocalEncodingService, "encode_lossy_mp4") + @patch.object(LocalEncodingService, "encode_lossless_mkv") + @patch.object(LocalEncodingService, "encode_720p") + def test_encode_all_formats_success( + self, mock_720p, mock_mkv, mock_lossy, mock_lossless, mock_convert, mock_remux + ): + """Test successful full encoding pipeline.""" + mock_remux.return_value = True + mock_convert.return_value = True + mock_lossless.return_value = True + mock_lossy.return_value = True + mock_mkv.return_value = True + mock_720p.return_value = True + + service = LocalEncodingService() + config = EncodingConfig( + title_video="/input/title.mov", + karaoke_video="/input/karaoke.mov", + instrumental_audio="/input/instrumental.flac", + output_karaoke_mp4="/output/karaoke.mp4", + output_with_vocals_mp4="/output/with_vocals.mp4", + output_lossless_4k_mp4="/output/lossless_4k.mp4", + output_lossy_4k_mp4="/output/lossy_4k.mp4", + output_lossless_mkv="/output/lossless.mkv", + output_720p_mp4="/output/720p.mp4", + ) + + result = service.encode_all_formats(config) + + assert result.success is True + assert "karaoke_mp4" in result.output_files + assert "lossless_4k_mp4" in result.output_files + assert "720p_mp4" in result.output_files + + @patch.object(LocalEncodingService, "remux_with_instrumental") + def test_encode_all_formats_failure_early(self, mock_remux): + """Test encoding pipeline failure at early step.""" + mock_remux.return_value = False + + service = LocalEncodingService() + config = EncodingConfig( + title_video="/input/title.mov", + karaoke_video="/input/karaoke.mov", + instrumental_audio="/input/instrumental.flac", + output_karaoke_mp4="/output/karaoke.mp4", + output_lossless_4k_mp4="/output/lossless_4k.mp4", + ) + + result = service.encode_all_formats(config) + + assert result.success is False + assert "Failed to remux" in result.error + + def test_encode_all_formats_dry_run(self): + """Test encoding pipeline in dry run mode.""" + service = LocalEncodingService(dry_run=True) + config = EncodingConfig( + title_video="/input/title.mov", + karaoke_video="/input/karaoke.mp4", # Already MP4 + instrumental_audio="/input/instrumental.flac", + output_karaoke_mp4="/output/karaoke.mp4", + output_lossless_4k_mp4="/output/lossless_4k.mp4", + output_lossy_4k_mp4="/output/lossy_4k.mp4", + output_lossless_mkv="/output/lossless.mkv", + output_720p_mp4="/output/720p.mp4", + ) + + result = service.encode_all_formats(config) + + # In dry run mode, all operations should "succeed" + assert result.success is True + + +class TestEncodingConfig: + """Test EncodingConfig dataclass.""" + + def test_config_required_fields(self): + """Test that required fields must be provided.""" + config = EncodingConfig( + title_video="/path/title.mov", + karaoke_video="/path/karaoke.mov", + instrumental_audio="/path/audio.flac", + ) + assert config.title_video == "/path/title.mov" + assert config.end_video is None # Optional field + + def test_config_all_fields(self): + """Test config with all fields.""" + config = EncodingConfig( + title_video="/path/title.mov", + karaoke_video="/path/karaoke.mov", + instrumental_audio="/path/audio.flac", + end_video="/path/end.mov", + output_karaoke_mp4="/output/karaoke.mp4", + output_720p_mp4="/output/720p.mp4", + ) + assert config.end_video == "/path/end.mov" + assert config.output_karaoke_mp4 == "/output/karaoke.mp4" + + +class TestEncodingResult: + """Test EncodingResult dataclass.""" + + def test_result_success(self): + """Test successful result.""" + result = EncodingResult( + success=True, + output_files={"key": "/path/file.mp4"} + ) + assert result.success is True + assert result.error is None + + def test_result_failure(self): + """Test failure result.""" + result = EncodingResult( + success=False, + output_files={}, + error="Something went wrong" + ) + assert result.success is False + assert result.error == "Something went wrong" + + +class TestGetLocalEncodingService: + """Test factory function.""" + + def test_get_service_creates_instance(self): + """Test that factory function creates a new instance.""" + import backend.services.local_encoding_service as module + module._local_encoding_service = None + + service = get_local_encoding_service() + + assert service is not None + assert isinstance(service, LocalEncodingService) + + def test_get_service_with_dry_run(self): + """Test factory function with dry run option.""" + import backend.services.local_encoding_service as module + module._local_encoding_service = None + + service = get_local_encoding_service(dry_run=True) + + assert service.dry_run is True diff --git a/backend/tests/test_packaging_service.py b/backend/tests/test_packaging_service.py new file mode 100644 index 00000000..b93d995c --- /dev/null +++ b/backend/tests/test_packaging_service.py @@ -0,0 +1,382 @@ +""" +Tests for PackagingService. + +Tests cover: +- Service initialization +- CDG package creation +- TXT package creation +- Dry run mode +- Error handling +""" + +import os +import zipfile +import tempfile +import pytest +from unittest.mock import MagicMock, patch + +from backend.services.packaging_service import ( + PackagingService, + get_packaging_service, +) + + +class TestPackagingServiceInit: + """Test service initialization.""" + + def test_init_default_values(self): + """Test default initialization.""" + service = PackagingService() + assert service.cdg_styles is None + assert service.dry_run is False + assert service.non_interactive is False + + def test_init_with_cdg_styles(self): + """Test initialization with CDG styles.""" + styles = {"font_size": 24, "bg_color": "black"} + service = PackagingService(cdg_styles=styles) + assert service.cdg_styles == styles + + def test_init_with_dry_run(self): + """Test initialization with dry run mode.""" + service = PackagingService(dry_run=True) + assert service.dry_run is True + + def test_init_with_non_interactive(self): + """Test initialization with non-interactive mode.""" + service = PackagingService(non_interactive=True) + assert service.non_interactive is True + + +class TestPackagingServiceCreateZipFromFiles: + """Test internal ZIP creation.""" + + def test_create_zip_from_files(self): + """Test creating a ZIP file from multiple files.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create test files + file1 = os.path.join(tmpdir, "test1.txt") + file2 = os.path.join(tmpdir, "test2.txt") + zip_path = os.path.join(tmpdir, "output.zip") + + with open(file1, "w") as f: + f.write("content1") + with open(file2, "w") as f: + f.write("content2") + + service = PackagingService() + service._create_zip_from_files( + zip_path, + [(file1, "test1.txt"), (file2, "test2.txt")] + ) + + assert os.path.isfile(zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + names = zf.namelist() + assert "test1.txt" in names + assert "test2.txt" in names + + +class TestPackagingServiceCDGPackage: + """Test CDG package creation.""" + + def test_create_cdg_package_missing_lrc_file(self): + """Test that missing LRC file raises FileNotFoundError.""" + service = PackagingService() + + with pytest.raises(FileNotFoundError) as exc_info: + service.create_cdg_package( + lrc_file="/nonexistent/file.lrc", + audio_file="/some/audio.flac", + output_zip_path="/output/test.zip", + artist="Test Artist", + title="Test Title", + ) + assert "LRC file not found" in str(exc_info.value) + + def test_create_cdg_package_missing_audio_file(self): + """Test that missing audio file raises FileNotFoundError.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + + service = PackagingService() + + with pytest.raises(FileNotFoundError) as exc_info: + service.create_cdg_package( + lrc_file=lrc_file, + audio_file="/nonexistent/audio.flac", + output_zip_path="/output/test.zip", + artist="Test Artist", + title="Test Title", + ) + assert "Audio file not found" in str(exc_info.value) + + def test_create_cdg_package_no_styles_raises(self): + """Test that CDG generation without styles raises ValueError.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + audio_file = os.path.join(tmpdir, "test.flac") + zip_path = os.path.join(tmpdir, "output.zip") + + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(audio_file, "w") as f: + f.write("fake audio") + + # Service without CDG styles + service = PackagingService(cdg_styles=None) + + with pytest.raises(ValueError) as exc_info: + service.create_cdg_package( + lrc_file=lrc_file, + audio_file=audio_file, + output_zip_path=zip_path, + artist="Test Artist", + title="Test Title", + ) + assert "CDG styles configuration is required" in str(exc_info.value) + + def test_create_cdg_package_dry_run(self): + """Test CDG package creation in dry run mode.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + audio_file = os.path.join(tmpdir, "test.flac") + zip_path = os.path.join(tmpdir, "output.zip") + + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(audio_file, "w") as f: + f.write("fake audio") + + service = PackagingService(dry_run=True) + + result = service.create_cdg_package( + lrc_file=lrc_file, + audio_file=audio_file, + output_zip_path=zip_path, + artist="Test Artist", + title="Test Title", + ) + + # In dry run, ZIP should not be created + assert not os.path.isfile(zip_path) + assert result[0] == zip_path + + def test_create_cdg_package_existing_files(self): + """Test CDG package creation with existing MP3 and CDG files.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + audio_file = os.path.join(tmpdir, "test.flac") + mp3_file = os.path.join(tmpdir, "test.mp3") + cdg_file = os.path.join(tmpdir, "test.cdg") + zip_path = os.path.join(tmpdir, "output.zip") + + # Create all files + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(audio_file, "w") as f: + f.write("fake audio") + with open(mp3_file, "w") as f: + f.write("fake mp3") + with open(cdg_file, "w") as f: + f.write("fake cdg") + + service = PackagingService() + + result = service.create_cdg_package( + lrc_file=lrc_file, + audio_file=audio_file, + output_zip_path=zip_path, + artist="Test Artist", + title="Test Title", + output_mp3_path=mp3_file, + output_cdg_path=cdg_file, + ) + + # ZIP should be created from existing files + assert os.path.isfile(zip_path) + assert result[0] == zip_path + assert result[1] == mp3_file + assert result[2] == cdg_file + + @patch("lyrics_transcriber.output.cdg.CDGGenerator") + def test_create_cdg_package_with_generator(self, mock_cdg_generator_class): + """Test CDG package creation using CDGGenerator.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + audio_file = os.path.join(tmpdir, "test.flac") + zip_path = os.path.join(tmpdir, "output.zip") + generated_zip = os.path.join(tmpdir, "generated.zip") + + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(audio_file, "w") as f: + f.write("fake audio") + + # Create the zip file that would be generated + with zipfile.ZipFile(generated_zip, "w") as zf: + zf.writestr("test.mp3", "fake mp3") + zf.writestr("test.cdg", "fake cdg") + + mock_generator = MagicMock() + mock_generator.generate_cdg_from_lrc.return_value = ( + "test.cdg", "test.mp3", generated_zip + ) + mock_cdg_generator_class.return_value = mock_generator + + styles = {"font_size": 24} + service = PackagingService(cdg_styles=styles) + + result = service.create_cdg_package( + lrc_file=lrc_file, + audio_file=audio_file, + output_zip_path=zip_path, + artist="Test Artist", + title="Test Title", + ) + + mock_generator.generate_cdg_from_lrc.assert_called_once_with( + lrc_file=lrc_file, + audio_file=audio_file, + title="Test Title", + artist="Test Artist", + cdg_styles=styles, + ) + assert os.path.isfile(zip_path) + + +class TestPackagingServiceTXTPackage: + """Test TXT package creation.""" + + def test_create_txt_package_missing_lrc_file(self): + """Test that missing LRC file raises FileNotFoundError.""" + service = PackagingService() + + with pytest.raises(FileNotFoundError) as exc_info: + service.create_txt_package( + lrc_file="/nonexistent/file.lrc", + mp3_file="/some/audio.mp3", + output_zip_path="/output/test.zip", + ) + assert "LRC file not found" in str(exc_info.value) + + def test_create_txt_package_missing_mp3_file(self): + """Test that missing MP3 file raises FileNotFoundError.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + + service = PackagingService() + + with pytest.raises(FileNotFoundError) as exc_info: + service.create_txt_package( + lrc_file=lrc_file, + mp3_file="/nonexistent/audio.mp3", + output_zip_path="/output/test.zip", + ) + assert "MP3 file not found" in str(exc_info.value) + + def test_create_txt_package_dry_run(self): + """Test TXT package creation in dry run mode.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + mp3_file = os.path.join(tmpdir, "test.mp3") + zip_path = os.path.join(tmpdir, "output.zip") + + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(mp3_file, "w") as f: + f.write("fake mp3") + + service = PackagingService(dry_run=True) + + result = service.create_txt_package( + lrc_file=lrc_file, + mp3_file=mp3_file, + output_zip_path=zip_path, + ) + + # In dry run, ZIP should not be created + assert not os.path.isfile(zip_path) + assert result[0] == zip_path + + @patch("lyrics_converter.LyricsConverter") + def test_create_txt_package_success(self, mock_converter_class): + """Test successful TXT package creation.""" + with tempfile.TemporaryDirectory() as tmpdir: + lrc_file = os.path.join(tmpdir, "test.lrc") + mp3_file = os.path.join(tmpdir, "test.mp3") + zip_path = os.path.join(tmpdir, "output.zip") + txt_path = os.path.join(tmpdir, "output.txt") + + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(mp3_file, "w") as f: + f.write("fake mp3") + + mock_converter = MagicMock() + mock_converter.convert_file.return_value = "Converted lyrics text" + mock_converter_class.return_value = mock_converter + + service = PackagingService() + + result = service.create_txt_package( + lrc_file=lrc_file, + mp3_file=mp3_file, + output_zip_path=zip_path, + output_txt_path=txt_path, + ) + + mock_converter_class.assert_called_once_with( + output_format="txt", + filepath=lrc_file + ) + mock_converter.convert_file.assert_called_once() + + assert os.path.isfile(zip_path) + assert os.path.isfile(txt_path) + assert result[0] == zip_path + assert result[1] == txt_path + + # Verify ZIP contents + with zipfile.ZipFile(zip_path, "r") as zf: + names = zf.namelist() + assert "test.mp3" in names + assert "output.txt" in names + + +class TestGetPackagingService: + """Test factory function.""" + + def test_get_service_creates_instance(self): + """Test that factory function creates a new instance.""" + import backend.services.packaging_service as module + module._packaging_service = None + + service = get_packaging_service() + + assert service is not None + assert isinstance(service, PackagingService) + + def test_get_service_with_cdg_styles(self): + """Test factory function with CDG styles.""" + import backend.services.packaging_service as module + module._packaging_service = None + + styles = {"font_size": 24} + service = get_packaging_service(cdg_styles=styles) + + assert service.cdg_styles == styles + + def test_get_service_with_options(self): + """Test factory function with additional options.""" + import backend.services.packaging_service as module + module._packaging_service = None + + service = get_packaging_service(dry_run=True, non_interactive=True) + + assert service.dry_run is True + assert service.non_interactive is True diff --git a/backend/tests/test_video_worker_orchestrator.py b/backend/tests/test_video_worker_orchestrator.py new file mode 100644 index 00000000..511511b3 --- /dev/null +++ b/backend/tests/test_video_worker_orchestrator.py @@ -0,0 +1,698 @@ +""" +Tests for Video Worker Orchestrator. + +Tests cover: +- OrchestratorConfig and OrchestratorResult dataclasses +- VideoWorkerOrchestrator initialization +- Individual stage methods +- Full pipeline execution +- Error handling and recovery +""" + +import os +import tempfile +import pytest +from unittest.mock import MagicMock, patch, AsyncMock +from dataclasses import asdict + +from backend.workers.video_worker_orchestrator import ( + OrchestratorConfig, + OrchestratorResult, + VideoWorkerOrchestrator, + create_orchestrator_config_from_job, +) + + +class TestOrchestratorConfig: + """Test OrchestratorConfig dataclass.""" + + def test_required_fields(self): + """Test that required fields must be provided.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + assert config.job_id == "test-job" + assert config.artist == "Test Artist" + assert config.encoding_backend == "auto" + + def test_default_values(self): + """Test default values for optional fields.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + assert config.enable_cdg is False + assert config.enable_txt is False + assert config.enable_youtube_upload is False + assert config.dry_run is False + assert config.non_interactive is True + assert config.end_video_path is None + + def test_all_fields(self): + """Test all fields including optional ones.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + end_video_path="/path/end.mov", + enable_cdg=True, + enable_txt=True, + enable_youtube_upload=True, + brand_prefix="TEST", + discord_webhook_url="https://discord.com/api/webhooks/123/abc", + encoding_backend="gce", + ) + assert config.enable_cdg is True + assert config.encoding_backend == "gce" + assert config.discord_webhook_url is not None + + +class TestOrchestratorResult: + """Test OrchestratorResult dataclass.""" + + def test_success_result(self): + """Test successful result.""" + result = OrchestratorResult( + success=True, + final_video="/output/video.mp4", + brand_code="TEST-1234", + youtube_url="https://youtube.com/watch?v=test", + ) + assert result.success is True + assert result.error_message is None + assert result.brand_code == "TEST-1234" + + def test_failure_result(self): + """Test failure result.""" + result = OrchestratorResult( + success=False, + error_message="Encoding failed", + ) + assert result.success is False + assert result.error_message == "Encoding failed" + assert result.final_video is None + + def test_all_output_files(self): + """Test all output file fields.""" + result = OrchestratorResult( + success=True, + final_video="/output/lossless.mp4", + final_video_mkv="/output/lossless.mkv", + final_video_lossy="/output/lossy.mp4", + final_video_720p="/output/720p.mp4", + final_karaoke_cdg_zip="/output/cdg.zip", + final_karaoke_txt_zip="/output/txt.zip", + ) + assert result.final_video == "/output/lossless.mp4" + assert result.final_video_mkv == "/output/lossless.mkv" + + +class TestVideoWorkerOrchestratorInit: + """Test VideoWorkerOrchestrator initialization.""" + + def test_init_with_config(self): + """Test initialization with config.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + orchestrator = VideoWorkerOrchestrator(config) + + assert orchestrator.config == config + assert orchestrator.result.success is False + + def test_init_with_job_manager(self): + """Test initialization with job manager.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + job_manager = MagicMock() + orchestrator = VideoWorkerOrchestrator(config, job_manager=job_manager) + + assert orchestrator.job_manager == job_manager + + +class TestVideoWorkerOrchestratorServices: + """Test service lazy-loading.""" + + def test_get_encoding_backend(self): + """Test encoding backend lazy loading.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + encoding_backend="local", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch("backend.services.encoding_interface.get_encoding_backend") as mock_get: + mock_backend = MagicMock() + mock_backend.name = "local" + mock_get.return_value = mock_backend + + backend = orchestrator._get_encoding_backend() + + assert backend == mock_backend + mock_get.assert_called_once() + + def test_get_packaging_service(self): + """Test packaging service lazy loading.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + cdg_styles={"background_color": "black"}, + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch("backend.services.packaging_service.PackagingService") as MockService: + mock_service = MagicMock() + MockService.return_value = mock_service + + service = orchestrator._get_packaging_service() + + assert service == mock_service + MockService.assert_called_once() + + def test_get_youtube_service(self): + """Test YouTube service lazy loading.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + youtube_credentials={"token": "test"}, + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch("backend.services.youtube_upload_service.YouTubeUploadService") as MockService: + mock_service = MagicMock() + MockService.return_value = mock_service + + service = orchestrator._get_youtube_service() + + assert service == mock_service + MockService.assert_called_once() + + def test_get_discord_service(self): + """Test Discord service lazy loading.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + discord_webhook_url="https://discord.com/api/webhooks/123/abc", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch("backend.services.discord_service.DiscordNotificationService") as MockService: + mock_service = MagicMock() + MockService.return_value = mock_service + + service = orchestrator._get_discord_service() + + assert service == mock_service + MockService.assert_called_once() + + +class TestVideoWorkerOrchestratorPackaging: + """Test packaging stage.""" + + @pytest.mark.asyncio + async def test_run_packaging_no_lrc(self): + """Test packaging stage with no LRC file.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=True, + lrc_file_path=None, + ) + orchestrator = VideoWorkerOrchestrator(config) + + # Should not raise, just skip + await orchestrator._run_packaging() + + assert orchestrator.result.final_karaoke_cdg_zip is None + + @pytest.mark.asyncio + async def test_run_packaging_cdg(self): + """Test CDG packaging.""" + with tempfile.TemporaryDirectory() as temp_dir: + lrc_file = os.path.join(temp_dir, "test.lrc") + audio_file = os.path.join(temp_dir, "test.flac") + + # Create dummy files + with open(lrc_file, "w") as f: + f.write("[00:00.00]Test lyrics") + with open(audio_file, "w") as f: + f.write("dummy audio") + + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path=audio_file, + lrc_file_path=lrc_file, + output_dir=temp_dir, + enable_cdg=True, + cdg_styles={"background_color": "black"}, + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch.object(orchestrator, "_get_packaging_service") as mock_get: + mock_service = MagicMock() + mock_service.create_cdg_package.return_value = ( + f"{temp_dir}/cdg.zip", + f"{temp_dir}/test.mp3", + f"{temp_dir}/test.cdg", + ) + mock_get.return_value = mock_service + + await orchestrator._run_packaging() + + mock_service.create_cdg_package.assert_called_once() + assert orchestrator.result.final_karaoke_cdg_zip == f"{temp_dir}/cdg.zip" + + +class TestVideoWorkerOrchestratorEncoding: + """Test encoding stage.""" + + @pytest.mark.asyncio + async def test_run_encoding_success(self): + """Test successful encoding.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + output_dir="/output", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch.object(orchestrator, "_get_encoding_backend") as mock_get: + from backend.services.encoding_interface import EncodingOutput + + mock_backend = MagicMock() + mock_backend.name = "local" + mock_backend.encode = AsyncMock(return_value=EncodingOutput( + success=True, + lossless_4k_mp4_path="/output/lossless.mp4", + lossy_4k_mp4_path="/output/lossy.mp4", + lossy_720p_mp4_path="/output/720p.mp4", + lossless_mkv_path="/output/lossless.mkv", + encoding_time_seconds=120.5, + encoding_backend="local", + )) + mock_get.return_value = mock_backend + + await orchestrator._run_encoding() + + mock_backend.encode.assert_called_once() + assert orchestrator.result.final_video == "/output/lossless.mp4" + assert orchestrator.result.final_video_720p == "/output/720p.mp4" + assert orchestrator.result.encoding_time_seconds == 120.5 + + @pytest.mark.asyncio + async def test_run_encoding_failure(self): + """Test encoding failure.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch.object(orchestrator, "_get_encoding_backend") as mock_get: + from backend.services.encoding_interface import EncodingOutput + + mock_backend = MagicMock() + mock_backend.name = "local" + mock_backend.encode = AsyncMock(return_value=EncodingOutput( + success=False, + error_message="FFmpeg failed", + encoding_backend="local", + )) + mock_get.return_value = mock_backend + + with pytest.raises(Exception) as exc_info: + await orchestrator._run_encoding() + + assert "Encoding failed" in str(exc_info.value) + + +class TestVideoWorkerOrchestratorOrganization: + """Test organization stage.""" + + @pytest.mark.asyncio + async def test_run_organization_keep_brand_code(self): + """Test organization with existing brand code.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + keep_brand_code="NOMAD-9999", + ) + orchestrator = VideoWorkerOrchestrator(config) + + await orchestrator._run_organization() + + assert orchestrator.result.brand_code == "NOMAD-9999" + + @pytest.mark.asyncio + async def test_run_organization_generate_brand_code(self): + """Test brand code generation from Dropbox.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + dropbox_path="/Karaoke/Tracks", + brand_prefix="TEST", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch("backend.services.dropbox_service.get_dropbox_service") as mock_get: + mock_dropbox = MagicMock() + mock_dropbox.is_configured = True + mock_dropbox.get_next_brand_code.return_value = "TEST-0001" + mock_get.return_value = mock_dropbox + + await orchestrator._run_organization() + + assert orchestrator.result.brand_code == "TEST-0001" + + +class TestVideoWorkerOrchestratorDistribution: + """Test distribution stage.""" + + @pytest.mark.asyncio + async def test_upload_to_youtube(self): + """Test YouTube upload.""" + with tempfile.TemporaryDirectory() as temp_dir: + video_file = os.path.join(temp_dir, "test.mp4") + with open(video_file, "w") as f: + f.write("dummy video") + + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_youtube_upload=True, + youtube_credentials={"token": "test"}, + ) + orchestrator = VideoWorkerOrchestrator(config) + orchestrator.result.final_video_lossy = video_file + + with patch.object(orchestrator, "_get_youtube_service") as mock_get: + mock_service = MagicMock() + mock_service.upload_video.return_value = ( + "video123", + "https://youtube.com/watch?v=video123" + ) + mock_get.return_value = mock_service + + await orchestrator._upload_to_youtube() + + mock_service.upload_video.assert_called_once() + assert orchestrator.result.youtube_url == "https://youtube.com/watch?v=video123" + + @pytest.mark.asyncio + async def test_upload_to_dropbox(self): + """Test Dropbox upload.""" + with tempfile.TemporaryDirectory() as temp_dir: + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + output_dir=temp_dir, + dropbox_path="/Karaoke/Tracks", + brand_prefix="TEST", + ) + orchestrator = VideoWorkerOrchestrator(config) + orchestrator.result.brand_code = "TEST-0001" + + with patch("backend.services.dropbox_service.get_dropbox_service") as mock_get: + mock_dropbox = MagicMock() + mock_dropbox.is_configured = True + mock_dropbox.create_shared_link.return_value = "https://dropbox.com/link" + mock_get.return_value = mock_dropbox + + await orchestrator._upload_to_dropbox() + + mock_dropbox.upload_folder.assert_called_once() + assert orchestrator.result.dropbox_link == "https://dropbox.com/link" + + +class TestVideoWorkerOrchestratorNotifications: + """Test notifications stage.""" + + @pytest.mark.asyncio + async def test_run_notifications_with_youtube_url(self): + """Test Discord notification with YouTube URL.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + discord_webhook_url="https://discord.com/api/webhooks/123/abc", + ) + orchestrator = VideoWorkerOrchestrator(config) + orchestrator.result.youtube_url = "https://youtube.com/watch?v=test" + + with patch.object(orchestrator, "_get_discord_service") as mock_get: + mock_service = MagicMock() + mock_service.post_video_notification.return_value = True + mock_get.return_value = mock_service + + await orchestrator._run_notifications() + + mock_service.post_video_notification.assert_called_once_with( + "https://youtube.com/watch?v=test" + ) + + @pytest.mark.asyncio + async def test_run_notifications_no_youtube_url(self): + """Test notification skipped without YouTube URL.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + discord_webhook_url="https://discord.com/api/webhooks/123/abc", + ) + orchestrator = VideoWorkerOrchestrator(config) + # No youtube_url set + + with patch.object(orchestrator, "_get_discord_service") as mock_get: + mock_service = MagicMock() + mock_get.return_value = mock_service + + await orchestrator._run_notifications() + + mock_service.post_video_notification.assert_not_called() + + +class TestVideoWorkerOrchestratorFullPipeline: + """Test full pipeline execution.""" + + @pytest.mark.asyncio + async def test_run_full_pipeline_success(self): + """Test successful full pipeline.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=True, # Enable to trigger packaging stage + ) + orchestrator = VideoWorkerOrchestrator(config) + + # Mock all stages + with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_packaging, \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \ + patch.object(orchestrator, "_run_organization", new_callable=AsyncMock) as mock_org, \ + patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock) as mock_dist, \ + patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock) as mock_notify: + + result = await orchestrator.run() + + assert result.success is True + mock_packaging.assert_called_once() + mock_encoding.assert_called_once() + mock_org.assert_called_once() + mock_dist.assert_called_once() + mock_notify.assert_called_once() + + @pytest.mark.asyncio + async def test_run_pipeline_skips_packaging_when_disabled(self): + """Test pipeline skips packaging when CDG/TXT disabled.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + enable_cdg=False, + enable_txt=False, + ) + orchestrator = VideoWorkerOrchestrator(config) + + # Mock all stages + with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock) as mock_packaging, \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \ + patch.object(orchestrator, "_run_organization", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_distribution", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_notifications", new_callable=AsyncMock): + + result = await orchestrator.run() + + assert result.success is True + mock_packaging.assert_not_called() + mock_encoding.assert_called_once() + + @pytest.mark.asyncio + async def test_run_full_pipeline_encoding_failure(self): + """Test pipeline failure during encoding.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + ) + orchestrator = VideoWorkerOrchestrator(config) + + with patch.object(orchestrator, "_run_packaging", new_callable=AsyncMock), \ + patch.object(orchestrator, "_run_encoding", new_callable=AsyncMock) as mock_encoding, \ + patch.object(orchestrator, "_run_organization", new_callable=AsyncMock) as mock_org: + + mock_encoding.side_effect = Exception("Encoding failed") + + result = await orchestrator.run() + + assert result.success is False + assert "Encoding failed" in result.error_message + mock_org.assert_not_called() + + +class TestCreateOrchestratorConfigFromJob: + """Test helper function for creating config from job.""" + + def test_create_config_from_job(self): + """Test creating config from a job object.""" + job = MagicMock() + job.job_id = "test-123" + job.artist = "Test Artist" + job.title = "Test Title" + job.state_data = {"instrumental_selection": "clean"} + job.enable_cdg = True + job.enable_txt = False + job.enable_youtube_upload = True + job.brand_prefix = "NOMAD" + job.discord_webhook_url = "https://discord.com/api/webhooks/123/abc" + job.youtube_description_template = "Test description" + job.dropbox_path = "/Karaoke" + job.gdrive_folder_id = None + job.keep_brand_code = None + job.existing_instrumental_gcs_path = None + + config = create_orchestrator_config_from_job( + job=job, + temp_dir="/tmp/test", + youtube_credentials={"token": "test"}, + cdg_styles={"background": "black"}, + ) + + assert config.job_id == "test-123" + assert config.artist == "Test Artist" + assert config.title == "Test Title" + assert config.enable_cdg is True + assert config.enable_youtube_upload is True + assert config.title_video_path == "/tmp/test/Test Artist - Test Title (Title).mov" + assert config.instrumental_audio_path == "/tmp/test/Test Artist - Test Title (Instrumental Clean).flac" + + def test_create_config_from_job_with_existing_instrumental(self): + """Test config with user-provided instrumental.""" + job = MagicMock() + job.job_id = "test-123" + job.artist = "Test Artist" + job.title = "Test Title" + job.state_data = {"instrumental_selection": "custom"} + job.enable_cdg = False + job.enable_txt = False + job.enable_youtube_upload = False + job.brand_prefix = None + job.discord_webhook_url = None + job.youtube_description_template = None + job.dropbox_path = None + job.gdrive_folder_id = None + job.keep_brand_code = None + job.existing_instrumental_gcs_path = "gs://bucket/instrumental.mp3" + + config = create_orchestrator_config_from_job( + job=job, + temp_dir="/tmp/test", + ) + + assert config.instrumental_audio_path == "/tmp/test/Test Artist - Test Title (Instrumental User).mp3" diff --git a/backend/tests/test_youtube_upload_service.py b/backend/tests/test_youtube_upload_service.py new file mode 100644 index 00000000..559fe547 --- /dev/null +++ b/backend/tests/test_youtube_upload_service.py @@ -0,0 +1,568 @@ +""" +Tests for YouTubeUploadService. + +Tests cover: +- Authentication with pre-stored credentials +- Authentication with client secrets file +- Duplicate video detection (exact and fuzzy matching) +- Video deletion +- Video upload with metadata and thumbnail +- Title truncation +- Dry run mode +""" + +import os +import pytest +from unittest.mock import MagicMock, patch, mock_open + +from backend.services.youtube_upload_service import YouTubeUploadService, get_youtube_upload_service + + +class TestYouTubeUploadServiceInit: + """Test service initialization.""" + + def test_init_with_credentials(self): + """Test initialization with pre-stored credentials.""" + creds = { + "token": "test_token", + "refresh_token": "test_refresh", + "token_uri": "https://oauth2.googleapis.com/token", + "client_id": "test_client_id", + "client_secret": "test_secret", + } + service = YouTubeUploadService( + credentials=creds, + non_interactive=True + ) + assert service.credentials == creds + assert service.non_interactive is True + + def test_init_with_client_secrets(self): + """Test initialization with client secrets file.""" + service = YouTubeUploadService( + client_secrets_file="/path/to/secrets.json" + ) + assert service.client_secrets_file == "/path/to/secrets.json" + assert service.non_interactive is False + + def test_init_default_values(self): + """Test default values on initialization.""" + service = YouTubeUploadService() + assert service.credentials is None + assert service.client_secrets_file is None + assert service.non_interactive is False + assert service.server_side_mode is False + assert service.dry_run is False + + +class TestYouTubeUploadServiceAuthentication: + """Test authentication methods.""" + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_authenticate_with_prestored_credentials( + self, mock_request, mock_credentials_class, mock_build + ): + """Test authentication using pre-stored credentials.""" + # Setup mocks + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + mock_youtube = MagicMock() + mock_build.return_value = mock_youtube + + creds = { + "token": "test_token", + "refresh_token": "test_refresh", + "token_uri": "https://oauth2.googleapis.com/token", + "client_id": "test_client_id", + "client_secret": "test_secret", + "scopes": ["https://www.googleapis.com/auth/youtube"], + } + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + # Access youtube_service to trigger authentication + result = service.youtube_service + + assert result == mock_youtube + mock_credentials_class.assert_called_once() + mock_build.assert_called_once_with('youtube', 'v3', credentials=mock_creds) + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_authenticate_refreshes_expired_token( + self, mock_request, mock_credentials_class, mock_build + ): + """Test that expired tokens are refreshed.""" + mock_creds = MagicMock() + mock_creds.expired = True + mock_creds.refresh_token = "test_refresh" + mock_credentials_class.return_value = mock_creds + + creds = { + "token": "expired_token", + "refresh_token": "test_refresh", + "token_uri": "https://oauth2.googleapis.com/token", + "client_id": "test_client_id", + "client_secret": "test_secret", + } + service = YouTubeUploadService(credentials=creds, non_interactive=True) + service.youtube_service + + mock_creds.refresh.assert_called_once() + + def test_authenticate_non_interactive_no_credentials_raises(self): + """Test that non-interactive mode without credentials raises error.""" + service = YouTubeUploadService(non_interactive=True) + + with pytest.raises(Exception) as exc_info: + service._authenticate() + + assert "non-interactive mode" in str(exc_info.value).lower() + + +class TestYouTubeUploadServiceChannelId: + """Test channel ID retrieval.""" + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_get_channel_id_success( + self, mock_request, mock_credentials_class, mock_build + ): + """Test successful channel ID retrieval.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_channels = MagicMock() + mock_list = MagicMock() + mock_list.execute.return_value = { + "items": [{"id": "UC123456"}] + } + mock_channels.list.return_value = mock_list + mock_youtube.channels.return_value = mock_channels + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + channel_id = service.get_channel_id() + + assert channel_id == "UC123456" + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_get_channel_id_no_items( + self, mock_request, mock_credentials_class, mock_build + ): + """Test channel ID returns None when no items found.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_channels = MagicMock() + mock_list = MagicMock() + mock_list.execute.return_value = {"items": []} + mock_channels.list.return_value = mock_list + mock_youtube.channels.return_value = mock_channels + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + channel_id = service.get_channel_id() + + assert channel_id is None + + +class TestYouTubeUploadServiceDuplicateCheck: + """Test duplicate video detection.""" + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_check_duplicate_exact_match_found( + self, mock_request, mock_credentials_class, mock_build + ): + """Test duplicate detection with exact match in server mode.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + # Mock channels().list() for get_channel_id + mock_channels = MagicMock() + mock_channels_list = MagicMock() + mock_channels_list.execute.return_value = {"items": [{"id": "UC123456"}]} + mock_channels.list.return_value = mock_channels_list + mock_youtube.channels.return_value = mock_channels + + # Mock search().list() for duplicate check + mock_search = MagicMock() + mock_search_list = MagicMock() + mock_search_list.execute.return_value = { + "items": [{ + "id": {"videoId": "VIDEO123"}, + "snippet": { + "channelId": "UC123456", + "title": "Test Artist - Test Song (Karaoke)" + } + }] + } + mock_search.list.return_value = mock_search_list + mock_youtube.search.return_value = mock_search + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService( + credentials=creds, + non_interactive=True, + server_side_mode=True + ) + + exists, video_id, video_url = service.check_duplicate( + "Test Artist - Test Song (Karaoke)" + ) + + assert exists is True + assert video_id == "VIDEO123" + assert "VIDEO123" in video_url + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_check_duplicate_no_match( + self, mock_request, mock_credentials_class, mock_build + ): + """Test duplicate detection when no match found.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_channels = MagicMock() + mock_channels_list = MagicMock() + mock_channels_list.execute.return_value = {"items": [{"id": "UC123456"}]} + mock_channels.list.return_value = mock_channels_list + mock_youtube.channels.return_value = mock_channels + + mock_search = MagicMock() + mock_search_list = MagicMock() + mock_search_list.execute.return_value = {"items": []} + mock_search.list.return_value = mock_search_list + mock_youtube.search.return_value = mock_search + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + exists, video_id, video_url = service.check_duplicate("Some Title") + + assert exists is False + assert video_id is None + assert video_url is None + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_check_duplicate_skips_other_channels( + self, mock_request, mock_credentials_class, mock_build + ): + """Test that videos from other channels are skipped.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_channels = MagicMock() + mock_channels_list = MagicMock() + mock_channels_list.execute.return_value = {"items": [{"id": "UC123456"}]} + mock_channels.list.return_value = mock_channels_list + mock_youtube.channels.return_value = mock_channels + + # Return video from different channel + mock_search = MagicMock() + mock_search_list = MagicMock() + mock_search_list.execute.return_value = { + "items": [{ + "id": {"videoId": "VIDEO123"}, + "snippet": { + "channelId": "UC_DIFFERENT", # Different channel + "title": "Test Title" + } + }] + } + mock_search.list.return_value = mock_search_list + mock_youtube.search.return_value = mock_search + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + exists, video_id, video_url = service.check_duplicate("Test Title") + + assert exists is False + + +class TestYouTubeUploadServiceDelete: + """Test video deletion.""" + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_delete_video_success( + self, mock_request, mock_credentials_class, mock_build + ): + """Test successful video deletion.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_videos = MagicMock() + mock_delete = MagicMock() + mock_delete.execute.return_value = None + mock_videos.delete.return_value = mock_delete + mock_youtube.videos.return_value = mock_videos + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + result = service.delete_video("VIDEO123") + + assert result is True + mock_videos.delete.assert_called_once_with(id="VIDEO123") + + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_delete_video_failure( + self, mock_request, mock_credentials_class, mock_build + ): + """Test video deletion failure handling.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_videos = MagicMock() + mock_delete = MagicMock() + mock_delete.execute.side_effect = Exception("API Error") + mock_videos.delete.return_value = mock_delete + mock_youtube.videos.return_value = mock_videos + mock_build.return_value = mock_youtube + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + result = service.delete_video("VIDEO123") + + assert result is False + + def test_delete_video_dry_run(self): + """Test video deletion in dry run mode.""" + service = YouTubeUploadService(dry_run=True) + + result = service.delete_video("VIDEO123") + + assert result is True + + +class TestYouTubeUploadServiceTitleTruncation: + """Test title truncation.""" + + def test_truncate_title_short_title(self): + """Test that short titles are not truncated.""" + title = "Short Title" + result = YouTubeUploadService.truncate_title(title) + assert result == title + + def test_truncate_title_exact_length(self): + """Test title at exact max length.""" + title = "A" * 95 + result = YouTubeUploadService.truncate_title(title, max_length=95) + assert result == title + assert len(result) == 95 + + def test_truncate_title_long_title(self): + """Test that long titles are truncated at word boundary.""" + title = "This is a very long title that exceeds the maximum length and needs to be truncated properly at a word boundary" + result = YouTubeUploadService.truncate_title(title, max_length=50) + assert len(result) <= 50 + assert result.endswith("...") + + def test_truncate_title_no_space(self): + """Test truncation of title without spaces.""" + title = "A" * 100 + result = YouTubeUploadService.truncate_title(title, max_length=50) + assert len(result) <= 50 + + +class TestYouTubeUploadServiceUpload: + """Test video upload.""" + + @patch("googleapiclient.http.MediaFileUpload") + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + def test_upload_video_success( + self, mock_request, mock_credentials_class, mock_build, mock_media_upload + ): + """Test successful video upload.""" + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + # Mock channels for get_channel_id + mock_channels = MagicMock() + mock_channels_list = MagicMock() + mock_channels_list.execute.return_value = {"items": [{"id": "UC123456"}]} + mock_channels.list.return_value = mock_channels_list + mock_youtube.channels.return_value = mock_channels + + # Mock search for duplicate check (no duplicates) + mock_search = MagicMock() + mock_search_list = MagicMock() + mock_search_list.execute.return_value = {"items": []} + mock_search.list.return_value = mock_search_list + mock_youtube.search.return_value = mock_search + + # Mock video insert + mock_videos = MagicMock() + mock_insert = MagicMock() + mock_insert.execute.return_value = {"id": "NEW_VIDEO_ID"} + mock_videos.insert.return_value = mock_insert + mock_youtube.videos.return_value = mock_videos + mock_build.return_value = mock_youtube + + mock_media_upload.return_value = MagicMock() + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + video_id, video_url = service.upload_video( + video_path="/path/to/video.mkv", + title="Test Artist - Test Song (Karaoke)", + description="Test description", + tags=["karaoke", "test"] + ) + + assert video_id == "NEW_VIDEO_ID" + assert "NEW_VIDEO_ID" in video_url + + def test_upload_video_dry_run(self): + """Test video upload in dry run mode.""" + service = YouTubeUploadService(dry_run=True) + + video_id, video_url = service.upload_video( + video_path="/path/to/video.mkv", + title="Test Title", + description="Test description" + ) + + assert video_id == "dry_run_video_id" + assert "dry_run_video_id" in video_url + + @patch("googleapiclient.http.MediaFileUpload") + @patch("googleapiclient.discovery.build") + @patch("google.oauth2.credentials.Credentials") + @patch("google.auth.transport.requests.Request") + @patch("os.path.isfile") + def test_upload_video_with_thumbnail( + self, mock_isfile, mock_request, mock_credentials_class, + mock_build, mock_media_upload + ): + """Test video upload with thumbnail.""" + mock_isfile.return_value = True + mock_creds = MagicMock() + mock_creds.expired = False + mock_credentials_class.return_value = mock_creds + + mock_youtube = MagicMock() + mock_channels = MagicMock() + mock_channels_list = MagicMock() + mock_channels_list.execute.return_value = {"items": [{"id": "UC123456"}]} + mock_channels.list.return_value = mock_channels_list + mock_youtube.channels.return_value = mock_channels + + mock_search = MagicMock() + mock_search_list = MagicMock() + mock_search_list.execute.return_value = {"items": []} + mock_search.list.return_value = mock_search_list + mock_youtube.search.return_value = mock_search + + mock_videos = MagicMock() + mock_insert = MagicMock() + mock_insert.execute.return_value = {"id": "NEW_VIDEO_ID"} + mock_videos.insert.return_value = mock_insert + mock_youtube.videos.return_value = mock_videos + + # Mock thumbnail upload + mock_thumbnails = MagicMock() + mock_set = MagicMock() + mock_set.execute.return_value = None + mock_thumbnails.set.return_value = mock_set + mock_youtube.thumbnails.return_value = mock_thumbnails + + mock_build.return_value = mock_youtube + mock_media_upload.return_value = MagicMock() + + creds = {"token": "test", "refresh_token": "test", "token_uri": "uri", + "client_id": "id", "client_secret": "secret"} + service = YouTubeUploadService(credentials=creds, non_interactive=True) + + video_id, video_url = service.upload_video( + video_path="/path/to/video.mkv", + title="Test Title", + description="Test description", + thumbnail_path="/path/to/thumbnail.jpg" + ) + + assert video_id == "NEW_VIDEO_ID" + mock_thumbnails.set.assert_called_once() + + +class TestGetYouTubeUploadService: + """Test factory function.""" + + def test_get_service_creates_instance(self): + """Test that factory function creates a new instance.""" + # Reset global + import backend.services.youtube_upload_service as module + module._youtube_upload_service = None + + service = get_youtube_upload_service( + credentials={"token": "test"}, + non_interactive=True + ) + + assert service is not None + assert isinstance(service, YouTubeUploadService) + + def test_get_service_with_client_secrets(self): + """Test factory function with client secrets file.""" + import backend.services.youtube_upload_service as module + module._youtube_upload_service = None + + service = get_youtube_upload_service( + client_secrets_file="/path/to/secrets.json" + ) + + assert service is not None + assert service.client_secrets_file == "/path/to/secrets.json" diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index 2ae34a82..f78efd7c 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -1,20 +1,21 @@ """ Video generation and finalization worker. -This worker uses KaraokeFinalise from the karaoke_gen package to handle: -1. Video encoding (4 formats) -2. CDG/TXT package generation -3. Discord notifications -4. Brand code generation -5. All other finalisation features +This worker has two code paths controlled by USE_NEW_ORCHESTRATOR: -The worker's job is simply to: -1. Download files from GCS and set up the directory structure KaraokeFinalise expects -2. Instantiate KaraokeFinalise with parameters from the job -3. Call process() -4. Upload results back to GCS +1. NEW PATH (USE_NEW_ORCHESTRATOR=true): + Uses VideoWorkerOrchestrator which provides a unified pipeline for all + encoding backends (GCE or local). This ensures features like YouTube upload, + Discord notifications, and CDG/TXT packaging work regardless of encoding backend. -This reuses 100% of the existing karaoke_gen functionality. +2. LEGACY PATH (USE_NEW_ORCHESTRATOR=false): + Uses KaraokeFinalise from the karaoke_gen package. This path has divergent + behavior between GCE and local encoding - GCE encoding bypasses many features. + +The orchestrator-based approach is preferred as it: +- Eliminates GCE vs local code path divergence +- Ensures all features work consistently +- Provides better testability and maintainability Observability: - All operations wrapped in tracing spans for Cloud Trace visibility @@ -48,6 +49,12 @@ from karaoke_gen.karaoke_finalise.karaoke_finalise import KaraokeFinalise logger = logging.getLogger(__name__) +# Feature flag for new orchestrator-based pipeline +# Set to True to use the new unified pipeline that works with any encoding backend +# Set to False to use the legacy KaraokeFinalise-based pipeline +USE_NEW_ORCHESTRATOR = os.environ.get("USE_NEW_ORCHESTRATOR", "true").lower() == "true" + + # Loggers to capture for video worker # Include the full module path to properly capture KaraokeFinalise logs VIDEO_WORKER_LOGGERS = [ @@ -172,16 +179,247 @@ async def _encode_via_gce( raise -async def generate_video(job_id: str) -> bool: +async def generate_video_orchestrated(job_id: str) -> bool: """ - Generate final karaoke videos using KaraokeFinalise. - - This sets up the directory structure and calls KaraokeFinalise.process() - which handles all encoding, packaging, and notifications. - + Generate final karaoke videos using the new VideoWorkerOrchestrator. + + This provides a unified pipeline that works with any encoding backend + (GCE or local), ensuring all features like YouTube upload, Discord + notifications, and CDG/TXT packaging work consistently. + Args: job_id: Job ID to process - + + Returns: + True if successful, False otherwise + """ + from backend.workers.video_worker_orchestrator import ( + VideoWorkerOrchestrator, + create_orchestrator_config_from_job, + ) + + start_time = time.time() + job_manager = JobManager() + storage = StorageService() + + # Create job logger for remote debugging + job_log = create_job_logger(job_id, "video") + + # Log with structured markers for easy Cloud Logging queries + logger.info(f"[job:{job_id}] WORKER_START worker=video orchestrator=true") + + # Set up log capture for KaraokeFinalise (still used by some services) + log_handler = setup_job_logging(job_id, "video", *VIDEO_WORKER_LOGGERS) + + job = job_manager.get_job(job_id) + if not job: + logger.error(f"[job:{job_id}] Job not found") + return False + + # Validate prerequisites + if not _validate_prerequisites(job): + logger.error(f"[job:{job_id}] Prerequisites not met for video generation") + return False + + # Create temporary working directory + temp_dir = tempfile.mkdtemp(prefix=f"karaoke_video_{job_id}_") + original_cwd = os.getcwd() + + # Set up rclone config if needed (for legacy Dropbox upload path) + rclone_service = None + if getattr(job, 'organised_dir_rclone_root', None): + rclone_service = get_rclone_service() + if rclone_service.setup_rclone_config(): + job_log.info("Rclone config loaded for Dropbox upload") + + # Load YouTube credentials if needed + youtube_credentials = None + if getattr(job, 'enable_youtube_upload', False): + youtube_service = get_youtube_service() + if youtube_service.is_configured: + youtube_credentials = youtube_service.get_credentials_dict() + job_log.info("YouTube credentials loaded for video upload") + else: + job_log.warning("YouTube credentials not available - upload will be skipped") + + try: + # Wrap entire worker in a tracing span + with job_span("video-worker-orchestrated", job_id, {"artist": job.artist, "title": job.title}) as root_span: + with job_logging_context(job_id): + job_log.info(f"Starting orchestrated video generation for {job.artist} - {job.title}") + logger.info(f"[job:{job_id}] Starting orchestrated video generation") + + # Transition to GENERATING_VIDEO state + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.GENERATING_VIDEO, + progress=70, + message="Preparing files for video generation" + ) + + # Download and set up files + base_name = f"{job.artist} - {job.title}" + with job_span("download-files", job_id): + job_log.info("Downloading files from GCS...") + await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log) + job_log.info("All files downloaded successfully") + + # Load style config for CDG styles + style_config = await load_style_config(job, storage, temp_dir) + cdg_styles = style_config.get_cdg_styles() + + # Change to working directory + os.chdir(temp_dir) + + # Create orchestrator config from job + config = create_orchestrator_config_from_job( + job=job, + temp_dir=temp_dir, + youtube_credentials=youtube_credentials, + cdg_styles=cdg_styles, + ) + + # Run the orchestrated pipeline + orchestrator = VideoWorkerOrchestrator( + config=config, + job_manager=job_manager, + storage=storage, + job_logger=job_log, + ) + + with job_span("orchestrator-run", job_id): + result = await orchestrator.run() + + if not result.success: + raise Exception(result.error_message or "Orchestrator failed") + + # Prepare distribution directory for native uploads + with job_span("distribution", job_id): + await _handle_native_distribution( + job_id=job_id, + job=job, + job_log=job_log, + job_manager=job_manager, + temp_dir=temp_dir, + result={ + 'brand_code': result.brand_code, + 'youtube_url': result.youtube_url, + 'final_video': result.final_video, + 'final_video_lossy': result.final_video_lossy, + 'final_video_720p': result.final_video_720p, + 'final_karaoke_cdg_zip': result.final_karaoke_cdg_zip, + 'dropbox_link': result.dropbox_link, + 'gdrive_files': result.gdrive_files, + }, + storage=storage, + ) + + # Upload generated files to GCS + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.PACKAGING, + progress=95, + message="Uploading final files" + ) + + with job_span("upload-results", job_id): + await _upload_results(job_id, job_manager, storage, temp_dir, { + 'final_video': result.final_video, + 'final_video_mkv': result.final_video_mkv, + 'final_video_lossy': result.final_video_lossy, + 'final_video_720p': result.final_video_720p, + 'final_karaoke_cdg_zip': result.final_karaoke_cdg_zip, + 'final_karaoke_txt_zip': result.final_karaoke_txt_zip, + }) + + # Mark job as complete + logger.info(f"[job:{job_id}] Video generation complete") + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.COMPLETE, + progress=100, + message="Karaoke generation complete!" + ) + + # Store result metadata in job + job_manager.update_job(job_id, { + 'state_data': { + **job.state_data, + 'brand_code': result.brand_code, + 'youtube_url': result.youtube_url, + 'dropbox_link': result.dropbox_link, + 'gdrive_files': result.gdrive_files, + } + }) + + duration = time.time() - start_time + root_span.set_attribute("duration_seconds", duration) + root_span.set_attribute("brand_code", result.brand_code or '') + logger.info(f"[job:{job_id}] WORKER_END worker=video orchestrator=true status=success duration={duration:.1f}s") + return True + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[job:{job_id}] WORKER_END worker=video orchestrator=true status=error duration={duration:.1f}s error={e}") + job_manager.mark_job_failed( + job_id=job_id, + error_message=f"Video generation failed: {str(e)}", + error_details={"stage": "video_generation", "error": str(e)} + ) + return False + + finally: + # Restore original working directory + os.chdir(original_cwd) + + # Remove log handler + for logger_name in VIDEO_WORKER_LOGGERS: + try: + logging.getLogger(logger_name).removeHandler(log_handler) + except Exception: + pass + + # Cleanup rclone config file + if rclone_service: + rclone_service.cleanup() + + # Cleanup temporary directory + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + logger.debug(f"Cleaned up temp directory: {temp_dir}") + + +async def generate_video(job_id: str) -> bool: + """ + Generate final karaoke videos. + + Routes to either the new orchestrator-based pipeline or the legacy + KaraokeFinalise-based pipeline based on USE_NEW_ORCHESTRATOR flag. + + Args: + job_id: Job ID to process + + Returns: + True if successful, False otherwise + """ + if USE_NEW_ORCHESTRATOR: + logger.info(f"[job:{job_id}] Using new orchestrator-based pipeline") + return await generate_video_orchestrated(job_id) + else: + logger.info(f"[job:{job_id}] Using legacy KaraokeFinalise pipeline") + return await generate_video_legacy(job_id) + + +async def generate_video_legacy(job_id: str) -> bool: + """ + Generate final karaoke videos using KaraokeFinalise (legacy path). + + This is the original implementation that has divergent behavior between + GCE and local encoding paths. Kept for rollback purposes. + + Args: + job_id: Job ID to process + Returns: True if successful, False otherwise """ diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py new file mode 100644 index 00000000..2f35654c --- /dev/null +++ b/backend/workers/video_worker_orchestrator.py @@ -0,0 +1,628 @@ +""" +Video Worker Orchestrator. + +Coordinates the video generation pipeline stages in a unified way, +regardless of whether encoding happens locally or on GCE. + +This resolves the code path divergence where GCE encoding bypassed +features like YouTube upload, Discord notifications, and CDG/TXT packaging. + +Pipeline stages: +1. Setup - Download files, prepare directories +2. Packaging - CDG/TXT generation (if enabled) +3. Encoding - GCE or Local via EncodingBackend interface +4. Organization - Brand code, folder structure +5. Distribution - YouTube, Dropbox, Google Drive uploads +6. Notification - Discord notifications +""" + +import logging +import os +import time +from dataclasses import dataclass, field +from typing import Optional, Dict, Any, List +from pathlib import Path + +from backend.models.job import JobStatus +from backend.services.job_manager import JobManager +from backend.services.storage_service import StorageService +from backend.services.tracing import job_span, add_span_event + +logger = logging.getLogger(__name__) + + +@dataclass +class OrchestratorConfig: + """Configuration for the video worker orchestrator.""" + job_id: str + artist: str + title: str + + # Input file paths (in temp_dir) + title_video_path: str + karaoke_video_path: str + instrumental_audio_path: str + end_video_path: Optional[str] = None + lrc_file_path: Optional[str] = None + title_jpg_path: Optional[str] = None + + # Output directory + output_dir: str = "" + + # Feature flags + enable_cdg: bool = False + enable_txt: bool = False + enable_youtube_upload: bool = False + + # Service configurations + brand_prefix: Optional[str] = None + discord_webhook_url: Optional[str] = None + youtube_credentials: Optional[Dict[str, Any]] = None + youtube_description_template: Optional[str] = None + cdg_styles: Optional[Dict[str, Any]] = None + + # Dropbox/GDrive configuration + dropbox_path: Optional[str] = None + gdrive_folder_id: Optional[str] = None + + # Keep existing brand code (for re-processing) + keep_brand_code: Optional[str] = None + + # Encoding backend preference + encoding_backend: str = "auto" # "auto", "local", "gce" + + # Additional options + dry_run: bool = False + non_interactive: bool = True + + +@dataclass +class OrchestratorResult: + """Result from the video worker orchestrator.""" + success: bool + error_message: Optional[str] = None + + # Generated files + final_video: Optional[str] = None # Lossless 4K MP4 + final_video_mkv: Optional[str] = None # Lossless 4K MKV + final_video_lossy: Optional[str] = None # Lossy 4K MP4 + final_video_720p: Optional[str] = None # Lossy 720p MP4 + final_karaoke_cdg_zip: Optional[str] = None + final_karaoke_txt_zip: Optional[str] = None + + # Organization + brand_code: Optional[str] = None + + # Distribution results + youtube_url: Optional[str] = None + dropbox_link: Optional[str] = None + gdrive_files: Optional[Dict[str, str]] = field(default_factory=dict) + + # Timing + encoding_time_seconds: Optional[float] = None + total_time_seconds: Optional[float] = None + + +class VideoWorkerOrchestrator: + """ + Orchestrates the video generation pipeline. + + This class coordinates all stages of video generation in a unified way, + ensuring that features like YouTube upload and Discord notifications + work regardless of whether GCE or local encoding is used. + """ + + def __init__( + self, + config: OrchestratorConfig, + job_manager: Optional[JobManager] = None, + storage: Optional[StorageService] = None, + job_logger: Optional[logging.Logger] = None, + ): + """ + Initialize the orchestrator. + + Args: + config: Orchestrator configuration + job_manager: Job manager for state updates (optional) + storage: Storage service for file downloads (optional) + job_logger: Job-specific logger (optional) + """ + self.config = config + self.job_manager = job_manager + self.storage = storage + self.job_log = job_logger or logging.getLogger(__name__) + + # Services (lazy-loaded) + self._encoding_backend = None + self._packaging_service = None + self._youtube_service = None + self._discord_service = None + + # State + self.result = OrchestratorResult(success=False) + + def _get_encoding_backend(self): + """Lazy-load the encoding backend.""" + if self._encoding_backend is None: + from backend.services.encoding_interface import get_encoding_backend + self._encoding_backend = get_encoding_backend( + backend_type=self.config.encoding_backend, + dry_run=self.config.dry_run, + logger=self.job_log, + ) + return self._encoding_backend + + def _get_packaging_service(self): + """Lazy-load the packaging service.""" + if self._packaging_service is None: + from backend.services.packaging_service import PackagingService + self._packaging_service = PackagingService( + cdg_styles=self.config.cdg_styles, + dry_run=self.config.dry_run, + non_interactive=self.config.non_interactive, + logger=self.job_log, + ) + return self._packaging_service + + def _get_youtube_service(self): + """Lazy-load the YouTube upload service.""" + if self._youtube_service is None: + from backend.services.youtube_upload_service import YouTubeUploadService + self._youtube_service = YouTubeUploadService( + credentials=self.config.youtube_credentials, + non_interactive=self.config.non_interactive, + server_side_mode=True, + dry_run=self.config.dry_run, + logger=self.job_log, + ) + return self._youtube_service + + def _get_discord_service(self): + """Lazy-load the Discord notification service.""" + if self._discord_service is None: + from backend.services.discord_service import DiscordNotificationService + self._discord_service = DiscordNotificationService( + webhook_url=self.config.discord_webhook_url, + dry_run=self.config.dry_run, + logger=self.job_log, + ) + return self._discord_service + + def _update_progress(self, status: JobStatus, progress: int, message: str): + """Update job progress if job_manager is available.""" + if self.job_manager: + self.job_manager.transition_to_state( + job_id=self.config.job_id, + new_status=status, + progress=progress, + message=message + ) + + async def run(self) -> OrchestratorResult: + """ + Run the full video generation pipeline. + + Returns: + OrchestratorResult with generated files and metadata + """ + start_time = time.time() + + try: + with job_span("orchestrator", self.config.job_id) as span: + self.job_log.info(f"Starting orchestrated video generation for {self.config.artist} - {self.config.title}") + + # Stage 1: Packaging (CDG/TXT) - runs BEFORE encoding + # This depends on LRC file, not on encoded videos + if self.config.enable_cdg or self.config.enable_txt: + await self._run_packaging() + + # Stage 2: Encoding + await self._run_encoding() + + # Stage 3: Organization (brand code) + await self._run_organization() + + # Stage 4: Distribution (YouTube, Dropbox, GDrive) + await self._run_distribution() + + # Stage 5: Notifications (Discord) + await self._run_notifications() + + self.result.success = True + self.result.total_time_seconds = time.time() - start_time + + self.job_log.info(f"Orchestrated video generation complete in {self.result.total_time_seconds:.1f}s") + + except Exception as e: + self.result.success = False + self.result.error_message = str(e) + self.result.total_time_seconds = time.time() - start_time + self.job_log.error(f"Orchestrated video generation failed: {e}") + logger.error(f"[job:{self.config.job_id}] Orchestration failed: {e}") + + return self.result + + async def _run_packaging(self): + """Run the packaging stage (CDG/TXT generation).""" + self.job_log.info("Starting packaging stage (CDG/TXT)") + + if not self.config.lrc_file_path or not os.path.isfile(self.config.lrc_file_path): + self.job_log.warning("No LRC file available, skipping CDG/TXT packaging") + return + + base_name = f"{self.config.artist} - {self.config.title}" + packaging_service = self._get_packaging_service() + + # Generate CDG package + if self.config.enable_cdg: + self.job_log.info("Generating CDG package") + try: + cdg_zip_path = os.path.join( + self.config.output_dir, + f"{base_name} (Final Karaoke CDG).zip" + ) + mp3_path = os.path.join( + self.config.output_dir, + f"{base_name} (Karaoke).mp3" + ) + cdg_path = os.path.join( + self.config.output_dir, + f"{base_name} (Karaoke).cdg" + ) + + zip_file, mp3_file, cdg_file = packaging_service.create_cdg_package( + lrc_file=self.config.lrc_file_path, + audio_file=self.config.instrumental_audio_path, + output_zip_path=cdg_zip_path, + artist=self.config.artist, + title=self.config.title, + output_mp3_path=mp3_path, + output_cdg_path=cdg_path, + ) + + self.result.final_karaoke_cdg_zip = zip_file + self.job_log.info(f"CDG package created: {zip_file}") + + except Exception as e: + self.job_log.error(f"CDG generation failed: {e}") + # Don't fail the pipeline, CDG is optional + + # Generate TXT package + if self.config.enable_txt: + self.job_log.info("Generating TXT package") + try: + # TXT package needs MP3 file (from CDG generation or create it) + mp3_path = os.path.join( + self.config.output_dir, + f"{base_name} (Karaoke).mp3" + ) + + if not os.path.isfile(mp3_path): + self.job_log.info("MP3 not found, CDG must be enabled first for TXT") + return + + txt_zip_path = os.path.join( + self.config.output_dir, + f"{base_name} (Final Karaoke TXT).zip" + ) + + zip_file, txt_file = packaging_service.create_txt_package( + lrc_file=self.config.lrc_file_path, + mp3_file=mp3_path, + output_zip_path=txt_zip_path, + ) + + self.result.final_karaoke_txt_zip = zip_file + self.job_log.info(f"TXT package created: {zip_file}") + + except Exception as e: + self.job_log.error(f"TXT generation failed: {e}") + # Don't fail the pipeline, TXT is optional + + async def _run_encoding(self): + """Run the encoding stage.""" + self.job_log.info("Starting encoding stage") + self._update_progress(JobStatus.ENCODING, 75, "Encoding videos") + + encoding_backend = self._get_encoding_backend() + self.job_log.info(f"Using encoding backend: {encoding_backend.name}") + + # Build encoding input + from backend.services.encoding_interface import EncodingInput + + encoding_input = EncodingInput( + title_video_path=self.config.title_video_path, + karaoke_video_path=self.config.karaoke_video_path, + instrumental_audio_path=self.config.instrumental_audio_path, + end_video_path=self.config.end_video_path, + artist=self.config.artist, + title=self.config.title, + brand_code=self.config.keep_brand_code, + output_dir=self.config.output_dir, + ) + + # Run encoding + with job_span("encoding", self.config.job_id) as span: + add_span_event("encoding_started", {"backend": encoding_backend.name}) + + output = await encoding_backend.encode(encoding_input) + + add_span_event("encoding_completed", { + "success": output.success, + "duration": output.encoding_time_seconds or 0 + }) + + if not output.success: + raise Exception(f"Encoding failed: {output.error_message}") + + # Store results + self.result.final_video = output.lossless_4k_mp4_path + self.result.final_video_mkv = output.lossless_mkv_path + self.result.final_video_lossy = output.lossy_4k_mp4_path + self.result.final_video_720p = output.lossy_720p_mp4_path + self.result.encoding_time_seconds = output.encoding_time_seconds + + self.job_log.info(f"Encoding complete ({encoding_backend.name}) in {output.encoding_time_seconds:.1f}s") + + async def _run_organization(self): + """Run the organization stage (brand code generation).""" + self.job_log.info("Starting organization stage") + + # Use existing brand code if provided + if self.config.keep_brand_code: + self.result.brand_code = self.config.keep_brand_code + self.job_log.info(f"Using preserved brand code: {self.result.brand_code}") + return + + # Generate brand code from Dropbox if configured + if self.config.dropbox_path and self.config.brand_prefix: + try: + from backend.services.dropbox_service import get_dropbox_service + + dropbox = get_dropbox_service() + if dropbox.is_configured: + brand_code = dropbox.get_next_brand_code( + self.config.dropbox_path, + self.config.brand_prefix + ) + self.result.brand_code = brand_code + self.job_log.info(f"Generated brand code: {brand_code}") + else: + self.job_log.warning("Dropbox not configured, skipping brand code generation") + + except Exception as e: + self.job_log.error(f"Brand code generation failed: {e}") + # Don't fail - brand code is optional + + async def _run_distribution(self): + """Run the distribution stage (YouTube, Dropbox, GDrive uploads).""" + self.job_log.info("Starting distribution stage") + self._update_progress(JobStatus.PACKAGING, 90, "Uploading files") + + # YouTube upload + if self.config.enable_youtube_upload and self.config.youtube_credentials: + await self._upload_to_youtube() + + # Dropbox upload + if self.config.dropbox_path and self.config.brand_prefix: + await self._upload_to_dropbox() + + # Google Drive upload + if self.config.gdrive_folder_id: + await self._upload_to_gdrive() + + async def _upload_to_youtube(self): + """Upload video to YouTube.""" + self.job_log.info("Uploading to YouTube") + + # Find the best video file to upload (prefer MKV for FLAC audio, then lossless MP4) + video_to_upload = None + if self.result.final_video_mkv and os.path.isfile(self.result.final_video_mkv): + video_to_upload = self.result.final_video_mkv + elif self.result.final_video and os.path.isfile(self.result.final_video): + video_to_upload = self.result.final_video + elif self.result.final_video_lossy and os.path.isfile(self.result.final_video_lossy): + video_to_upload = self.result.final_video_lossy + + if not video_to_upload: + self.job_log.warning("No video file available for YouTube upload") + return + + try: + youtube_service = self._get_youtube_service() + + # Build video title + title = f"{self.config.artist} - {self.config.title} (Karaoke)" + + # Build description + description = self.config.youtube_description_template or "" + if self.result.brand_code: + description = f"{description}\n\nBrand Code: {self.result.brand_code}".strip() + + # Upload + video_id, video_url = youtube_service.upload_video( + video_path=video_to_upload, + title=title, + description=description, + thumbnail_path=self.config.title_jpg_path, + tags=["karaoke", self.config.artist, self.config.title], + replace_existing=True, # Server-side always replaces + ) + + if video_url: + self.result.youtube_url = video_url + self.job_log.info(f"Uploaded to YouTube: {video_url}") + else: + self.job_log.warning("YouTube upload did not return a URL") + + except Exception as e: + self.job_log.error(f"YouTube upload failed: {e}") + # Don't fail the pipeline - YouTube is optional + + async def _upload_to_dropbox(self): + """Upload files to Dropbox.""" + self.job_log.info("Uploading to Dropbox") + + try: + from backend.services.dropbox_service import get_dropbox_service + + dropbox = get_dropbox_service() + if not dropbox.is_configured: + self.job_log.warning("Dropbox not configured, skipping upload") + return + + base_name = f"{self.config.artist} - {self.config.title}" + folder_name = f"{self.result.brand_code or 'TRACK-0000'} - {base_name}" + remote_folder = f"{self.config.dropbox_path}/{folder_name}" + + # Upload entire output directory + dropbox.upload_folder(self.config.output_dir, remote_folder) + + # Create sharing link + try: + sharing_link = dropbox.create_shared_link(remote_folder) + self.result.dropbox_link = sharing_link + self.job_log.info(f"Dropbox sharing link: {sharing_link}") + except Exception as e: + self.job_log.warning(f"Failed to create Dropbox sharing link: {e}") + + self.job_log.info("Dropbox upload complete") + + except Exception as e: + self.job_log.error(f"Dropbox upload failed: {e}") + # Don't fail the pipeline - Dropbox is optional + + async def _upload_to_gdrive(self): + """Upload files to Google Drive.""" + self.job_log.info("Uploading to Google Drive") + + try: + from backend.services.gdrive_service import get_gdrive_service + + gdrive = get_gdrive_service() + if not gdrive.is_configured: + self.job_log.warning("Google Drive not configured, skipping upload") + return + + base_name = f"{self.config.artist} - {self.config.title}" + brand_code = self.result.brand_code or f"{self.config.brand_prefix or 'TRACK'}-0000" + + # Map result files to expected keys + output_files = { + 'final_karaoke_lossy_mp4': self.result.final_video_lossy, + 'final_karaoke_lossy_720p_mp4': self.result.final_video_720p, + 'final_karaoke_cdg_zip': self.result.final_karaoke_cdg_zip, + } + + uploaded = gdrive.upload_to_public_share( + root_folder_id=self.config.gdrive_folder_id, + brand_code=brand_code, + base_name=base_name, + output_files=output_files, + ) + + self.result.gdrive_files = uploaded + self.job_log.info(f"Google Drive upload complete: {len(uploaded)} files") + + except Exception as e: + self.job_log.error(f"Google Drive upload failed: {e}") + # Don't fail the pipeline - GDrive is optional + + async def _run_notifications(self): + """Run the notifications stage (Discord).""" + self.job_log.info("Starting notifications stage") + + if not self.config.discord_webhook_url: + self.job_log.debug("No Discord webhook configured, skipping notification") + return + + if not self.result.youtube_url: + self.job_log.info("No YouTube URL available, skipping Discord notification") + return + + try: + discord_service = self._get_discord_service() + discord_service.post_video_notification(self.result.youtube_url) + self.job_log.info("Discord notification sent") + except Exception as e: + self.job_log.error(f"Discord notification failed: {e}") + # Don't fail the pipeline - notifications are optional + + +def create_orchestrator_config_from_job( + job, + temp_dir: str, + youtube_credentials: Optional[Dict[str, Any]] = None, + cdg_styles: Optional[Dict[str, Any]] = None, +) -> OrchestratorConfig: + """ + Create an OrchestratorConfig from a job object. + + This is a helper function to bridge the existing job structure + with the new orchestrator configuration. + + Args: + job: Job object from Firestore + temp_dir: Temporary directory with downloaded files + youtube_credentials: Pre-loaded YouTube credentials + cdg_styles: CDG style configuration + + Returns: + OrchestratorConfig for the orchestrator + """ + base_name = f"{job.artist} - {job.title}" + + # Determine instrumental file path + instrumental_selection = job.state_data.get('instrumental_selection', 'clean') + existing_instrumental = getattr(job, 'existing_instrumental_gcs_path', None) + + if existing_instrumental: + ext = Path(existing_instrumental).suffix.lower() + instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental User){ext}") + else: + instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing" + instrumental_path = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac") + + return OrchestratorConfig( + job_id=job.job_id, + artist=job.artist, + title=job.title, + + # Input files + title_video_path=os.path.join(temp_dir, f"{base_name} (Title).mov"), + karaoke_video_path=os.path.join(temp_dir, f"{base_name} (With Vocals).mov"), + instrumental_audio_path=instrumental_path, + end_video_path=os.path.join(temp_dir, f"{base_name} (End).mov"), + lrc_file_path=os.path.join(temp_dir, f"{base_name} (Karaoke).lrc"), + title_jpg_path=os.path.join(temp_dir, f"{base_name} (Title).jpg"), + + # Output directory + output_dir=temp_dir, + + # Feature flags + enable_cdg=getattr(job, 'enable_cdg', False), + enable_txt=getattr(job, 'enable_txt', False), + enable_youtube_upload=getattr(job, 'enable_youtube_upload', False), + + # Service configurations + brand_prefix=getattr(job, 'brand_prefix', None), + discord_webhook_url=getattr(job, 'discord_webhook_url', None), + youtube_credentials=youtube_credentials, + youtube_description_template=getattr(job, 'youtube_description_template', None), + cdg_styles=cdg_styles, + + # Dropbox/GDrive + dropbox_path=getattr(job, 'dropbox_path', None), + gdrive_folder_id=getattr(job, 'gdrive_folder_id', None), + + # Keep existing brand code + keep_brand_code=getattr(job, 'keep_brand_code', None), + + # Encoding backend - auto selects GCE if available + encoding_backend="auto", + + # Server-side defaults + dry_run=False, + non_interactive=True, + ) diff --git a/pyproject.toml b/pyproject.toml index bfc103e4..42061530 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.88.0" +version = "0.89.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"