diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yml similarity index 50% rename from .github/workflows/test.yml rename to .github/workflows/ci.yml index 1af849ef..f4c7eaef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/ci.yml @@ -1,35 +1,167 @@ -name: Backend Tests +name: CI on: - push: # Run on all branch pushes + push: pull_request: - branches: [ main ] + branches: [main] + workflow_dispatch: + +permissions: + contents: write + pages: write + id-token: write + +concurrency: + group: "ci-${{ github.ref }}" + cancel-in-progress: true jobs: - unit-tests: - name: Unit Tests + # ============================================ + # PACKAGE TESTS (karaoke-gen Python CLI) + # ============================================ + + package-unit-tests: + name: "Package - Unit Tests (70% coverage)" runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: false - + - uses: actions/checkout@v4 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: '3.13' - + python-version: "3.13" + - name: Install Poetry uses: snok/install-poetry@v1 with: virtualenvs-create: true virtualenvs-in-project: true - + + - name: Install dependencies + run: poetry install --with dev + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + + - name: Install spaCy language models + run: poetry run python -m spacy download en_core_web_sm + + - name: Run unit tests + run: | + poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70 + + - name: Upload coverage reports + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: unittests + name: codecov-umbrella + + package-integration-tests: + name: "Package - Integration Tests" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Install dependencies + run: poetry install --with dev + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + + - name: Install spaCy language models + run: poetry run python -m spacy download en_core_web_sm + + - name: Run integration tests + run: poetry run pytest tests/integration/ -v + + package-build: + name: "Package - Build & Install" + runs-on: ubuntu-latest + needs: [package-unit-tests, package-integration-tests] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Build package + run: poetry build + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + haskell: true + dotnet: false + large-packages: false + docker-images: false + swap-storage: false + + - name: Test package installation + run: | + pip install dist/*.whl + python -m karaoke_gen.utils.gen_cli --help + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: package-dist + path: dist/ + retention-days: 1 + + # ============================================ + # BACKEND TESTS (Cloud Run API service) + # ============================================ + + backend-unit-tests: + name: "Backend - Unit Tests" + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: false + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + - name: Install dependencies run: poetry install - + - name: Run unit tests env: GOOGLE_CLOUD_PROJECT: test-project @@ -43,11 +175,10 @@ jobs: backend/tests/test_file_upload.py \ backend/tests/test_services.py \ -v --tb=short - - emulator-tests: - name: Emulator Integration Tests + + backend-emulator-tests: + name: "Backend - Emulator Integration Tests" runs-on: ubuntu-latest - steps: - name: Free disk space run: | @@ -55,144 +186,250 @@ jobs: sudo rm -rf /opt/ghc sudo rm -rf /usr/local/share/boost df -h - + - name: Checkout code uses: actions/checkout@v4 with: submodules: false - + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: '3.13' - + python-version: "3.13" + - name: Install Poetry uses: snok/install-poetry@v1 with: virtualenvs-create: true virtualenvs-in-project: true - + - name: Set up Java 21 (required by Firestore emulator) uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '21' - + distribution: "temurin" + java-version: "21" + - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2 with: - version: 'latest' - + version: "latest" + - name: Install Firestore emulator - run: | - gcloud components install beta cloud-firestore-emulator --quiet - + run: gcloud components install beta cloud-firestore-emulator --quiet + - name: Install dependencies run: poetry install - + - name: Start Docker for GCS emulator - run: | - docker pull fsouza/fake-gcs-server:latest - + run: docker pull fsouza/fake-gcs-server:latest + - name: Run emulator integration tests run: | chmod +x scripts/*.sh ./scripts/run-emulator-tests.sh - + - name: Upload test results if: always() uses: actions/upload-artifact@v4 with: - name: test-results + name: backend-test-results path: backend/htmlcov/ retention-days: 7 - lint: - name: Code Quality + backend-lint: + name: "Backend - Code Quality" runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 with: submodules: false - + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: '3.13' - + python-version: "3.13" + - name: Install Poetry uses: snok/install-poetry@v1 with: virtualenvs-create: true virtualenvs-in-project: true - + - name: Install dependencies run: poetry install - + - name: Run validation (syntax check only) run: | - # Skip full validation as it requires GCP credentials - # Just check Python syntax and imports poetry run python -m py_compile backend/main.py backend/config.py echo "✅ Basic syntax validation passed" - - deploy: - name: Deploy via Cloud Build + + # ============================================ + # DEPLOYMENTS (main branch only) + # ============================================ + + package-publish: + name: "Deploy - Publish to PyPI" runs-on: ubuntu-latest - needs: [unit-tests, emulator-tests, lint] - # Deploy on any push where tests pass (pre-alpha: rapid iteration mode) - # TODO: Change back to main-only when ready for production: github.ref == 'refs/heads/main' - if: github.event_name == 'push' - + needs: [package-build] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: package-dist + path: dist/ + + - name: Extract version + id: extract_version + run: | + VERSION=$(poetry version --short) + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "📋 Extracted version: $VERSION" + + - name: Check if tag exists + id: check_tag + run: | + VERSION="${{ steps.extract_version.outputs.version }}" + if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then + echo "tag_exists=true" >> $GITHUB_OUTPUT + echo "🏷️ Tag v$VERSION already exists, skipping release" + else + echo "tag_exists=false" >> $GITHUB_OUTPUT + echo "✨ Tag v$VERSION does not exist, proceeding with release" + fi + + - name: Publish to PyPI + if: steps.check_tag.outputs.tag_exists == 'false' + run: | + echo "📦 Publishing the following files to PyPI:" + ls -la dist/ + echo "🚀 Publishing to PyPI..." + poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} + poetry publish + + - name: Create Git Tag + if: steps.check_tag.outputs.tag_exists == 'false' + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}" + git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}" + + - name: Create GitHub Release + if: steps.check_tag.outputs.tag_exists == 'false' + uses: softprops/action-gh-release@v1 + with: + tag_name: "v${{ steps.extract_version.outputs.version }}" + name: "v${{ steps.extract_version.outputs.version }}" + body: | + ## Changes in v${{ steps.extract_version.outputs.version }} + + This release was automatically created from the main branch. + + ### Installation + ```bash + pip install karaoke-gen==${{ steps.extract_version.outputs.version }} + ``` + + ### Assets + - Python package published to PyPI + - Frontend deployed to GitHub Pages at gen.nomadkaraoke.com + + For detailed changes, see the commit history. + draft: false + prerelease: false + files: | + dist/*.whl + dist/*.tar.gz + + deploy-frontend: + name: "Deploy - Frontend (GitHub Pages)" + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Prepare frontend files + run: | + echo "📁 Preparing frontend files for deployment..." + mkdir -p _site + cp -r frontend/* _site/ + echo "✅ Frontend files copied to _site/" + ls -la _site/ + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: "_site" + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + deploy-backend: + name: "Deploy - Backend (Cloud Run)" + runs-on: ubuntu-latest + needs: [backend-unit-tests, backend-emulator-tests, backend-lint] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' permissions: contents: read - id-token: write # Required for Workload Identity Federation - + id-token: write steps: - name: Checkout code uses: actions/checkout@v4 with: submodules: false - + - name: Authenticate to Google Cloud (Workload Identity) id: auth uses: google-github-actions/auth@v2 with: workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} - + - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2 with: - version: 'latest' - + version: "latest" + - name: Check if base image needs rebuild id: check-base run: | echo "🔍 Checking if base image needs rebuild..." - - # Calculate current poetry.lock hash + CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1) echo "Current poetry.lock hash: $CURRENT_HASH" - - # Try to get the hash from the existing base image - # First, authenticate to Artifact Registry + gcloud auth configure-docker us-central1-docker.pkg.dev --quiet - - # Try to pull the base image and get its label + BASE_IMAGE="us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest" EXISTING_HASH="" - + if docker pull $BASE_IMAGE 2>/dev/null; then EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "") echo "Existing base image hash: $EXISTING_HASH" else echo "⚠️ No existing base image found" fi - - # Compare hashes + if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then echo "🔄 Base image needs rebuild (poetry.lock changed)" echo "needs_rebuild=true" >> $GITHUB_OUTPUT @@ -200,29 +437,27 @@ jobs: echo "✅ Base image is up to date" echo "needs_rebuild=false" >> $GITHUB_OUTPUT fi - + - name: Rebuild base image (if needed) if: steps.check-base.outputs.needs_rebuild == 'true' run: | echo "🏗️ Rebuilding base image with updated dependencies..." - - # Submit base image build and wait for completion + BASE_BUILD_ID=$(gcloud builds submit \ --config=cloudbuild-base.yaml \ --project=nomadkaraoke \ --format='value(id)' \ --async \ .) - + echo "Base Build ID: $BASE_BUILD_ID" echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BASE_BUILD_ID?project=nomadkaraoke" - - # Poll for build completion + echo "Waiting for base image build to complete..." while true; do BUILD_STATUS=$(gcloud builds describe $BASE_BUILD_ID --project=nomadkaraoke --format='value(status)') echo "Base build status: $BUILD_STATUS" - + if [ "$BUILD_STATUS" = "SUCCESS" ]; then echo "✅ Base image build completed successfully" break @@ -239,32 +474,30 @@ jobs: gcloud builds log $BASE_BUILD_ID --project=nomadkaraoke 2>&1 || true exit 1 fi - + sleep 30 done - + - name: Trigger Cloud Build run: | echo "🚀 Triggering Cloud Build for deployment..." echo "Commit: ${{ github.sha }}" - - # Submit build and wait for completion + BUILD_ID=$(gcloud builds submit \ --config=cloudbuild.yaml \ --project=nomadkaraoke \ --format='value(id)' \ --async \ .) - + echo "Build ID: $BUILD_ID" echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke" - - # Poll for build completion + echo "Waiting for build to complete..." while true; do BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)') echo "Build status: $BUILD_STATUS" - + if [ "$BUILD_STATUS" = "SUCCESS" ]; then echo "✅ Build completed successfully" break @@ -272,7 +505,6 @@ jobs: echo "❌ Build failed with status: $BUILD_STATUS" echo "" echo "==================== BUILD LOGS ====================" - # Use Cloud Logging to get build logs (works with roles/logging.viewer) gcloud logging read "resource.type=build AND resource.labels.build_id=$BUILD_ID" \ --project=nomadkaraoke \ --format="value(textPayload)" \ @@ -280,30 +512,24 @@ jobs: --order=asc || echo "Failed to retrieve logs from Cloud Logging" echo "====================================================" echo "" - # Also try the direct logs command as fallback gcloud builds log $BUILD_ID --project=nomadkaraoke 2>&1 || true exit 1 fi - + sleep 30 done - + - name: Verify deployment run: | SERVICE_URL=$(gcloud run services describe karaoke-backend \ --region us-central1 \ --format 'value(status.url)') - + echo "Service URL: $SERVICE_URL" - - # Wait a few seconds for the service to be ready sleep 10 - - # Test health endpoint curl -f "$SERVICE_URL/api/health" || exit 1 - echo "✅ Deployment verified - service is healthy" - + - name: Summary if: success() run: | diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml deleted file mode 100644 index a52ad237..00000000 --- a/.github/workflows/test-and-publish.yml +++ /dev/null @@ -1,187 +0,0 @@ -name: Test and Publish - -on: [push, pull_request, workflow_dispatch] - -# Add permissions for GitHub Pages deployment and git operations -permissions: - contents: write # Changed from 'read' to 'write' to allow creating git tags - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.13'] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Install dependencies - run: | - poetry install --with dev - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y ffmpeg - - - name: Install spaCy language models - run: | - poetry run python -m spacy download en_core_web_sm - - - name: Run unit tests - run: | - poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70 - - - name: Upload coverage reports - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - flags: unittests - name: codecov-umbrella - - - name: Run integration tests - run: | - poetry run pytest tests/integration/ -v - - - name: Build package - run: | - poetry build - - - name: Free Disk Space (GHA Ubuntu Runner) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - haskell: true - dotnet: false - large-packages: false - docker-images: false - swap-storage: false - - - name: Test package installation - run: | - pip install dist/*.whl - python -m karaoke_gen.utils.gen_cli --help - - - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - name: Extract version - id: extract_version - run: | - VERSION=$(poetry version --short) - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "📋 Extracted version: $VERSION" - - - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - name: Check if tag exists - id: check_tag - run: | - VERSION="${{ steps.extract_version.outputs.version }}" - if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then - echo "tag_exists=true" >> $GITHUB_OUTPUT - echo "🏷️ Tag v$VERSION already exists, skipping release" - else - echo "tag_exists=false" >> $GITHUB_OUTPUT - echo "✨ Tag v$VERSION does not exist, proceeding with release" - fi - - - if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.check_tag.outputs.tag_exists == 'false' - name: Publish to PyPI - run: | - echo "📦 Publishing the following files to PyPI:" - ls -la dist/ - echo "🚀 Publishing to PyPI..." - poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} - poetry publish - - - if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.check_tag.outputs.tag_exists == 'false' - name: Create Git Tag - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}" - git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}" - - - if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.check_tag.outputs.tag_exists == 'false' - name: Create GitHub Release - uses: softprops/action-gh-release@v1 - with: - tag_name: "v${{ steps.extract_version.outputs.version }}" - name: "v${{ steps.extract_version.outputs.version }}" - body: | - ## Changes in v${{ steps.extract_version.outputs.version }} - - This release was automatically created from the main branch. - - ### Installation - ```bash - pip install karaoke-gen==${{ steps.extract_version.outputs.version }} - ``` - - ### Assets - - Python package published to PyPI - - Frontend deployed to GitHub Pages at gen.nomadkaraoke.com - - For detailed changes, see the commit history. - draft: false - prerelease: false - files: | - dist/*.whl - dist/*.tar.gz - - # Deploy frontend to GitHub Pages - deploy-frontend: - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Pages - uses: actions/configure-pages@v4 - - - name: Prepare frontend files - run: | - echo "📁 Preparing frontend files for deployment..." - mkdir -p _site - cp -r frontend/* _site/ - echo "✅ Frontend files copied to _site/" - ls -la _site/ - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: '_site' - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 - \ No newline at end of file diff --git a/backend/api/routes/file_upload.py b/backend/api/routes/file_upload.py index 71586941..b98b0d3f 100644 --- a/backend/api/routes/file_upload.py +++ b/backend/api/routes/file_upload.py @@ -1057,12 +1057,23 @@ async def mark_uploads_complete( # URL-based Job Creation - For YouTube and other online video URLs # ============================================================================ -def _validate_url(url: str) -> bool: +def _validate_url(url: Optional[str]) -> bool: """ Validate that a URL is a supported video/audio URL. Supports YouTube, Vimeo, SoundCloud, and other platforms supported by yt-dlp. + + Args: + url: The URL to validate. Can be None or non-string. + + Returns: + True if valid URL, False otherwise. """ + # Handle None and non-string inputs safely + if url is None or not isinstance(url, str): + return False + + # Handle empty string if not url: return False diff --git a/backend/workers/audio_worker.py b/backend/workers/audio_worker.py index 020b060f..ebe3881e 100644 --- a/backend/workers/audio_worker.py +++ b/backend/workers/audio_worker.py @@ -19,7 +19,7 @@ from backend.models.job import JobStatus from backend.services.job_manager import JobManager from backend.services.storage_service import StorageService from backend.config import get_settings -from backend.workers.worker_logging import create_job_logger, setup_job_logging +from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context # Import from karaoke_gen package from karaoke_gen.audio_processor import AudioProcessor @@ -241,84 +241,88 @@ async def process_audio_separation(job_id: str) -> bool: job_log.info(f"Created temp directory: {temp_dir}") try: - job_log.info(f"Starting audio separation for {job.artist} - {job.title}") - logger.info(f"Starting audio separation for job {job_id}") - - # Ensure AUDIO_SEPARATOR_API_URL is set - api_url = os.environ.get("AUDIO_SEPARATOR_API_URL") - if not api_url: - raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. " - "Cannot perform audio separation without remote API access.") - job_log.info(f"Audio separator API: {api_url}") - - # Download audio file from GCS or URL - job_log.info("Downloading audio file...") - audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager) - if not audio_path: - raise Exception("Failed to download audio file") - job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}") - - # Update progress using state_data (don't change status during parallel processing) - # The status is managed at a higher level - workers just track their progress - job_manager.update_state_data(job_id, 'audio_progress', { - 'stage': 'separating_stage1', - 'progress': 10, - 'message': 'Starting audio separation (Stage 1: Clean instrumental)' - }) - - # Create AudioProcessor instance (reuses karaoke_gen code) - # Use model configuration from job if provided, otherwise use defaults - job_log.info("Creating AudioProcessor instance...") - if job.clean_instrumental_model: - job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}") - if job.backing_vocals_models: - job_log.info(f" Using backing vocals models: {job.backing_vocals_models}") - if job.other_stems_models: - job_log.info(f" Using other stems models: {job.other_stems_models}") - - audio_processor = create_audio_processor( - temp_dir, - clean_instrumental_model=job.clean_instrumental_model, - backing_vocals_models=job.backing_vocals_models, - other_stems_models=job.other_stems_models - ) - - # Format artist-title for file naming (matches CLI behavior) - artist_title = f"{job.artist} - {job.title}" - - # Run audio separation (calls Modal API internally) - # This returns a dict with paths to all separated stems - job_log.info("Starting audio separation (this may take 5-10 minutes)...") - job_log.info(" Stage 1: Clean instrumental separation (MDX models)") - job_log.info(" Stage 2: Backing vocals separation (Demucs model)") - logger.info(f"Job {job_id}: Calling audio_processor.process_audio_separation()") - separation_result = audio_processor.process_audio_separation( - audio_file=audio_path, - artist_title=artist_title, - track_output_dir=temp_dir - ) - - job_log.info("Audio separation complete!") - job_log.info(f" Generated {len(separation_result)} stem files") - logger.info(f"Job {job_id}: Audio separation complete, organizing results") - - # Update progress using state_data (don't change status during parallel processing) - job_manager.update_state_data(job_id, 'audio_progress', { - 'stage': 'audio_complete', - 'progress': 45, - 'message': 'Audio separation complete, uploading stems' - }) - - # Upload all stems to GCS - await upload_separation_results(job_id, separation_result, storage, job_manager) - - logger.info(f"Job {job_id}: All stems uploaded successfully") - - # Mark audio processing complete - # This will check if lyrics are also complete and transition to next stage if so - job_manager.mark_audio_complete(job_id) - - return True + # Use job_logging_context for proper log isolation when multiple jobs run concurrently + # This ensures logs from third-party libraries (karaoke_gen.audio_processor) are + # only captured by this job's handler, not handlers from other concurrent jobs + with job_logging_context(job_id): + job_log.info(f"Starting audio separation for {job.artist} - {job.title}") + logger.info(f"Starting audio separation for job {job_id}") + + # Ensure AUDIO_SEPARATOR_API_URL is set + api_url = os.environ.get("AUDIO_SEPARATOR_API_URL") + if not api_url: + raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. " + "Cannot perform audio separation without remote API access.") + job_log.info(f"Audio separator API: {api_url}") + + # Download audio file from GCS or URL + job_log.info("Downloading audio file...") + audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager) + if not audio_path: + raise Exception("Failed to download audio file") + job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}") + + # Update progress using state_data (don't change status during parallel processing) + # The status is managed at a higher level - workers just track their progress + job_manager.update_state_data(job_id, 'audio_progress', { + 'stage': 'separating_stage1', + 'progress': 10, + 'message': 'Starting audio separation (Stage 1: Clean instrumental)' + }) + + # Create AudioProcessor instance (reuses karaoke_gen code) + # Use model configuration from job if provided, otherwise use defaults + job_log.info("Creating AudioProcessor instance...") + if job.clean_instrumental_model: + job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}") + if job.backing_vocals_models: + job_log.info(f" Using backing vocals models: {job.backing_vocals_models}") + if job.other_stems_models: + job_log.info(f" Using other stems models: {job.other_stems_models}") + + audio_processor = create_audio_processor( + temp_dir, + clean_instrumental_model=job.clean_instrumental_model, + backing_vocals_models=job.backing_vocals_models, + other_stems_models=job.other_stems_models + ) + + # Format artist-title for file naming (matches CLI behavior) + artist_title = f"{job.artist} - {job.title}" + + # Run audio separation (calls Modal API internally) + # This returns a dict with paths to all separated stems + job_log.info("Starting audio separation (this may take 5-10 minutes)...") + job_log.info(" Stage 1: Clean instrumental separation (MDX models)") + job_log.info(" Stage 2: Backing vocals separation (Demucs model)") + logger.info(f"Job {job_id}: Calling audio_processor.process_audio_separation()") + separation_result = audio_processor.process_audio_separation( + audio_file=audio_path, + artist_title=artist_title, + track_output_dir=temp_dir + ) + + job_log.info("Audio separation complete!") + job_log.info(f" Generated {len(separation_result)} stem files") + logger.info(f"Job {job_id}: Audio separation complete, organizing results") + + # Update progress using state_data (don't change status during parallel processing) + job_manager.update_state_data(job_id, 'audio_progress', { + 'stage': 'audio_complete', + 'progress': 45, + 'message': 'Audio separation complete, uploading stems' + }) + + # Upload all stems to GCS + await upload_separation_results(job_id, separation_result, storage, job_manager) + + logger.info(f"Job {job_id}: All stems uploaded successfully") + + # Mark audio processing complete + # This will check if lyrics are also complete and transition to next stage if so + job_manager.mark_audio_complete(job_id) + + return True except Exception as e: logger.error(f"Job {job_id}: Audio separation failed: {e}", exc_info=True) diff --git a/backend/workers/lyrics_worker.py b/backend/workers/lyrics_worker.py index 6e043ba9..f76866b9 100644 --- a/backend/workers/lyrics_worker.py +++ b/backend/workers/lyrics_worker.py @@ -26,7 +26,7 @@ from backend.models.job import JobStatus from karaoke_gen.utils import sanitize_filename from backend.services.job_manager import JobManager from backend.services.storage_service import StorageService -from backend.workers.worker_logging import create_job_logger, setup_job_logging +from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context from backend.workers.style_helper import load_style_config # Import from karaoke_gen package @@ -127,158 +127,162 @@ async def process_lyrics_transcription(job_id: str) -> bool: temp_dir = None try: - job = job_manager.get_job(job_id) - if not job: - logger.error(f"Job {job_id} not found") - job_log.error(f"Job {job_id} not found in Firestore!") - return False - - # Create temporary working directory - temp_dir = tempfile.mkdtemp(prefix=f"karaoke_lyrics_{job_id}_") - job_log.info(f"Created temp directory: {temp_dir}") - job_log.info(f"Starting lyrics transcription for {job.artist} - {job.title}") - job_log.info(f"Log capture enabled for: {', '.join(LYRICS_WORKER_LOGGERS)}") - logger.info(f"Starting lyrics transcription for job {job_id}") - - # Log environment configuration - job_log.info("Checking API configuration...") - if os.environ.get("GENIUS_API_TOKEN"): - job_log.info(" Genius API: configured") - else: - job_log.warning(" Genius API: NOT configured") - if os.environ.get("SPOTIFY_COOKIE_SP_DC"): - job_log.info(" Spotify: configured") - else: - job_log.warning(" Spotify: NOT configured") - if os.environ.get("RAPIDAPI_KEY"): - job_log.info(" Musixmatch (RapidAPI): configured") - else: - job_log.warning(" Musixmatch (RapidAPI): NOT configured") - - # Ensure required environment variables are set - if not os.environ.get("AUDIOSHAKE_API_TOKEN"): - job_log.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail") - logger.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail") - - # Download audio file from GCS (waits for audio worker if URL job) - job_log.info("Downloading audio file from GCS...") - audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager) - if not audio_path: - raise Exception("Failed to download audio file") - job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}") - - # Update progress using state_data (don't change status during parallel processing) - # The status is managed at a higher level - workers just track their progress - job_manager.update_state_data(job_id, 'lyrics_progress', { - 'stage': 'transcribing', - 'progress': 10, - 'message': 'Starting lyrics transcription via AudioShake' - }) - - # Load style configuration (downloads assets from GCS if available) - # This is needed for max_line_length and other style settings in LyricsTranscriber - job_log.info("Loading style configuration for lyrics processing...") - style_config = await load_style_config(job, storage, temp_dir) - style_params_json_path = style_config.get_style_params_path() if style_config.has_custom_styles() else None - - if style_params_json_path: - job_log.info(f"Using custom style params: {style_params_json_path}") - # Log the contents of the style params for debugging - try: - with open(style_params_json_path, 'r') as f: - style_content = json.load(f) - job_log.info(f"Style params sections: {list(style_content.keys())}") - if 'karaoke' in style_content: - karaoke_style = style_content['karaoke'] - job_log.info(f" karaoke.background_image: {karaoke_style.get('background_image', 'NOT SET')}") - job_log.info(f" karaoke.font_path: {karaoke_style.get('font_path', 'NOT SET')}") - except Exception as e: - job_log.warning(f"Could not read style params for logging: {e}") - else: - job_log.info("No custom style params found, using defaults") - - # Download user-provided lyrics file if available - lyrics_file_path = None - if hasattr(job, 'lyrics_file_gcs_path') and job.lyrics_file_gcs_path: - lyrics_file_path = os.path.join(temp_dir, "user_lyrics.txt") - job_log.info(f"Downloading user-provided lyrics file: {job.lyrics_file_gcs_path}") - storage.download_file(job.lyrics_file_gcs_path, lyrics_file_path) - job_log.info(f"User lyrics file downloaded to: {lyrics_file_path}") - - # Get lyrics configuration from job - subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0 - if subtitle_offset != 0: - job_log.info(f"Subtitle offset: {subtitle_offset}ms") - - # Create LyricsProcessor instance (reuses karaoke_gen code) - job_log.info("Creating LyricsProcessor instance...") - job_log.info(f" style_params_json: {style_params_json_path}") - job_log.info(f" lyrics_file: {lyrics_file_path}") - job_log.info(f" subtitle_offset_ms: {subtitle_offset}") - lyrics_processor = create_lyrics_processor( - style_params_json=style_params_json_path, - lyrics_file=lyrics_file_path, - subtitle_offset_ms=subtitle_offset - ) - - # Run transcription + correction - # This will: - # 1. Fetch reference lyrics from Genius/Spotify/Musixmatch/LRCLib - # 2. Transcribe audio via AudioShake API - # 3. Run automatic correction with lyrics_transcriber - # 4. Generate corrections JSON (no interactive review) - # 5. Skip video generation (render_video=False) - - # Use lyrics_artist/lyrics_title overrides if provided, else fall back to job artist/title - lyrics_search_artist = getattr(job, 'lyrics_artist', None) or job.artist - lyrics_search_title = getattr(job, 'lyrics_title', None) or job.title - - job_log.info("Starting LyricsTranscriber processing...") - job_log.info(f" Artist: {job.artist}") - job_log.info(f" Title: {job.title}") - if lyrics_search_artist != job.artist: - job_log.info(f" Lyrics search artist override: {lyrics_search_artist}") - if lyrics_search_title != job.title: - job_log.info(f" Lyrics search title override: {lyrics_search_title}") - logger.info(f"Job {job_id}: Calling lyrics_processor.transcribe_lyrics()") - - # Run transcription in thread pool to avoid blocking the event loop - # (transcribe_lyrics is synchronous and takes 1-2 minutes) - # Note: We pass the original artist/title for file naming, but the LyricsProcessor - # will use lyrics_artist/lyrics_title for the search (set via constructor args) - result = await asyncio.to_thread( - lyrics_processor.transcribe_lyrics, - input_audio_wav=audio_path, - artist=job.artist, # Original artist for file naming - title=job.title, # Original title for file naming - track_output_dir=temp_dir, - lyrics_artist=lyrics_search_artist, # Override for lyrics search - lyrics_title=lyrics_search_title # Override for lyrics search - ) - - job_log.info("Transcription processing complete") - logger.info(f"Job {job_id}: Transcription complete, uploading results") - - # Upload lyrics results to GCS - job_log.info("Uploading lyrics results to GCS...") - await upload_lyrics_results(job_id, temp_dir, result, storage, job_manager, job_log) - - job_log.info("All lyrics data uploaded successfully") - logger.info(f"Job {job_id}: All lyrics data uploaded successfully") - - # Update progress using state_data (don't change status during parallel processing) - job_manager.update_state_data(job_id, 'lyrics_progress', { - 'stage': 'lyrics_complete', - 'progress': 45, - 'message': 'Lyrics transcription complete' - }) - - # Mark lyrics processing complete - # This will check if audio is also complete and transition to next stage if so - job_log.info("Lyrics worker complete, checking if audio is also done...") - job_manager.mark_lyrics_complete(job_id) - - return True + # Use job_logging_context for proper log isolation when multiple jobs run concurrently + # This ensures logs from third-party libraries (lyrics_transcriber) are only captured + # by this job's handler, not handlers from other concurrent jobs + with job_logging_context(job_id): + job = job_manager.get_job(job_id) + if not job: + logger.error(f"Job {job_id} not found") + job_log.error(f"Job {job_id} not found in Firestore!") + return False + + # Create temporary working directory + temp_dir = tempfile.mkdtemp(prefix=f"karaoke_lyrics_{job_id}_") + job_log.info(f"Created temp directory: {temp_dir}") + job_log.info(f"Starting lyrics transcription for {job.artist} - {job.title}") + job_log.info(f"Log capture enabled for: {', '.join(LYRICS_WORKER_LOGGERS)}") + logger.info(f"Starting lyrics transcription for job {job_id}") + + # Log environment configuration + job_log.info("Checking API configuration...") + if os.environ.get("GENIUS_API_TOKEN"): + job_log.info(" Genius API: configured") + else: + job_log.warning(" Genius API: NOT configured") + if os.environ.get("SPOTIFY_COOKIE_SP_DC"): + job_log.info(" Spotify: configured") + else: + job_log.warning(" Spotify: NOT configured") + if os.environ.get("RAPIDAPI_KEY"): + job_log.info(" Musixmatch (RapidAPI): configured") + else: + job_log.warning(" Musixmatch (RapidAPI): NOT configured") + + # Ensure required environment variables are set + if not os.environ.get("AUDIOSHAKE_API_TOKEN"): + job_log.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail") + logger.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail") + + # Download audio file from GCS (waits for audio worker if URL job) + job_log.info("Downloading audio file from GCS...") + audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager) + if not audio_path: + raise Exception("Failed to download audio file") + job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}") + + # Update progress using state_data (don't change status during parallel processing) + # The status is managed at a higher level - workers just track their progress + job_manager.update_state_data(job_id, 'lyrics_progress', { + 'stage': 'transcribing', + 'progress': 10, + 'message': 'Starting lyrics transcription via AudioShake' + }) + + # Load style configuration (downloads assets from GCS if available) + # This is needed for max_line_length and other style settings in LyricsTranscriber + job_log.info("Loading style configuration for lyrics processing...") + style_config = await load_style_config(job, storage, temp_dir) + style_params_json_path = style_config.get_style_params_path() if style_config.has_custom_styles() else None + + if style_params_json_path: + job_log.info(f"Using custom style params: {style_params_json_path}") + # Log the contents of the style params for debugging + try: + with open(style_params_json_path, 'r') as f: + style_content = json.load(f) + job_log.info(f"Style params sections: {list(style_content.keys())}") + if 'karaoke' in style_content: + karaoke_style = style_content['karaoke'] + job_log.info(f" karaoke.background_image: {karaoke_style.get('background_image', 'NOT SET')}") + job_log.info(f" karaoke.font_path: {karaoke_style.get('font_path', 'NOT SET')}") + except Exception as e: + job_log.warning(f"Could not read style params for logging: {e}") + else: + job_log.info("No custom style params found, using defaults") + + # Download user-provided lyrics file if available + lyrics_file_path = None + if hasattr(job, 'lyrics_file_gcs_path') and job.lyrics_file_gcs_path: + lyrics_file_path = os.path.join(temp_dir, "user_lyrics.txt") + job_log.info(f"Downloading user-provided lyrics file: {job.lyrics_file_gcs_path}") + storage.download_file(job.lyrics_file_gcs_path, lyrics_file_path) + job_log.info(f"User lyrics file downloaded to: {lyrics_file_path}") + + # Get lyrics configuration from job + subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0 + if subtitle_offset != 0: + job_log.info(f"Subtitle offset: {subtitle_offset}ms") + + # Create LyricsProcessor instance (reuses karaoke_gen code) + job_log.info("Creating LyricsProcessor instance...") + job_log.info(f" style_params_json: {style_params_json_path}") + job_log.info(f" lyrics_file: {lyrics_file_path}") + job_log.info(f" subtitle_offset_ms: {subtitle_offset}") + lyrics_processor = create_lyrics_processor( + style_params_json=style_params_json_path, + lyrics_file=lyrics_file_path, + subtitle_offset_ms=subtitle_offset + ) + + # Run transcription + correction + # This will: + # 1. Fetch reference lyrics from Genius/Spotify/Musixmatch/LRCLib + # 2. Transcribe audio via AudioShake API + # 3. Run automatic correction with lyrics_transcriber + # 4. Generate corrections JSON (no interactive review) + # 5. Skip video generation (render_video=False) + + # Use lyrics_artist/lyrics_title overrides if provided, else fall back to job artist/title + lyrics_search_artist = getattr(job, 'lyrics_artist', None) or job.artist + lyrics_search_title = getattr(job, 'lyrics_title', None) or job.title + + job_log.info("Starting LyricsTranscriber processing...") + job_log.info(f" Artist: {job.artist}") + job_log.info(f" Title: {job.title}") + if lyrics_search_artist != job.artist: + job_log.info(f" Lyrics search artist override: {lyrics_search_artist}") + if lyrics_search_title != job.title: + job_log.info(f" Lyrics search title override: {lyrics_search_title}") + logger.info(f"Job {job_id}: Calling lyrics_processor.transcribe_lyrics()") + + # Run transcription in thread pool to avoid blocking the event loop + # (transcribe_lyrics is synchronous and takes 1-2 minutes) + # Note: We pass the original artist/title for file naming, but the LyricsProcessor + # will use lyrics_artist/lyrics_title for the search (set via constructor args) + result = await asyncio.to_thread( + lyrics_processor.transcribe_lyrics, + input_audio_wav=audio_path, + artist=job.artist, # Original artist for file naming + title=job.title, # Original title for file naming + track_output_dir=temp_dir, + lyrics_artist=lyrics_search_artist, # Override for lyrics search + lyrics_title=lyrics_search_title # Override for lyrics search + ) + + job_log.info("Transcription processing complete") + logger.info(f"Job {job_id}: Transcription complete, uploading results") + + # Upload lyrics results to GCS + job_log.info("Uploading lyrics results to GCS...") + await upload_lyrics_results(job_id, temp_dir, result, storage, job_manager, job_log) + + job_log.info("All lyrics data uploaded successfully") + logger.info(f"Job {job_id}: All lyrics data uploaded successfully") + + # Update progress using state_data (don't change status during parallel processing) + job_manager.update_state_data(job_id, 'lyrics_progress', { + 'stage': 'lyrics_complete', + 'progress': 45, + 'message': 'Lyrics transcription complete' + }) + + # Mark lyrics processing complete + # This will check if audio is also complete and transition to next stage if so + job_log.info("Lyrics worker complete, checking if audio is also done...") + job_manager.mark_lyrics_complete(job_id) + + return True except Exception as e: job_log.error(f"Lyrics transcription failed: {str(e)}", exc_info=True) diff --git a/backend/workers/render_video_worker.py b/backend/workers/render_video_worker.py index 7a4da415..cfa6f9d5 100644 --- a/backend/workers/render_video_worker.py +++ b/backend/workers/render_video_worker.py @@ -26,7 +26,7 @@ from backend.models.job import JobStatus from backend.services.job_manager import JobManager from backend.services.storage_service import StorageService from backend.config import get_settings -from backend.workers.worker_logging import create_job_logger, setup_job_logging +from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context # Import from lyrics_transcriber (submodule) from lyrics_transcriber.output.generator import OutputGenerator @@ -85,194 +85,198 @@ async def process_render_video(job_id: str) -> bool: logger.info(f"Job {job_id}: Starting video render (post-review)") try: - # Transition to RENDERING_VIDEO - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.RENDERING_VIDEO, - progress=75, - message="Rendering karaoke video with corrected lyrics" - ) - - with tempfile.TemporaryDirectory() as temp_dir: - job_log.info(f"Created temp directory: {temp_dir}") - # 1. Download corrected corrections data - corrections_path = os.path.join(temp_dir, "corrections.json") - - # Try updated corrections first (from human review), fall back to original - corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json" - corrections_gcs_original = f"jobs/{job_id}/lyrics/corrections.json" - - # Get GCS path from file_urls - corrections_url = job.file_urls.get('lyrics', {}).get('corrections_updated') - if not corrections_url: - corrections_url = job.file_urls.get('lyrics', {}).get('corrections') - - if not corrections_url: - # Try direct GCS paths - if storage.file_exists(corrections_gcs_updated): - corrections_gcs = corrections_gcs_updated - elif storage.file_exists(corrections_gcs_original): - corrections_gcs = corrections_gcs_original - else: - raise FileNotFoundError(f"No corrections file found for job {job_id}") - else: - # Extract GCS path from URL - corrections_gcs = _extract_gcs_path(corrections_url) - - # 2. Load the ORIGINAL corrections (has full structure) - original_corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json" - original_corrections_path = os.path.join(temp_dir, "corrections_original.json") - - job_log.info(f"Downloading original corrections from {original_corrections_gcs}") - logger.info(f"Job {job_id}: Downloading original corrections from {original_corrections_gcs}") - storage.download_file(original_corrections_gcs, original_corrections_path) - - with open(original_corrections_path, 'r', encoding='utf-8') as f: - original_data = json.load(f) - - # 3. Check if there are updated corrections (from review UI) - # The frontend sends only partial data: {corrections, corrected_segments} - updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json" - - if storage.file_exists(updated_corrections_gcs): - job_log.info("Found updated corrections from review, merging") - logger.info(f"Job {job_id}: Found updated corrections, merging") - updated_path = os.path.join(temp_dir, "corrections_updated.json") - storage.download_file(updated_corrections_gcs, updated_path) - - with open(updated_path, 'r', encoding='utf-8') as f: - updated_data = json.load(f) - - # Merge: update the original with the user's corrections - if 'corrections' in updated_data: - original_data['corrections'] = updated_data['corrections'] - if 'corrected_segments' in updated_data: - original_data['corrected_segments'] = updated_data['corrected_segments'] - - job_log.info("Merged user corrections into original data") - logger.info(f"Job {job_id}: Merged user corrections into original data") - - # 4. Convert to CorrectionResult - correction_result = CorrectionResult.from_dict(original_data) - job_log.info(f"Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments") - logger.info(f"Job {job_id}: Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments") - - # 3. Download audio file - audio_path = os.path.join(temp_dir, "audio.flac") - audio_gcs_path = job.input_media_gcs_path - - if not audio_gcs_path: - raise FileNotFoundError(f"No input audio path for job {job_id}") - - job_log.info(f"Downloading audio from {audio_gcs_path}") - logger.info(f"Job {job_id}: Downloading audio from {audio_gcs_path}") - storage.download_file(audio_gcs_path, audio_path) - job_log.info(f"Audio downloaded: {os.path.getsize(audio_path)} bytes") - - # 4. Get or create styles using the unified style loader - job_log.info("Loading style configuration...") - job_log.info(f" job.style_params_gcs_path: {job.style_params_gcs_path}") - job_log.info(f" job.style_assets: {list(job.style_assets.keys()) if job.style_assets else 'None'}") - - styles_path, style_data = load_styles_from_gcs( - style_params_gcs_path=job.style_params_gcs_path, - style_assets=job.style_assets, - temp_dir=temp_dir, - download_func=storage.download_file, - logger=job_log, - ) - - # 5. Configure OutputGenerator - output_dir = os.path.join(temp_dir, "output") - cache_dir = os.path.join(temp_dir, "cache") - os.makedirs(output_dir, exist_ok=True) - os.makedirs(cache_dir, exist_ok=True) - - job_log.info(f"Using styles from: {styles_path}") - - # Get subtitle offset from job (user-specified timing adjustment) - subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0 - if subtitle_offset != 0: - job_log.info(f"Applying subtitle offset: {subtitle_offset}ms") - - config = OutputConfig( - output_dir=output_dir, - cache_dir=cache_dir, - output_styles_json=styles_path, - render_video=True, - generate_cdg=False, # CDG optional, generated separately - generate_plain_text=True, - generate_lrc=True, - video_resolution="4k", - subtitle_offset_ms=subtitle_offset - ) - - job_log.info(f"OutputConfig: output_styles_json={config.output_styles_json}, render_video={config.render_video}") - - output_generator = OutputGenerator(config, logger) - - # 6. Generate outputs (video, LRC, ASS, etc.) - output_prefix = f"{job.artist or 'Unknown'} - {job.title or 'Unknown'}" - job_log.info(f"Generating outputs with prefix '{output_prefix}'") - logger.info(f"Job {job_id}: Generating outputs with prefix '{output_prefix}'") - - outputs = output_generator.generate_outputs( - transcription_corrected=correction_result, - lyrics_results={}, # Reference lyrics already in correction_result - output_prefix=output_prefix, - audio_filepath=audio_path, - artist=job.artist, - title=job.title - ) - - # 7. Upload video to GCS - if outputs.video and os.path.exists(outputs.video): - video_size = os.path.getsize(outputs.video) - job_log.info(f"Video generated: {video_size} bytes") - video_gcs_path = f"jobs/{job_id}/videos/with_vocals.mkv" - video_url = storage.upload_file(outputs.video, video_gcs_path) - job_manager.update_file_url(job_id, 'videos', 'with_vocals', video_url) - job_log.info(f"Uploaded with_vocals.mkv to GCS") - logger.info(f"Job {job_id}: Uploaded with_vocals.mkv ({video_size} bytes)") - else: - job_log.error("Video generation failed - no output file produced!") - raise Exception("Video generation failed - no output file produced") - - # 8. Upload LRC file - if outputs.lrc and os.path.exists(outputs.lrc): - lrc_gcs_path = f"jobs/{job_id}/lyrics/karaoke.lrc" - lrc_url = storage.upload_file(outputs.lrc, lrc_gcs_path) - job_manager.update_file_url(job_id, 'lyrics', 'lrc', lrc_url) - job_log.info("Uploaded karaoke.lrc") - logger.info(f"Job {job_id}: Uploaded karaoke.lrc") - - # 9. Upload ASS subtitle file - if outputs.ass and os.path.exists(outputs.ass): - ass_gcs_path = f"jobs/{job_id}/lyrics/karaoke.ass" - ass_url = storage.upload_file(outputs.ass, ass_gcs_path) - job_manager.update_file_url(job_id, 'lyrics', 'ass', ass_url) - job_log.info("Uploaded karaoke.ass") - logger.info(f"Job {job_id}: Uploaded karaoke.ass") - - # 10. Upload corrected text files - if outputs.corrected_txt and os.path.exists(outputs.corrected_txt): - txt_gcs_path = f"jobs/{job_id}/lyrics/corrected.txt" - txt_url = storage.upload_file(outputs.corrected_txt, txt_gcs_path) - job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', txt_url) - job_log.info("Uploaded corrected.txt") - logger.info(f"Job {job_id}: Uploaded corrected.txt") - - # 11. Transition to awaiting instrumental selection + # Use job_logging_context for proper log isolation when multiple jobs run concurrently + # This ensures logs from third-party libraries (lyrics_transcriber.output) are only + # captured by this job's handler, not handlers from other concurrent jobs + with job_logging_context(job_id): + # Transition to RENDERING_VIDEO job_manager.transition_to_state( job_id=job_id, - new_status=JobStatus.AWAITING_INSTRUMENTAL_SELECTION, - progress=80, - message="Video rendered - select your instrumental" + new_status=JobStatus.RENDERING_VIDEO, + progress=75, + message="Rendering karaoke video with corrected lyrics" ) - job_log.info("=== RENDER VIDEO WORKER COMPLETE ===") - logger.info(f"Job {job_id}: Video render complete, awaiting instrumental selection") - return True + with tempfile.TemporaryDirectory() as temp_dir: + job_log.info(f"Created temp directory: {temp_dir}") + # 1. Download corrected corrections data + corrections_path = os.path.join(temp_dir, "corrections.json") + + # Try updated corrections first (from human review), fall back to original + corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json" + corrections_gcs_original = f"jobs/{job_id}/lyrics/corrections.json" + + # Get GCS path from file_urls + corrections_url = job.file_urls.get('lyrics', {}).get('corrections_updated') + if not corrections_url: + corrections_url = job.file_urls.get('lyrics', {}).get('corrections') + + if not corrections_url: + # Try direct GCS paths + if storage.file_exists(corrections_gcs_updated): + corrections_gcs = corrections_gcs_updated + elif storage.file_exists(corrections_gcs_original): + corrections_gcs = corrections_gcs_original + else: + raise FileNotFoundError(f"No corrections file found for job {job_id}") + else: + # Extract GCS path from URL + corrections_gcs = _extract_gcs_path(corrections_url) + + # 2. Load the ORIGINAL corrections (has full structure) + original_corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json" + original_corrections_path = os.path.join(temp_dir, "corrections_original.json") + + job_log.info(f"Downloading original corrections from {original_corrections_gcs}") + logger.info(f"Job {job_id}: Downloading original corrections from {original_corrections_gcs}") + storage.download_file(original_corrections_gcs, original_corrections_path) + + with open(original_corrections_path, 'r', encoding='utf-8') as f: + original_data = json.load(f) + + # 3. Check if there are updated corrections (from review UI) + # The frontend sends only partial data: {corrections, corrected_segments} + updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json" + + if storage.file_exists(updated_corrections_gcs): + job_log.info("Found updated corrections from review, merging") + logger.info(f"Job {job_id}: Found updated corrections, merging") + updated_path = os.path.join(temp_dir, "corrections_updated.json") + storage.download_file(updated_corrections_gcs, updated_path) + + with open(updated_path, 'r', encoding='utf-8') as f: + updated_data = json.load(f) + + # Merge: update the original with the user's corrections + if 'corrections' in updated_data: + original_data['corrections'] = updated_data['corrections'] + if 'corrected_segments' in updated_data: + original_data['corrected_segments'] = updated_data['corrected_segments'] + + job_log.info("Merged user corrections into original data") + logger.info(f"Job {job_id}: Merged user corrections into original data") + + # 4. Convert to CorrectionResult + correction_result = CorrectionResult.from_dict(original_data) + job_log.info(f"Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments") + logger.info(f"Job {job_id}: Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments") + + # 3. Download audio file + audio_path = os.path.join(temp_dir, "audio.flac") + audio_gcs_path = job.input_media_gcs_path + + if not audio_gcs_path: + raise FileNotFoundError(f"No input audio path for job {job_id}") + + job_log.info(f"Downloading audio from {audio_gcs_path}") + logger.info(f"Job {job_id}: Downloading audio from {audio_gcs_path}") + storage.download_file(audio_gcs_path, audio_path) + job_log.info(f"Audio downloaded: {os.path.getsize(audio_path)} bytes") + + # 4. Get or create styles using the unified style loader + job_log.info("Loading style configuration...") + job_log.info(f" job.style_params_gcs_path: {job.style_params_gcs_path}") + job_log.info(f" job.style_assets: {list(job.style_assets.keys()) if job.style_assets else 'None'}") + + styles_path, style_data = load_styles_from_gcs( + style_params_gcs_path=job.style_params_gcs_path, + style_assets=job.style_assets, + temp_dir=temp_dir, + download_func=storage.download_file, + logger=job_log, + ) + + # 5. Configure OutputGenerator + output_dir = os.path.join(temp_dir, "output") + cache_dir = os.path.join(temp_dir, "cache") + os.makedirs(output_dir, exist_ok=True) + os.makedirs(cache_dir, exist_ok=True) + + job_log.info(f"Using styles from: {styles_path}") + + # Get subtitle offset from job (user-specified timing adjustment) + subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0 + if subtitle_offset != 0: + job_log.info(f"Applying subtitle offset: {subtitle_offset}ms") + + config = OutputConfig( + output_dir=output_dir, + cache_dir=cache_dir, + output_styles_json=styles_path, + render_video=True, + generate_cdg=False, # CDG optional, generated separately + generate_plain_text=True, + generate_lrc=True, + video_resolution="4k", + subtitle_offset_ms=subtitle_offset + ) + + job_log.info(f"OutputConfig: output_styles_json={config.output_styles_json}, render_video={config.render_video}") + + output_generator = OutputGenerator(config, logger) + + # 6. Generate outputs (video, LRC, ASS, etc.) + output_prefix = f"{job.artist or 'Unknown'} - {job.title or 'Unknown'}" + job_log.info(f"Generating outputs with prefix '{output_prefix}'") + logger.info(f"Job {job_id}: Generating outputs with prefix '{output_prefix}'") + + outputs = output_generator.generate_outputs( + transcription_corrected=correction_result, + lyrics_results={}, # Reference lyrics already in correction_result + output_prefix=output_prefix, + audio_filepath=audio_path, + artist=job.artist, + title=job.title + ) + + # 7. Upload video to GCS + if outputs.video and os.path.exists(outputs.video): + video_size = os.path.getsize(outputs.video) + job_log.info(f"Video generated: {video_size} bytes") + video_gcs_path = f"jobs/{job_id}/videos/with_vocals.mkv" + video_url = storage.upload_file(outputs.video, video_gcs_path) + job_manager.update_file_url(job_id, 'videos', 'with_vocals', video_url) + job_log.info(f"Uploaded with_vocals.mkv to GCS") + logger.info(f"Job {job_id}: Uploaded with_vocals.mkv ({video_size} bytes)") + else: + job_log.error("Video generation failed - no output file produced!") + raise Exception("Video generation failed - no output file produced") + + # 8. Upload LRC file + if outputs.lrc and os.path.exists(outputs.lrc): + lrc_gcs_path = f"jobs/{job_id}/lyrics/karaoke.lrc" + lrc_url = storage.upload_file(outputs.lrc, lrc_gcs_path) + job_manager.update_file_url(job_id, 'lyrics', 'lrc', lrc_url) + job_log.info("Uploaded karaoke.lrc") + logger.info(f"Job {job_id}: Uploaded karaoke.lrc") + + # 9. Upload ASS subtitle file + if outputs.ass and os.path.exists(outputs.ass): + ass_gcs_path = f"jobs/{job_id}/lyrics/karaoke.ass" + ass_url = storage.upload_file(outputs.ass, ass_gcs_path) + job_manager.update_file_url(job_id, 'lyrics', 'ass', ass_url) + job_log.info("Uploaded karaoke.ass") + logger.info(f"Job {job_id}: Uploaded karaoke.ass") + + # 10. Upload corrected text files + if outputs.corrected_txt and os.path.exists(outputs.corrected_txt): + txt_gcs_path = f"jobs/{job_id}/lyrics/corrected.txt" + txt_url = storage.upload_file(outputs.corrected_txt, txt_gcs_path) + job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', txt_url) + job_log.info("Uploaded corrected.txt") + logger.info(f"Job {job_id}: Uploaded corrected.txt") + + # 11. Transition to awaiting instrumental selection + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.AWAITING_INSTRUMENTAL_SELECTION, + progress=80, + message="Video rendered - select your instrumental" + ) + + job_log.info("=== RENDER VIDEO WORKER COMPLETE ===") + logger.info(f"Job {job_id}: Video render complete, awaiting instrumental selection") + return True except Exception as e: job_log.error(f"Video render failed: {e}") diff --git a/backend/workers/screens_worker.py b/backend/workers/screens_worker.py index 200c2df8..9fab35e0 100644 --- a/backend/workers/screens_worker.py +++ b/backend/workers/screens_worker.py @@ -35,7 +35,7 @@ from backend.services.job_manager import JobManager from backend.services.storage_service import StorageService from backend.config import get_settings from backend.workers.style_helper import load_style_config, StyleConfig -from backend.workers.worker_logging import create_job_logger, setup_job_logging +from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context # Import from karaoke_gen package from karaoke_gen.video_generator import VideoGenerator @@ -88,94 +88,98 @@ async def generate_screens(job_id: str) -> bool: temp_dir = tempfile.mkdtemp(prefix=f"karaoke_screens_{job_id}_") try: - job_log.info(f"Starting screen generation for {job.artist} - {job.title}") - logger.info(f"Starting screen generation for job {job_id}") - - # Transition to GENERATING_SCREENS state - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.GENERATING_SCREENS, - progress=50, - message="Generating title and end screens" - ) - - # Log style assets info - style_assets = getattr(job, 'style_assets', {}) or {} - job_log.info(f"Style assets from job: {list(style_assets.keys()) if style_assets else 'None'}") - if style_assets: - for key, path in style_assets.items(): - job_log.info(f" {key}: {path}") - - # Load style configuration (downloads assets from GCS if available) - job_log.info("Loading style configuration from GCS...") - style_config = await load_style_config(job, storage, temp_dir) - if style_config.has_custom_styles(): - job_log.info("Using CUSTOM style configuration") - logger.info(f"Job {job_id}: Using custom style configuration") - else: - job_log.warning("Using DEFAULT style configuration (no custom styles found)") - logger.info(f"Job {job_id}: Using default style configuration") - - # Initialize video generator - video_generator = _create_video_generator(temp_dir) - - # Generate title screen with style config - job_log.info("Generating title screen...") - title_screen_path = await _generate_title_screen( - job_id=job_id, - job=job, - video_generator=video_generator, - style_config=style_config, - temp_dir=temp_dir, - job_log=job_log - ) - - if not title_screen_path: - raise Exception("Title screen generation failed") - job_log.info(f"Title screen generated: {title_screen_path}") - - # Generate end screen with style config - job_log.info("Generating end screen...") - end_screen_path = await _generate_end_screen( - job_id=job_id, - job=job, - video_generator=video_generator, - style_config=style_config, - temp_dir=temp_dir, - job_log=job_log - ) - - if not end_screen_path: - raise Exception("End screen generation failed") - job_log.info(f"End screen generated: {end_screen_path}") - - # Upload screens to GCS - await _upload_screens( - job_id=job_id, - job_manager=job_manager, - storage=storage, - title_screen_path=title_screen_path, - end_screen_path=end_screen_path - ) - - # Apply countdown padding if needed - # (This is handled automatically by checking lyrics metadata) - await _apply_countdown_padding_if_needed(job_id, job_manager, job) - - # Transition to AWAITING_REVIEW - # Human must review lyrics before video can be rendered - logger.info(f"Job {job_id}: Screens generated, awaiting lyrics review") - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.AWAITING_REVIEW, - progress=55, - message="Ready for lyrics review. Please review and correct lyrics." - ) - - # TODO: Send notification to user that selection is needed - # await notify_user(job_id, "instrumental_selection_ready") - - return True + # Use job_logging_context for proper log isolation when multiple jobs run concurrently + # This ensures logs from third-party libraries (karaoke_gen.video_generator) are only + # captured by this job's handler, not handlers from other concurrent jobs + with job_logging_context(job_id): + job_log.info(f"Starting screen generation for {job.artist} - {job.title}") + logger.info(f"Starting screen generation for job {job_id}") + + # Transition to GENERATING_SCREENS state + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.GENERATING_SCREENS, + progress=50, + message="Generating title and end screens" + ) + + # Log style assets info + style_assets = getattr(job, 'style_assets', {}) or {} + job_log.info(f"Style assets from job: {list(style_assets.keys()) if style_assets else 'None'}") + if style_assets: + for key, path in style_assets.items(): + job_log.info(f" {key}: {path}") + + # Load style configuration (downloads assets from GCS if available) + job_log.info("Loading style configuration from GCS...") + style_config = await load_style_config(job, storage, temp_dir) + if style_config.has_custom_styles(): + job_log.info("Using CUSTOM style configuration") + logger.info(f"Job {job_id}: Using custom style configuration") + else: + job_log.warning("Using DEFAULT style configuration (no custom styles found)") + logger.info(f"Job {job_id}: Using default style configuration") + + # Initialize video generator + video_generator = _create_video_generator(temp_dir) + + # Generate title screen with style config + job_log.info("Generating title screen...") + title_screen_path = await _generate_title_screen( + job_id=job_id, + job=job, + video_generator=video_generator, + style_config=style_config, + temp_dir=temp_dir, + job_log=job_log + ) + + if not title_screen_path: + raise Exception("Title screen generation failed") + job_log.info(f"Title screen generated: {title_screen_path}") + + # Generate end screen with style config + job_log.info("Generating end screen...") + end_screen_path = await _generate_end_screen( + job_id=job_id, + job=job, + video_generator=video_generator, + style_config=style_config, + temp_dir=temp_dir, + job_log=job_log + ) + + if not end_screen_path: + raise Exception("End screen generation failed") + job_log.info(f"End screen generated: {end_screen_path}") + + # Upload screens to GCS + await _upload_screens( + job_id=job_id, + job_manager=job_manager, + storage=storage, + title_screen_path=title_screen_path, + end_screen_path=end_screen_path + ) + + # Apply countdown padding if needed + # (This is handled automatically by checking lyrics metadata) + await _apply_countdown_padding_if_needed(job_id, job_manager, job) + + # Transition to AWAITING_REVIEW + # Human must review lyrics before video can be rendered + logger.info(f"Job {job_id}: Screens generated, awaiting lyrics review") + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.AWAITING_REVIEW, + progress=55, + message="Ready for lyrics review. Please review and correct lyrics." + ) + + # TODO: Send notification to user that selection is needed + # await notify_user(job_id, "instrumental_selection_ready") + + return True except Exception as e: logger.error(f"Job {job_id}: Screen generation failed: {e}", exc_info=True) diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index f7f13cfb..ca426a88 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -31,7 +31,7 @@ from backend.services.rclone_service import get_rclone_service from backend.services.youtube_service import get_youtube_service from backend.config import get_settings from backend.workers.style_helper import load_style_config -from backend.workers.worker_logging import create_job_logger, setup_job_logging +from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context # Import from karaoke_gen package - reuse existing implementation from karaoke_gen.karaoke_finalise.karaoke_finalise import KaraokeFinalise @@ -105,162 +105,166 @@ async def generate_video(job_id: str) -> bool: job_log.warning("YouTube credentials not available - upload will be skipped") try: - job_log.info(f"Starting video finalization for {job.artist} - {job.title}") - logger.info(f"Starting video generation for job {job_id}") - - # Transition to GENERATING_VIDEO state - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.GENERATING_VIDEO, - progress=70, - message="Preparing files for video generation" - ) - - # Download and set up files in the format KaraokeFinalise expects - base_name = f"{job.artist} - {job.title}" - job_log.info("Downloading files from GCS (this may take a few minutes for large files)...") - await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log) - job_log.info("All files downloaded successfully") - - # Load style config for CDG styles - job_log.info("Loading CDG style configuration...") - style_config = await load_style_config(job, storage, temp_dir) - cdg_styles = style_config.get_cdg_styles() - if cdg_styles: - job_log.info("CDG styles loaded from custom configuration") - else: - job_log.info("Using default CDG styles") - - # Change to working directory (KaraokeFinalise works in cwd) - os.chdir(temp_dir) - - # Get the selected instrumental file path - # Batch 3: If user provided existing instrumental, use that; otherwise use AI-separated - instrumental_selection = job.state_data['instrumental_selection'] - existing_instrumental_path = getattr(job, 'existing_instrumental_gcs_path', None) - - if existing_instrumental_path: - # User provided existing instrumental - ext = Path(existing_instrumental_path).suffix.lower() - instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental User){ext}") - instrumental_source = "user-provided" - else: - # Use AI-separated instrumental - instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing" - instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac") - instrumental_source = instrumental_selection - - # Transition to encoding state - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.ENCODING, - progress=75, - message="Encoding videos (15-20 min)" - ) - - # Log finalization parameters - job_log.info("Creating KaraokeFinalise with parameters:") - job_log.info(f" enable_cdg: {getattr(job, 'enable_cdg', False)}") - job_log.info(f" enable_txt: {getattr(job, 'enable_txt', False)}") - job_log.info(f" brand_prefix: {getattr(job, 'brand_prefix', None)}") - job_log.info(f" discord_webhook: {'configured' if getattr(job, 'discord_webhook_url', None) else 'not configured'}") - job_log.info(f" instrumental source: {instrumental_source}") - if existing_instrumental_path: - job_log.info(f" using user-provided instrumental (selection was: {instrumental_selection})") - - # Create KaraokeFinalise with ALL the parameters from the job - # This reuses all existing functionality! - - # Set up YouTube description file if template is provided - youtube_desc_path = None - if youtube_credentials and getattr(job, 'youtube_description_template', None): - youtube_desc_path = os.path.join(temp_dir, "youtube_description.txt") - with open(youtube_desc_path, 'w') as f: - f.write(job.youtube_description_template) - job_log.info("YouTube description template written to temp file") - - finalise = KaraokeFinalise( - logger=logger, - log_level=logging.INFO, - dry_run=False, - instrumental_format="flac", - # CDG/TXT generation - enable_cdg=getattr(job, 'enable_cdg', False), - enable_txt=getattr(job, 'enable_txt', False), - cdg_styles=cdg_styles, - # Brand code and organization (server-side mode uses rclone) - brand_prefix=getattr(job, 'brand_prefix', None), - organised_dir=None, # Not used in cloud - files stay in GCS - organised_dir_rclone_root=getattr(job, 'organised_dir_rclone_root', None), - public_share_dir=None, # Not used in cloud - # Notifications - discord_webhook_url=getattr(job, 'discord_webhook_url', None), - # YouTube upload (server-side with pre-loaded credentials) - youtube_client_secrets_file=None, # Not used with pre-stored credentials - youtube_description_file=youtube_desc_path, - user_youtube_credentials=youtube_credentials, # Pre-loaded from Secret Manager - rclone_destination=None, - email_template_file=None, - # Server-side optimizations - non_interactive=True, - server_side_mode=True, - selected_instrumental_file=instrumental_file, - ) - - # Call process() - this does ALL the work: - # - Encodes to 4 video formats - # - Generates CDG/TXT packages if enabled - # - Posts Discord notification if configured - # - Handles brand code generation - job_log.info("Starting KaraokeFinalise.process() - this may take 15-20 minutes...") - logger.info(f"Job {job_id}: Starting KaraokeFinalise.process()") - result = finalise.process() - - job_log.info("KaraokeFinalise.process() complete!") - if result.get('brand_code'): - job_log.info(f"Brand code: {result.get('brand_code')}") - logger.info(f"Job {job_id}: KaraokeFinalise.process() complete") - logger.info(f"Job {job_id}: Brand code: {result.get('brand_code')}") - - # Native API distribution uploads (used by remote CLI instead of rclone) - await _handle_native_distribution( - job_id=job_id, - job=job, - job_log=job_log, - job_manager=job_manager, - temp_dir=temp_dir, - result=result, - ) - - # Upload generated files to GCS - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.PACKAGING, - progress=95, - message="Uploading final files" - ) - - await _upload_results(job_id, job_manager, storage, temp_dir, result) - - # Mark job as complete - logger.info(f"Job {job_id}: Video generation complete") - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.COMPLETE, - progress=100, - message="Karaoke generation complete!" - ) - - # Store result metadata in job - job_manager.update_job(job_id, { - 'state_data': { - **job.state_data, - 'brand_code': result.get('brand_code'), - 'youtube_url': result.get('youtube_url'), - } - }) - - return True + # Use job_logging_context for proper log isolation when multiple jobs run concurrently + # This ensures logs from third-party libraries (karaoke_gen.karaoke_finalise) are only + # captured by this job's handler, not handlers from other concurrent jobs + with job_logging_context(job_id): + job_log.info(f"Starting video finalization for {job.artist} - {job.title}") + logger.info(f"Starting video generation for job {job_id}") + + # Transition to GENERATING_VIDEO state + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.GENERATING_VIDEO, + progress=70, + message="Preparing files for video generation" + ) + + # Download and set up files in the format KaraokeFinalise expects + base_name = f"{job.artist} - {job.title}" + job_log.info("Downloading files from GCS (this may take a few minutes for large files)...") + await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log) + job_log.info("All files downloaded successfully") + + # Load style config for CDG styles + job_log.info("Loading CDG style configuration...") + style_config = await load_style_config(job, storage, temp_dir) + cdg_styles = style_config.get_cdg_styles() + if cdg_styles: + job_log.info("CDG styles loaded from custom configuration") + else: + job_log.info("Using default CDG styles") + + # Change to working directory (KaraokeFinalise works in cwd) + os.chdir(temp_dir) + + # Get the selected instrumental file path + # Batch 3: If user provided existing instrumental, use that; otherwise use AI-separated + instrumental_selection = job.state_data['instrumental_selection'] + existing_instrumental_path = getattr(job, 'existing_instrumental_gcs_path', None) + + if existing_instrumental_path: + # User provided existing instrumental + ext = Path(existing_instrumental_path).suffix.lower() + instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental User){ext}") + instrumental_source = "user-provided" + else: + # Use AI-separated instrumental + instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing" + instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac") + instrumental_source = instrumental_selection + + # Transition to encoding state + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.ENCODING, + progress=75, + message="Encoding videos (15-20 min)" + ) + + # Log finalization parameters + job_log.info("Creating KaraokeFinalise with parameters:") + job_log.info(f" enable_cdg: {getattr(job, 'enable_cdg', False)}") + job_log.info(f" enable_txt: {getattr(job, 'enable_txt', False)}") + job_log.info(f" brand_prefix: {getattr(job, 'brand_prefix', None)}") + job_log.info(f" discord_webhook: {'configured' if getattr(job, 'discord_webhook_url', None) else 'not configured'}") + job_log.info(f" instrumental source: {instrumental_source}") + if existing_instrumental_path: + job_log.info(f" using user-provided instrumental (selection was: {instrumental_selection})") + + # Create KaraokeFinalise with ALL the parameters from the job + # This reuses all existing functionality! + + # Set up YouTube description file if template is provided + youtube_desc_path = None + if youtube_credentials and getattr(job, 'youtube_description_template', None): + youtube_desc_path = os.path.join(temp_dir, "youtube_description.txt") + with open(youtube_desc_path, 'w') as f: + f.write(job.youtube_description_template) + job_log.info("YouTube description template written to temp file") + + finalise = KaraokeFinalise( + logger=logger, + log_level=logging.INFO, + dry_run=False, + instrumental_format="flac", + # CDG/TXT generation + enable_cdg=getattr(job, 'enable_cdg', False), + enable_txt=getattr(job, 'enable_txt', False), + cdg_styles=cdg_styles, + # Brand code and organization (server-side mode uses rclone) + brand_prefix=getattr(job, 'brand_prefix', None), + organised_dir=None, # Not used in cloud - files stay in GCS + organised_dir_rclone_root=getattr(job, 'organised_dir_rclone_root', None), + public_share_dir=None, # Not used in cloud + # Notifications + discord_webhook_url=getattr(job, 'discord_webhook_url', None), + # YouTube upload (server-side with pre-loaded credentials) + youtube_client_secrets_file=None, # Not used with pre-stored credentials + youtube_description_file=youtube_desc_path, + user_youtube_credentials=youtube_credentials, # Pre-loaded from Secret Manager + rclone_destination=None, + email_template_file=None, + # Server-side optimizations + non_interactive=True, + server_side_mode=True, + selected_instrumental_file=instrumental_file, + ) + + # Call process() - this does ALL the work: + # - Encodes to 4 video formats + # - Generates CDG/TXT packages if enabled + # - Posts Discord notification if configured + # - Handles brand code generation + job_log.info("Starting KaraokeFinalise.process() - this may take 15-20 minutes...") + logger.info(f"Job {job_id}: Starting KaraokeFinalise.process()") + result = finalise.process() + + job_log.info("KaraokeFinalise.process() complete!") + if result.get('brand_code'): + job_log.info(f"Brand code: {result.get('brand_code')}") + logger.info(f"Job {job_id}: KaraokeFinalise.process() complete") + logger.info(f"Job {job_id}: Brand code: {result.get('brand_code')}") + + # Native API distribution uploads (used by remote CLI instead of rclone) + await _handle_native_distribution( + job_id=job_id, + job=job, + job_log=job_log, + job_manager=job_manager, + temp_dir=temp_dir, + result=result, + ) + + # Upload generated files to GCS + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.PACKAGING, + progress=95, + message="Uploading final files" + ) + + await _upload_results(job_id, job_manager, storage, temp_dir, result) + + # Mark job as complete + logger.info(f"Job {job_id}: Video generation complete") + job_manager.transition_to_state( + job_id=job_id, + new_status=JobStatus.COMPLETE, + progress=100, + message="Karaoke generation complete!" + ) + + # Store result metadata in job + job_manager.update_job(job_id, { + 'state_data': { + **job.state_data, + 'brand_code': result.get('brand_code'), + 'youtube_url': result.get('youtube_url'), + } + }) + + return True except Exception as e: logger.error(f"Job {job_id}: Video generation failed: {e}", exc_info=True) diff --git a/backend/workers/worker_logging.py b/backend/workers/worker_logging.py index f4aca3d8..d16a3ef8 100644 --- a/backend/workers/worker_logging.py +++ b/backend/workers/worker_logging.py @@ -7,11 +7,48 @@ for remote debugging via the CLI. Two approaches are provided: 1. JobLogger - A custom logger class for explicit logging in workers 2. JobLogHandler - A logging.Handler that captures logs from any logger (including dependencies) + +IMPORTANT: Uses contextvars to ensure log isolation between concurrent jobs. +When multiple jobs run in parallel on the same Cloud Run instance, each job's +worker logs are correctly filtered to only include logs from that job. """ +import contextvars import logging +from contextlib import contextmanager from typing import Optional, Set +# Context variable to track the current job being processed +# This is used to filter logs when multiple jobs run concurrently +_current_job_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + 'current_job_id', default=None +) + + +@contextmanager +def job_logging_context(job_id: str): + """ + Context manager to set the current job ID for log filtering. + + When multiple jobs run concurrently on the same Cloud Run instance, + this ensures that each job's handler only captures logs from its own + processing thread/context. + + Usage: + with job_logging_context(job_id): + # All logs emitted here will be associated with this job_id + process_job(job_id) + + Args: + job_id: The job ID to set as the current context + """ + token = _current_job_id.set(job_id) + try: + yield + finally: + _current_job_id.reset(token) + + class JobLogHandler(logging.Handler): """ A logging handler that captures log records and stores them in Firestore. @@ -53,6 +90,15 @@ class JobLogHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: """Process a log record.""" try: + # CRITICAL: Filter by current job context to prevent log mixing + # When multiple jobs run concurrently, each attaches handlers to + # the same global loggers. Without this check, logs from Job A + # would be captured by Job B's handler and vice versa. + current_job = _current_job_id.get() + if current_job is not None and current_job != self.job_id: + # This log is from a different job's context, skip it + return + # Format the log message message = self.format(record) @@ -182,6 +228,10 @@ def setup_job_logging(job_id: str, worker_name: str, *logger_names: str) -> JobL This adds a JobLogHandler to capture logs from specified loggers (including dependency loggers like lyrics_transcriber). + IMPORTANT: When using this function, wrap your job processing code in + `job_logging_context(job_id)` to ensure proper log isolation when multiple + jobs run concurrently on the same Cloud Run instance. + Args: job_id: Job ID worker_name: Worker name @@ -199,7 +249,10 @@ def setup_job_logging(job_id: str, worker_name: str, *logger_names: str) -> JobL "lyrics_transcriber", # Capture LyricsTranscriber logs ) - # ... do work ... + # IMPORTANT: Use job_logging_context for proper isolation + with job_logging_context(job_id): + # ... do work ... + pass # Optional: remove handler when done for name in logger_names: diff --git a/docs/00-current-plan/BACKEND-FEATURE-PARITY-PLAN.md b/docs/00-current-plan/BACKEND-FEATURE-PARITY-PLAN.md index bb1bb83b..576a9f21 100644 --- a/docs/00-current-plan/BACKEND-FEATURE-PARITY-PLAN.md +++ b/docs/00-current-plan/BACKEND-FEATURE-PARITY-PLAN.md @@ -704,96 +704,17 @@ karaoke-gen-remote \ --- -## Future Architecture: Shared Pipeline +## Future Architecture -Once feature parity is achieved, the codebase should be refactored toward a **shared pipeline architecture** where both local and remote execution use the same abstractions. +For the comprehensive architecture plan covering both **horizontal scalability** (Cloud Tasks, Cloud Run Jobs) and **software architecture** (shared pipeline, SOLID, DRY), see: -### Current State (Divergent Paths) +**[SCALABLE-ARCHITECTURE-PLAN.md](./SCALABLE-ARCHITECTURE-PLAN.md)** -``` -LOCAL CLI CLOUD BACKEND -───────── ───────────── -KaraokeGen.process() API Routes - │ │ - ├─► AudioProcessor ├─► audio_worker.py - │ └── Modal API or local │ └── Modal API - │ │ - ├─► LyricsProcessor ├─► lyrics_worker.py - │ └── Orchestrates everything │ └── Transcription only - │ including video generation │ - │ ├─► screens_worker.py - │ │ - │ ├─► render_video_worker.py - │ │ └── OutputGenerator directly - │ │ - └─► KaraokeFinalise └─► video_worker.py - └── Encoding, distribution └── KaraokeFinalise -``` - -**Problems:** -- Video generation called differently (via LyricsProcessor vs OutputGenerator directly) -- LyricsProcessor does too many things (fetching, transcription, video, file management) -- Testing requires mocking different things for local vs remote -- Bug fixes may need to be applied in multiple places - -### Target State (Shared Pipeline) - -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ SHARED PIPELINE ARCHITECTURE │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ AudioInput → Separation → Transcription → Review → Render → Finalize │ -│ │ │ │ │ │ │ │ -│ ▼ ▼ ▼ ▼ ▼ ▼ │ -│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │ -│ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ -│ │ API │ │ API │ │ API │ │ API │ │ API │ │ API │ │ -│ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ │ -│ │ │ │ │ │ │ │ -│ ┌────┴───────────┴────────────┴────────────┴──────────┴──────────┴────┐ │ -│ │ EXECUTION LAYER │ │ -│ │ │ │ -│ │ Local Mode: │ Remote Mode: │ │ -│ │ - Direct calls │ - HTTP to backend │ │ -│ │ - Local GPU/CPU │ - Workers + Modal │ │ -│ │ - Blocking │ - Async + polling │ │ -│ └──────────────────────┴───────────────────────────────────────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────────┘ -``` - -**Benefits:** -- Single source of truth for each pipeline stage -- Each stage independently testable -- Same business logic regardless of execution mode -- Easier to add new stages or modify existing ones -- DRY - fix bugs once, works everywhere - -### Implementation Plan for Shared Pipeline - -This refactor should be done in a separate branch after the v0.71.0 release: - -1. **Extract stage interfaces** - ```python - class PipelineStage(Protocol): - async def execute(self, context: PipelineContext) -> StageResult: - ... - ``` - -2. **Create execution adapters** - ```python - class LocalExecutor: - """Runs stages directly in-process""" - - class RemoteExecutor: - """Runs stages via backend API/workers""" - ``` - -3. **Refactor incrementally** - - Start with one stage (e.g., Separation) - - Prove the pattern works - - Migrate other stages +Key goals: +- **100+ concurrent jobs** with no performance degradation +- **Dedicated resources per job** (no CPU/memory contention) +- **Shared pipeline architecture** - single source of truth for business logic +- **DRY codebase** - bug fixes apply to both local and remote execution --- @@ -1003,9 +924,10 @@ DEFAULT_DISCORD_WEBHOOK_URL=https://discord.com/... 5. **Edit Lyrics Mode** (Batch 9) - Allow fixing lyrics on completed jobs -### Long-Term (Shared Pipeline Refactor) -1. Create new branch for shared pipeline architecture -2. Extract stage interfaces -3. Implement execution adapters -4. Migrate stages incrementally -5. Achieve true code sharing between local and remote +### Long-Term (Scalable Architecture) + +See **[SCALABLE-ARCHITECTURE-PLAN.md](./SCALABLE-ARCHITECTURE-PLAN.md)** for the comprehensive plan covering: +1. Cloud Tasks integration for horizontal scalability +2. Cloud Run Jobs for long-running video encoding +3. Shared pipeline architecture for code reuse +4. Implementation roadmap and migration strategy diff --git a/docs/00-current-plan/SCALABLE-ARCHITECTURE-PLAN.md b/docs/00-current-plan/SCALABLE-ARCHITECTURE-PLAN.md new file mode 100644 index 00000000..7b279730 --- /dev/null +++ b/docs/00-current-plan/SCALABLE-ARCHITECTURE-PLAN.md @@ -0,0 +1,656 @@ +# Scalable Architecture Plan + +**Last Updated:** 2025-12-11 + +This document outlines the plan for evolving karaoke-gen's cloud backend to support **100+ concurrent jobs** with no performance degradation, while also achieving a clean, maintainable codebase with proper separation of concerns. + +--- + +## Goals + +### Infrastructure Goals +1. **Horizontal scalability** - Each job gets dedicated resources (no resource contention) +2. **Maximum processing speed** - Same speed for 1 job or 100 concurrent jobs +3. **Reliability** - Automatic retries, no lost jobs from container restarts +4. **Cost efficiency** - Scale to zero when idle, pay only for active processing + +### Software Architecture Goals +1. **SOLID principles** - Single responsibility, clean interfaces +2. **DRY** - Share code between local CLI and remote backend +3. **Testability** - Each component independently testable +4. **Maintainability** - Bug fixes apply everywhere, easy to add features + +--- + +## Current Architecture + +### What Works Well + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CURRENT ARCHITECTURE (v0.71.x) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ✅ Heavy work already offloaded to external services: │ +│ │ +│ Audio Separation ──────────────────▶ Modal API (GPU, ~5-8 min) │ +│ Lyrics Transcription ──────────────▶ AudioShake API (~2-3 min) │ +│ │ +│ ✅ State persistence in Firestore (survives restarts) │ +│ ✅ Files in GCS (no local storage dependencies) │ +│ ✅ Internal HTTP endpoints exist for worker triggers │ +│ ✅ Workers are self-contained (only need job_id) │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Current Limitations + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CURRENT FLOW (PROBLEMATIC) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ POST /api/jobs/upload │ +│ │ │ +│ ├──▶ Save job to Firestore ✅ │ +│ │ │ +│ └──▶ BackgroundTasks.add_task(trigger_workers) │ +│ │ │ +│ └──▶ HTTP POST /api/internal/workers/audio │ +│ │ │ +│ └──▶ BackgroundTasks.add_task(process_audio) ⚠️ │ +│ │ │ +│ └── Runs IN THE SAME CONTAINER │ +│ as the API request handler │ +│ │ +│ PROBLEMS: │ +│ ⚠️ Container may be killed before background tasks complete │ +│ ⚠️ No guaranteed delivery - if container crashes, job is lost │ +│ ⚠️ Multiple concurrent jobs share CPU/memory (resource contention) │ +│ ⚠️ Long-running video encoding (15-20 min) vulnerable to timeouts │ +│ ⚠️ FFmpeg renders compete for resources when jobs run in parallel │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Target Architecture: Horizontally Scalable Infrastructure + +### Option A: Google Cloud Tasks (Recommended for Workers) + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CLOUD TASKS ARCHITECTURE │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ POST /api/jobs/upload │ +│ │ │ +│ ├──▶ Save job to Firestore │ +│ │ │ +│ └──▶ Enqueue Cloud Tasks: │ +│ │ │ +│ ├──▶ audio-worker-queue ──────────────────────────────┐ │ +│ │ {job_id, target: /internal/workers/audio} │ │ +│ │ │ │ +│ └──▶ lyrics-worker-queue ─────────────────────────────┤ │ +│ {job_id, target: /internal/workers/lyrics} │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ Cloud Tasks Queue │ │ +│ │ │ │ +│ │ • Guaranteed delivery (retries on failure) │ │ +│ │ • Rate limiting / concurrency control │ │ +│ │ • Task deduplication │ │ +│ │ • Configurable timeouts (up to 30 min per task) │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ Cloud Run (Auto-Scaled Worker Instances) │ │ +│ │ │ │ +│ │ Instance 1 Instance 2 Instance 3 ... │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ +│ │ │ Job A │ │ Job B │ │ Job C │ │ │ +│ │ │ Audio │ │ Audio │ │ Lyrics │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ │ │ +│ │ │ │ +│ │ Each task gets a DEDICATED container instance │ │ +│ │ • Full CPU/memory for that task alone │ │ +│ │ • No resource contention with other jobs │ │ +│ │ • Container lifecycle tied to task completion │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +**Benefits:** +- ✅ Each job gets dedicated resources (no contention) +- ✅ Automatic retries on failure +- ✅ Queue-based backpressure (won't overwhelm external APIs) +- ✅ Container lifecycle matches task lifecycle +- ✅ Cloud Run auto-scales based on queue depth +- ✅ Scales to 1000s of concurrent jobs + +**Migration Effort:** LOW - Internal HTTP endpoints already exist + +### Option B: Cloud Run Jobs (For Long-Running Tasks) + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CLOUD RUN JOBS ARCHITECTURE │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ For tasks that exceed 30-minute Cloud Tasks timeout (video encoding): │ +│ │ +│ POST /api/jobs/{id}/select-instrumental │ +│ │ │ +│ └──▶ Start Cloud Run Job execution: │ +│ │ │ +│ └──▶ gcloud run jobs execute video-worker-job \ │ +│ --args job_id=$JOB_ID │ +│ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ Cloud Run Job Execution │ │ +│ │ │ │ +│ │ • Container runs until completion (up to 24 hours) │ │ +│ │ • Clean separation from API service │ │ +│ │ • Can request higher CPU/memory than service │ │ +│ │ • Ideal for video encoding (15-20 min) │ │ +│ │ • Progress written to Firestore for CLI monitoring │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +**Use Cases:** +- Video rendering (15-20 min of FFmpeg encoding) +- Final video generation with multiple output formats +- Any task that might exceed 30 minutes + +### Option C: Pub/Sub + Cloud Run (Event-Driven) + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PUB/SUB EVENT-DRIVEN ARCHITECTURE │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Job State Changes ──────────▶ Pub/Sub Topics │ +│ │ │ +│ job.created ───────────────────────┼──▶ audio-worker subscription │ +│ │ └──▶ /internal/workers/audio │ +│ │ │ +│ job.created ───────────────────────┼──▶ lyrics-worker subscription │ +│ │ └──▶ /internal/workers/lyrics │ +│ │ │ +│ audio.complete + lyrics.complete ──┼──▶ screens-worker subscription │ +│ │ └──▶ /internal/workers/screens │ +│ │ │ +│ review.complete ───────────────────┼──▶ render-worker subscription │ +│ │ └──▶ /internal/workers/render │ +│ │ │ +│ instrumental.selected ─────────────┼──▶ video-worker subscription │ +│ └──▶ /internal/workers/video │ +│ │ +│ Benefits: │ +│ • True decoupling (workers don't know about each other) │ +│ • Dead-letter queues for failed messages │ +│ • Can add new worker types without changing existing code │ +│ • Natural fit for event-sourced architecture │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Recommended Hybrid Approach + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ RECOMMENDED: HYBRID ARCHITECTURE │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Cloud Tasks (fast workers < 30 min): │ +│ ├── audio-worker-queue (5-8 min via Modal API) │ +│ ├── lyrics-worker-queue (2-3 min via AudioShake) │ +│ ├── screens-worker-queue (30 sec FFmpeg) │ +│ └── render-worker-queue (10-15 min LyricsTranscriber) │ +│ │ +│ Cloud Run Jobs (long workers > 30 min): │ +│ └── video-encoding-job (15-20 min FFmpeg multi-format encoding) │ +│ │ +│ Both use the SAME internal HTTP endpoints - minimal code changes! │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Target Architecture: Shared Pipeline (Software) + +### Current State (Divergent Paths) + +``` +LOCAL CLI CLOUD BACKEND +───────── ───────────── +KaraokeGen.process() API Routes + │ │ + ├─► AudioProcessor ├─► audio_worker.py + │ └── Modal API or local │ └── Modal API + │ │ + ├─► LyricsProcessor ├─► lyrics_worker.py + │ └── Orchestrates everything │ └── Transcription only + │ including video generation │ + │ ├─► screens_worker.py + │ │ + │ ├─► render_video_worker.py + │ │ └── OutputGenerator directly + │ │ + └─► KaraokeFinalise └─► video_worker.py + └── Encoding, distribution └── KaraokeFinalise +``` + +**Problems:** +- Video generation called differently (via LyricsProcessor vs OutputGenerator directly) +- LyricsProcessor does too many things (fetching, transcription, video, file management) +- Testing requires mocking different things for local vs remote +- Bug fixes may need to be applied in multiple places + +### Target State (Shared Pipeline) + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ SHARED PIPELINE ARCHITECTURE │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ AudioInput → Separation → Transcription → Review → Render → Finalize │ +│ │ │ │ │ │ │ │ +│ ▼ ▼ ▼ ▼ ▼ ▼ │ +│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │ +│ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ +│ │ API │ │ API │ │ API │ │ API │ │ API │ │ API │ │ +│ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ │ +│ │ │ │ │ │ │ │ +│ ┌────┴───────────┴────────────┴────────────┴──────────┴──────────┴────┐ │ +│ │ EXECUTION LAYER │ │ +│ │ │ │ +│ │ Local Mode: │ Remote Mode: │ │ +│ │ - Direct calls │ - HTTP to backend │ │ +│ │ - Local GPU/CPU │ - Cloud Tasks queue │ │ +│ │ - Blocking │ - Async + polling │ │ +│ └──────────────────────┴───────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Stage Interface Design + +```python +# karaoke_gen/pipeline/base.py + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Protocol + +@dataclass +class PipelineContext: + """Shared context passed between pipeline stages.""" + job_id: str + artist: str + title: str + temp_dir: str + style_config: dict + # ... other shared state + +@dataclass +class StageResult: + """Result from a pipeline stage.""" + success: bool + outputs: dict # Stage-specific outputs + error: Optional[str] = None + +class PipelineStage(Protocol): + """Interface for all pipeline stages.""" + + @property + def name(self) -> str: + """Stage identifier.""" + ... + + async def execute(self, context: PipelineContext) -> StageResult: + """Execute the stage.""" + ... + + async def validate_inputs(self, context: PipelineContext) -> bool: + """Verify required inputs exist.""" + ... +``` + +### Execution Adapters + +```python +# karaoke_gen/pipeline/executors/local.py + +class LocalExecutor: + """Runs stages directly in-process (for CLI).""" + + async def run_stage(self, stage: PipelineStage, context: PipelineContext) -> StageResult: + # Validate inputs + if not await stage.validate_inputs(context): + return StageResult(success=False, outputs={}, error="Missing inputs") + + # Run directly + return await stage.execute(context) + +# karaoke_gen/pipeline/executors/remote.py + +class RemoteExecutor: + """Runs stages via backend API/Cloud Tasks (for remote CLI).""" + + async def run_stage(self, stage: PipelineStage, context: PipelineContext) -> StageResult: + # Enqueue to Cloud Tasks + task = cloud_tasks_client.create_task( + queue=f"{stage.name}-worker-queue", + http_request={ + "url": f"{self.backend_url}/api/internal/workers/{stage.name}", + "body": json.dumps({"job_id": context.job_id}), + } + ) + + # Poll for completion + while True: + status = await self.get_job_status(context.job_id) + if status.stage_complete(stage.name): + return StageResult(success=True, outputs=status.stage_outputs) + await asyncio.sleep(5) +``` + +--- + +## Combined Architecture Vision + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ COMPLETE TARGET ARCHITECTURE │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ PIPELINE STAGES (SHARED) │ │ +│ │ │ │ +│ │ SeparationStage TranscriptionStage RenderStage FinalizeStage │ │ +│ │ │ │ │ │ │ │ +│ │ Single source of truth for business logic │ │ +│ │ Used by both local CLI and cloud workers │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌───────────┴───────────┐ │ +│ │ │ │ +│ ┌──────────────────────▼───────┐ ┌──────────▼──────────────────────┐ │ +│ │ LOCAL EXECUTOR │ │ REMOTE EXECUTOR │ │ +│ │ │ │ │ │ +│ │ karaoke-gen CLI │ │ karaoke-gen-remote CLI │ │ +│ │ • Direct stage calls │ │ • HTTP to backend │ │ +│ │ • Local GPU/CPU │ │ • Progress polling │ │ +│ │ • Blocking execution │ │ │ │ +│ │ • Browser review UI │ │ │ │ +│ └──────────────────────────────┘ └──────────┬──────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ CLOUD INFRASTRUCTURE │ │ +│ │ │ │ +│ │ Cloud Run Service (API) │ │ +│ │ ├── Job submission, status, review endpoints │ │ +│ │ └── Enqueues tasks to Cloud Tasks │ │ +│ │ │ │ +│ │ Cloud Tasks Queues (Guaranteed Delivery) │ │ +│ │ ├── audio-worker-queue ──▶ Cloud Run (auto-scaled) │ │ +│ │ ├── lyrics-worker-queue ──▶ Cloud Run (auto-scaled) │ │ +│ │ ├── screens-worker-queue ──▶ Cloud Run (auto-scaled) │ │ +│ │ └── render-worker-queue ──▶ Cloud Run (auto-scaled) │ │ +│ │ │ │ +│ │ Cloud Run Jobs (Long-Running) │ │ +│ │ └── video-encoding-job ──▶ Dedicated high-CPU instance │ │ +│ │ │ │ +│ │ Each task gets DEDICATED RESOURCES: │ │ +│ │ • 2-4 vCPU, 4-8GB RAM per task │ │ +│ │ • No contention with other jobs │ │ +│ │ • Same speed whether 1 job or 100 jobs │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ External Services (Unchanged): │ +│ ├── Modal API (GPU audio separation) │ +│ ├── AudioShake API (transcription) │ +│ ├── Firestore (job state) │ +│ └── GCS (file storage) │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Implementation Plan + +### Phase 1: Infrastructure Scaling (Priority: HIGH) + +**Goal:** Enable 100+ concurrent jobs with no performance degradation + +**Changes:** + +1. **Add Cloud Tasks infrastructure (Pulumi)** + ```python + # infrastructure/__main__.py + + # Create Cloud Tasks queues + audio_queue = gcp.cloudtasks.Queue("audio-worker-queue", + location=region, + rate_limits={ + "max_dispatches_per_second": 10, # Rate limit to protect Modal API + "max_concurrent_dispatches": 50, # Max parallel workers + }, + retry_config={ + "max_attempts": 3, + "min_backoff": "10s", + "max_backoff": "300s", + } + ) + # Similar for lyrics, screens, render queues + ``` + +2. **Update WorkerService to use Cloud Tasks** + ```python + # backend/services/worker_service.py + + async def trigger_worker(self, worker_type: str, job_id: str) -> bool: + if self.use_cloud_tasks: + # Production: enqueue to Cloud Tasks + return await self._enqueue_cloud_task(worker_type, job_id) + else: + # Development: direct HTTP call (existing behavior) + return await self._trigger_http(worker_type, job_id) + ``` + +3. **Add Cloud Run Job for video encoding** + ```yaml + # cloudbuild-video-job.yaml + - name: gcr.io/cloud-builders/gcloud + args: + - run + - jobs + - create + - video-encoding-job + - --image=${_IMAGE} + - --cpu=4 + - --memory=8Gi + - --max-retries=2 + - --task-timeout=3600 # 1 hour max + ``` + +4. **Update internal endpoints for Cloud Tasks** + - Add request validation for Cloud Tasks headers + - Extend timeout handling for long-running tasks + - Add idempotency for safe retries + +**Estimated Effort:** 2-3 days +**Risk:** Low (existing endpoints work unchanged) + +### Phase 2: Shared Pipeline Architecture (Priority: MEDIUM) + +**Goal:** Single source of truth for business logic, DRY codebase + +**Changes:** + +1. **Extract stage interfaces** + ``` + karaoke_gen/pipeline/ + ├── __init__.py + ├── base.py # PipelineStage, PipelineContext, StageResult + ├── context.py # Context management + ├── stages/ + │ ├── __init__.py + │ ├── separation.py # Audio separation stage + │ ├── transcription.py # Lyrics transcription stage + │ ├── screens.py # Title/end screen generation + │ ├── render.py # Video rendering with lyrics + │ └── finalize.py # Final encoding and distribution + └── executors/ + ├── __init__.py + ├── local.py # Direct execution for CLI + └── remote.py # Cloud Tasks execution for remote CLI + ``` + +2. **Migrate stages incrementally** + - Start with SeparationStage (simplest, well-isolated) + - Prove pattern works with both executors + - Migrate remaining stages one at a time + - Keep old code paths working during migration + +3. **Update CLIs to use pipeline** + ```python + # karaoke_gen/utils/gen_cli.py (local) + executor = LocalExecutor() + pipeline = Pipeline([ + SeparationStage(), + TranscriptionStage(), + ReviewStage(), + RenderStage(), + FinalizeStage(), + ]) + await pipeline.run(executor, context) + + # karaoke_gen/utils/remote_cli.py (remote) + executor = RemoteExecutor(backend_url) + # Same pipeline, different executor + await pipeline.run(executor, context) + ``` + +**Estimated Effort:** 1-2 weeks +**Risk:** Medium (requires careful refactoring) + +### Phase 3: Advanced Features (Priority: LOW) + +1. **Event-driven architecture with Pub/Sub** (optional) +2. **Workflow orchestration with Cloud Workflows** (optional) +3. **Cost optimization with preemptible VMs** (for video encoding) +4. **Multi-region deployment** for lower latency + +--- + +## Migration Strategy + +### Zero-Downtime Migration + +``` +Week 1: Deploy Cloud Tasks infrastructure (no code changes) + └── Queues exist but aren't used yet + +Week 2: Add feature flag for Cloud Tasks + └── ENABLE_CLOUD_TASKS=false (default) + └── Test with flag enabled in staging + +Week 3: Enable Cloud Tasks in production + └── ENABLE_CLOUD_TASKS=true + └── Monitor for issues + +Week 4: Remove old BackgroundTasks code paths + └── Clean up feature flag + └── Update documentation +``` + +### Rollback Plan + +If issues occur after enabling Cloud Tasks: +1. Set `ENABLE_CLOUD_TASKS=false` +2. Redeploy (takes ~2 minutes) +3. New jobs use BackgroundTasks (old behavior) +4. In-flight Cloud Tasks jobs continue to completion + +--- + +## Resource Configuration + +### Cloud Tasks Queues + +| Queue | Max Concurrent | Rate Limit | Timeout | Retries | +|-------|---------------|------------|---------|---------| +| audio-worker | 50 | 10/sec | 15 min | 3 | +| lyrics-worker | 50 | 10/sec | 10 min | 3 | +| screens-worker | 100 | 50/sec | 2 min | 3 | +| render-worker | 20 | 5/sec | 30 min | 2 | + +### Cloud Run Worker Instances + +| Worker | CPU | Memory | Timeout | Notes | +|--------|-----|--------|---------|-------| +| audio | 2 | 2Gi | 15 min | Mostly I/O (Modal API) | +| lyrics | 2 | 2Gi | 10 min | Mostly I/O (AudioShake) | +| screens | 2 | 2Gi | 2 min | FFmpeg (quick) | +| render | 4 | 4Gi | 30 min | LyricsTranscriber + FFmpeg | +| video-job | 4 | 8Gi | 60 min | Heavy FFmpeg encoding | + +### Cost Estimate (100 concurrent jobs) + +``` +Cloud Tasks: ~$0.40/million tasks ≈ $0.01/day +Cloud Run: ~$0.00002/vCPU-sec ≈ $5-10/day (100 jobs) +Cloud Run Jobs: ~$0.00002/vCPU-sec ≈ $2-5/day (100 jobs) +Modal (GPU): ~$2.50/hour ≈ $20-40/day (100 jobs) +AudioShake: Usage-based ≈ $10-20/day (100 jobs) + +Total: ~$40-80/day for 100 concurrent jobs +``` + +--- + +## Success Criteria + +### Infrastructure +- [ ] 100 concurrent jobs complete without failures +- [ ] Processing time per job unchanged (same as 1 job) +- [ ] No resource contention (CPU/memory per job consistent) +- [ ] Automatic retry on transient failures +- [ ] Zero lost jobs from container restarts + +### Software Architecture +- [ ] Single stage implementation used by both CLIs +- [ ] Bug fixes apply to both local and remote execution +- [ ] Each stage independently testable +- [ ] Clear interfaces between stages +- [ ] 80%+ code coverage on pipeline module + +--- + +## Related Documents + +- [ARCHITECTURE.md](../01-reference/ARCHITECTURE.md) - Current system architecture +- [BACKEND-FEATURE-PARITY-PLAN.md](./BACKEND-FEATURE-PARITY-PLAN.md) - Feature parity roadmap +- [WORKER-IMPLEMENTATION-PLAN.md](./WORKER-IMPLEMENTATION-PLAN.md) - Worker details + +--- + +## Changelog + +### 2025-12-11: Initial Plan +- Documented current architecture limitations +- Proposed Cloud Tasks + Cloud Run Jobs hybrid +- Outlined shared pipeline architecture +- Created implementation roadmap diff --git a/karaoke_gen/file_handler.py b/karaoke_gen/file_handler.py index de6195ae..e8896ac5 100644 --- a/karaoke_gen/file_handler.py +++ b/karaoke_gen/file_handler.py @@ -121,16 +121,18 @@ class FileHandler: }, } - # Handle cookies if provided + # Handle cookies if provided - use safe tempfile pattern to avoid leaks + cookie_file_path = None if cookies_str: - # Write cookies to a temporary file - cookie_file = tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) try: - cookie_file.write(cookies_str) - cookie_file.close() - ydl_opts['cookiefile'] = cookie_file.name + # Use context manager to safely write cookies file + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as cookie_file: + cookie_file.write(cookies_str) + cookie_file_path = cookie_file.name + ydl_opts['cookiefile'] = cookie_file_path except Exception as e: self.logger.warning(f"Failed to write cookies file: {e}") + cookie_file_path = None try: with yt_dlp.YoutubeDL(ydl_opts) as ydl: @@ -177,9 +179,9 @@ class FileHandler: return None finally: # Clean up cookie file if we created one - if cookies_str and 'cookie_file' in locals(): + if cookie_file_path is not None: try: - os.unlink(cookie_file.name) + os.unlink(cookie_file_path) except Exception: pass diff --git a/karaoke_gen/utils/remote_cli.py b/karaoke_gen/utils/remote_cli.py index 73b63ab3..21b332ca 100644 --- a/karaoke_gen/utils/remote_cli.py +++ b/karaoke_gen/utils/remote_cli.py @@ -252,7 +252,6 @@ class RemoteKaraokeClient: url: str, artist: Optional[str] = None, title: Optional[str] = None, - style_params_path: Optional[str] = None, enable_cdg: bool = True, enable_txt: bool = True, brand_prefix: Optional[str] = None, @@ -275,11 +274,14 @@ class RemoteKaraokeClient: The backend will download the audio from the URL and process it. Artist and title will be auto-detected from the URL if not provided. + Note: Custom style configuration is not supported for URL-based jobs. + If you need custom styles, download the audio locally first and use + the regular file upload flow with submit_job(). + Args: url: YouTube or other video URL to download audio from artist: Artist name (optional - auto-detected if not provided) title: Song title (optional - auto-detected if not provided) - style_params_path: Path to style_params.json (optional) enable_cdg: Generate CDG+MP3 package enable_txt: Generate TXT+MP3 package brand_prefix: Brand code prefix (e.g., "NOMAD") @@ -2045,11 +2047,16 @@ def main(): # Submit job - different endpoint for URL vs file if is_url_input: # URL-based job submission + # Note: style_params_path is not supported for URL-based jobs + # If custom styles are needed, download the audio locally first + if args.style_params_json: + logger.warning("Custom styles (--style_params_json) are not supported for URL-based jobs. " + "Download the audio locally first and use file upload for custom styles.") + result = client.submit_job_from_url( url=input_media, artist=artist, title=title, - style_params_path=args.style_params_json, enable_cdg=args.enable_cdg, enable_txt=args.enable_txt, brand_prefix=args.brand_prefix, diff --git a/pyproject.toml b/pyproject.toml index 30292a51..5e2a56c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.71.26" +version = "0.71.29" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"