fix: address CodeRabbit review comments for Batch 4

1. _validate_url now handles None and non-string inputs safely
   - Added early guard: if url is None or not isinstance(url, str)
   - Updated function signature to accept Optional[str]

2. Fixed cookie tempfile leak in file_handler.py
   - Use context manager pattern to safely write cookies
   - Track cookie_file_path instead of cookie_file object
   - Guaranteed cleanup in finally block

3. Removed unused style_params_path from submit_job_from_url
   - URL-based jobs don't support custom styles (would need signed URL upload)
   - Added warning when user provides --style_params_json with URL input
   - Updated docstring to clarify limitation

Also merged latest main which added job_logging_context for log isolation.
This commit is contained in:
Andrew Beveridge committed 2025-12-11 03:23:49 -05:00
commit 5630cfd253
14 files changed
+1765 -1055

No files matched your search

@@ -1,35 +1,167 @@
name: Backend Tests
name: CI
on:
push: # Run on all branch pushes
push:
pull_request:
branches: [ main ]
branches: [main]
workflow_dispatch:
permissions:
contents: write
pages: write
id-token: write
concurrency:
group: "ci-${{ github.ref }}"
cancel-in-progress: true
jobs:
unit-tests:
name: Unit Tests
# ============================================
# PACKAGE TESTS (karaoke-gen Python CLI)
# ============================================
package-unit-tests:
name: "Package - Unit Tests (70% coverage)"
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- uses: actions/checkout@v4
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: '3.13'
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Install dependencies
run: poetry install --with dev
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
- name: Install spaCy language models
run: poetry run python -m spacy download en_core_web_sm
- name: Run unit tests
run: |
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70
- name: Upload coverage reports
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: unittests
name: codecov-umbrella
package-integration-tests:
name: "Package - Integration Tests"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Install dependencies
run: poetry install --with dev
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
- name: Install spaCy language models
run: poetry run python -m spacy download en_core_web_sm
- name: Run integration tests
run: poetry run pytest tests/integration/ -v
package-build:
name: "Package - Build & Install"
runs-on: ubuntu-latest
needs: [package-unit-tests, package-integration-tests]
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Build package
run: poetry build
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
haskell: true
dotnet: false
large-packages: false
docker-images: false
swap-storage: false
- name: Test package installation
run: |
pip install dist/*.whl
python -m karaoke_gen.utils.gen_cli --help
- name: Upload build artifacts
uses: actions/upload-artifact@v4
with:
name: package-dist
path: dist/
retention-days: 1
# ============================================
# BACKEND TESTS (Cloud Run API service)
# ============================================
backend-unit-tests:
name: "Backend - Unit Tests"
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Install dependencies
run: poetry install
- name: Run unit tests
env:
GOOGLE_CLOUD_PROJECT: test-project
@@ -43,11 +175,10 @@ jobs:
backend/tests/test_file_upload.py \
backend/tests/test_services.py \
-v --tb=short
emulator-tests:
name: Emulator Integration Tests
backend-emulator-tests:
name: "Backend - Emulator Integration Tests"
runs-on: ubuntu-latest
steps:
- name: Free disk space
run: |
@@ -55,144 +186,250 @@ jobs:
sudo rm -rf /opt/ghc
sudo rm -rf /usr/local/share/boost
df -h
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: '3.13'
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Set up Java 21 (required by Firestore emulator)
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
distribution: "temurin"
java-version: "21"
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
with:
version: 'latest'
version: "latest"
- name: Install Firestore emulator
run: |
gcloud components install beta cloud-firestore-emulator --quiet
run: gcloud components install beta cloud-firestore-emulator --quiet
- name: Install dependencies
run: poetry install
- name: Start Docker for GCS emulator
run: |
docker pull fsouza/fake-gcs-server:latest
run: docker pull fsouza/fake-gcs-server:latest
- name: Run emulator integration tests
run: |
chmod +x scripts/*.sh
./scripts/run-emulator-tests.sh
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results
name: backend-test-results
path: backend/htmlcov/
retention-days: 7
lint:
name: Code Quality
backend-lint:
name: "Backend - Code Quality"
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: '3.13'
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Install dependencies
run: poetry install
- name: Run validation (syntax check only)
run: |
# Skip full validation as it requires GCP credentials
# Just check Python syntax and imports
poetry run python -m py_compile backend/main.py backend/config.py
echo "✅ Basic syntax validation passed"
deploy:
name: Deploy via Cloud Build
# ============================================
# DEPLOYMENTS (main branch only)
# ============================================
package-publish:
name: "Deploy - Publish to PyPI"
runs-on: ubuntu-latest
needs: [unit-tests, emulator-tests, lint]
# Deploy on any push where tests pass (pre-alpha: rapid iteration mode)
# TODO: Change back to main-only when ready for production: github.ref == 'refs/heads/main'
if: github.event_name == 'push'
needs: [package-build]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: package-dist
path: dist/
- name: Extract version
id: extract_version
run: |
VERSION=$(poetry version --short)
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "📋 Extracted version: $VERSION"
- name: Check if tag exists
id: check_tag
run: |
VERSION="${{ steps.extract_version.outputs.version }}"
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
echo "tag_exists=true" >> $GITHUB_OUTPUT
echo "🏷️ Tag v$VERSION already exists, skipping release"
else
echo "tag_exists=false" >> $GITHUB_OUTPUT
echo "✨ Tag v$VERSION does not exist, proceeding with release"
fi
- name: Publish to PyPI
if: steps.check_tag.outputs.tag_exists == 'false'
run: |
echo "📦 Publishing the following files to PyPI:"
ls -la dist/
echo "🚀 Publishing to PyPI..."
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
poetry publish
- name: Create Git Tag
if: steps.check_tag.outputs.tag_exists == 'false'
run: |
git config user.name github-actions
git config user.email github-actions@github.com
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}"
- name: Create GitHub Release
if: steps.check_tag.outputs.tag_exists == 'false'
uses: softprops/action-gh-release@v1
with:
tag_name: "v${{ steps.extract_version.outputs.version }}"
name: "v${{ steps.extract_version.outputs.version }}"
body: |
## Changes in v${{ steps.extract_version.outputs.version }}
This release was automatically created from the main branch.
### Installation
```bash
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
```
### Assets
- Python package published to PyPI
- Frontend deployed to GitHub Pages at gen.nomadkaraoke.com
For detailed changes, see the commit history.
draft: false
prerelease: false
files: |
dist/*.whl
dist/*.tar.gz
deploy-frontend:
name: "Deploy - Frontend (GitHub Pages)"
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Prepare frontend files
run: |
echo "📁 Preparing frontend files for deployment..."
mkdir -p _site
cp -r frontend/* _site/
echo "✅ Frontend files copied to _site/"
ls -la _site/
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: "_site"
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
deploy-backend:
name: "Deploy - Backend (Cloud Run)"
runs-on: ubuntu-latest
needs: [backend-unit-tests, backend-emulator-tests, backend-lint]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
permissions:
contents: read
id-token: write # Required for Workload Identity Federation
id-token: write
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Authenticate to Google Cloud (Workload Identity)
id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
with:
version: 'latest'
version: "latest"
- name: Check if base image needs rebuild
id: check-base
run: |
echo "🔍 Checking if base image needs rebuild..."
# Calculate current poetry.lock hash
CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1)
echo "Current poetry.lock hash: $CURRENT_HASH"
# Try to get the hash from the existing base image
# First, authenticate to Artifact Registry
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
# Try to pull the base image and get its label
BASE_IMAGE="us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest"
EXISTING_HASH=""
if docker pull $BASE_IMAGE 2>/dev/null; then
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
echo "Existing base image hash: $EXISTING_HASH"
else
echo "⚠️ No existing base image found"
fi
# Compare hashes
if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then
echo "🔄 Base image needs rebuild (poetry.lock changed)"
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
@@ -200,29 +437,27 @@ jobs:
echo "✅ Base image is up to date"
echo "needs_rebuild=false" >> $GITHUB_OUTPUT
fi
- name: Rebuild base image (if needed)
if: steps.check-base.outputs.needs_rebuild == 'true'
run: |
echo "🏗️ Rebuilding base image with updated dependencies..."
# Submit base image build and wait for completion
BASE_BUILD_ID=$(gcloud builds submit \
--config=cloudbuild-base.yaml \
--project=nomadkaraoke \
--format='value(id)' \
--async \
.)
echo "Base Build ID: $BASE_BUILD_ID"
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BASE_BUILD_ID?project=nomadkaraoke"
# Poll for build completion
echo "Waiting for base image build to complete..."
while true; do
BUILD_STATUS=$(gcloud builds describe $BASE_BUILD_ID --project=nomadkaraoke --format='value(status)')
echo "Base build status: $BUILD_STATUS"
if [ "$BUILD_STATUS" = "SUCCESS" ]; then
echo "✅ Base image build completed successfully"
break
@@ -239,32 +474,30 @@ jobs:
gcloud builds log $BASE_BUILD_ID --project=nomadkaraoke 2>&1 || true
exit 1
fi
sleep 30
done
- name: Trigger Cloud Build
run: |
echo "🚀 Triggering Cloud Build for deployment..."
echo "Commit: ${{ github.sha }}"
# Submit build and wait for completion
BUILD_ID=$(gcloud builds submit \
--config=cloudbuild.yaml \
--project=nomadkaraoke \
--format='value(id)' \
--async \
.)
echo "Build ID: $BUILD_ID"
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke"
# Poll for build completion
echo "Waiting for build to complete..."
while true; do
BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)')
echo "Build status: $BUILD_STATUS"
if [ "$BUILD_STATUS" = "SUCCESS" ]; then
echo "✅ Build completed successfully"
break
@@ -272,7 +505,6 @@ jobs:
echo "❌ Build failed with status: $BUILD_STATUS"
echo ""
echo "==================== BUILD LOGS ===================="
# Use Cloud Logging to get build logs (works with roles/logging.viewer)
gcloud logging read "resource.type=build AND resource.labels.build_id=$BUILD_ID" \
--project=nomadkaraoke \
--format="value(textPayload)" \
@@ -280,30 +512,24 @@ jobs:
--order=asc || echo "Failed to retrieve logs from Cloud Logging"
echo "===================================================="
echo ""
# Also try the direct logs command as fallback
gcloud builds log $BUILD_ID --project=nomadkaraoke 2>&1 || true
exit 1
fi
sleep 30
done
- name: Verify deployment
run: |
SERVICE_URL=$(gcloud run services describe karaoke-backend \
--region us-central1 \
--format 'value(status.url)')
echo "Service URL: $SERVICE_URL"
# Wait a few seconds for the service to be ready
sleep 10
# Test health endpoint
curl -f "$SERVICE_URL/api/health" || exit 1
echo "✅ Deployment verified - service is healthy"
- name: Summary
if: success()
run: |
-187
View File
@@ -1,187 +0,0 @@
name: Test and Publish
on: [push, pull_request, workflow_dispatch]
# Add permissions for GitHub Pages deployment and git operations
permissions:
contents: write # Changed from 'read' to 'write' to allow creating git tags
pages: write
id-token: write
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
concurrency:
group: "pages"
cancel-in-progress: false
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.13']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Install dependencies
run: |
poetry install --with dev
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
- name: Install spaCy language models
run: |
poetry run python -m spacy download en_core_web_sm
- name: Run unit tests
run: |
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70
- name: Upload coverage reports
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: unittests
name: codecov-umbrella
- name: Run integration tests
run: |
poetry run pytest tests/integration/ -v
- name: Build package
run: |
poetry build
- name: Free Disk Space (GHA Ubuntu Runner)
uses: jlumbroso/free-disk-space@main
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
haskell: true
dotnet: false
large-packages: false
docker-images: false
swap-storage: false
- name: Test package installation
run: |
pip install dist/*.whl
python -m karaoke_gen.utils.gen_cli --help
- if: github.ref == 'refs/heads/main' && github.event_name == 'push'
name: Extract version
id: extract_version
run: |
VERSION=$(poetry version --short)
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "📋 Extracted version: $VERSION"
- if: github.ref == 'refs/heads/main' && github.event_name == 'push'
name: Check if tag exists
id: check_tag
run: |
VERSION="${{ steps.extract_version.outputs.version }}"
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
echo "tag_exists=true" >> $GITHUB_OUTPUT
echo "🏷️ Tag v$VERSION already exists, skipping release"
else
echo "tag_exists=false" >> $GITHUB_OUTPUT
echo "✨ Tag v$VERSION does not exist, proceeding with release"
fi
- if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.check_tag.outputs.tag_exists == 'false'
name: Publish to PyPI
run: |
echo "📦 Publishing the following files to PyPI:"
ls -la dist/
echo "🚀 Publishing to PyPI..."
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
poetry publish
- if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.check_tag.outputs.tag_exists == 'false'
name: Create Git Tag
run: |
git config user.name github-actions
git config user.email github-actions@github.com
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}"
- if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.check_tag.outputs.tag_exists == 'false'
name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
tag_name: "v${{ steps.extract_version.outputs.version }}"
name: "v${{ steps.extract_version.outputs.version }}"
body: |
## Changes in v${{ steps.extract_version.outputs.version }}
This release was automatically created from the main branch.
### Installation
```bash
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
```
### Assets
- Python package published to PyPI
- Frontend deployed to GitHub Pages at gen.nomadkaraoke.com
For detailed changes, see the commit history.
draft: false
prerelease: false
files: |
dist/*.whl
dist/*.tar.gz
# Deploy frontend to GitHub Pages
deploy-frontend:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Prepare frontend files
run: |
echo "📁 Preparing frontend files for deployment..."
mkdir -p _site
cp -r frontend/* _site/
echo "✅ Frontend files copied to _site/"
ls -la _site/
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: '_site'
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
+12 -1
View File
@@ -1057,12 +1057,23 @@ async def mark_uploads_complete(
# URL-based Job Creation - For YouTube and other online video URLs
# ============================================================================
def _validate_url(url: str) -> bool:
def _validate_url(url: Optional[str]) -> bool:
"""
Validate that a URL is a supported video/audio URL.
Supports YouTube, Vimeo, SoundCloud, and other platforms supported by yt-dlp.
Args:
url: The URL to validate. Can be None or non-string.
Returns:
True if valid URL, False otherwise.
"""
# Handle None and non-string inputs safely
if url is None or not isinstance(url, str):
return False
# Handle empty string
if not url:
return False
+83 -79
View File
@@ -19,7 +19,7 @@ from backend.models.job import JobStatus
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.config import get_settings
from backend.workers.worker_logging import create_job_logger, setup_job_logging
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
# Import from karaoke_gen package
from karaoke_gen.audio_processor import AudioProcessor
@@ -241,84 +241,88 @@ async def process_audio_separation(job_id: str) -> bool:
job_log.info(f"Created temp directory: {temp_dir}")
try:
job_log.info(f"Starting audio separation for {job.artist} - {job.title}")
logger.info(f"Starting audio separation for job {job_id}")
# Ensure AUDIO_SEPARATOR_API_URL is set
api_url = os.environ.get("AUDIO_SEPARATOR_API_URL")
if not api_url:
raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. "
"Cannot perform audio separation without remote API access.")
job_log.info(f"Audio separator API: {api_url}")
# Download audio file from GCS or URL
job_log.info("Downloading audio file...")
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager)
if not audio_path:
raise Exception("Failed to download audio file")
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
# Update progress using state_data (don't change status during parallel processing)
# The status is managed at a higher level - workers just track their progress
job_manager.update_state_data(job_id, 'audio_progress', {
'stage': 'separating_stage1',
'progress': 10,
'message': 'Starting audio separation (Stage 1: Clean instrumental)'
})
# Create AudioProcessor instance (reuses karaoke_gen code)
# Use model configuration from job if provided, otherwise use defaults
job_log.info("Creating AudioProcessor instance...")
if job.clean_instrumental_model:
job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}")
if job.backing_vocals_models:
job_log.info(f" Using backing vocals models: {job.backing_vocals_models}")
if job.other_stems_models:
job_log.info(f" Using other stems models: {job.other_stems_models}")
audio_processor = create_audio_processor(
temp_dir,
clean_instrumental_model=job.clean_instrumental_model,
backing_vocals_models=job.backing_vocals_models,
other_stems_models=job.other_stems_models
)
# Format artist-title for file naming (matches CLI behavior)
artist_title = f"{job.artist} - {job.title}"
# Run audio separation (calls Modal API internally)
# This returns a dict with paths to all separated stems
job_log.info("Starting audio separation (this may take 5-10 minutes)...")
job_log.info(" Stage 1: Clean instrumental separation (MDX models)")
job_log.info(" Stage 2: Backing vocals separation (Demucs model)")
logger.info(f"Job {job_id}: Calling audio_processor.process_audio_separation()")
separation_result = audio_processor.process_audio_separation(
audio_file=audio_path,
artist_title=artist_title,
track_output_dir=temp_dir
)
job_log.info("Audio separation complete!")
job_log.info(f" Generated {len(separation_result)} stem files")
logger.info(f"Job {job_id}: Audio separation complete, organizing results")
# Update progress using state_data (don't change status during parallel processing)
job_manager.update_state_data(job_id, 'audio_progress', {
'stage': 'audio_complete',
'progress': 45,
'message': 'Audio separation complete, uploading stems'
})
# Upload all stems to GCS
await upload_separation_results(job_id, separation_result, storage, job_manager)
logger.info(f"Job {job_id}: All stems uploaded successfully")
# Mark audio processing complete
# This will check if lyrics are also complete and transition to next stage if so
job_manager.mark_audio_complete(job_id)
return True
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
# This ensures logs from third-party libraries (karaoke_gen.audio_processor) are
# only captured by this job's handler, not handlers from other concurrent jobs
with job_logging_context(job_id):
job_log.info(f"Starting audio separation for {job.artist} - {job.title}")
logger.info(f"Starting audio separation for job {job_id}")
# Ensure AUDIO_SEPARATOR_API_URL is set
api_url = os.environ.get("AUDIO_SEPARATOR_API_URL")
if not api_url:
raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. "
"Cannot perform audio separation without remote API access.")
job_log.info(f"Audio separator API: {api_url}")
# Download audio file from GCS or URL
job_log.info("Downloading audio file...")
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager)
if not audio_path:
raise Exception("Failed to download audio file")
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
# Update progress using state_data (don't change status during parallel processing)
# The status is managed at a higher level - workers just track their progress
job_manager.update_state_data(job_id, 'audio_progress', {
'stage': 'separating_stage1',
'progress': 10,
'message': 'Starting audio separation (Stage 1: Clean instrumental)'
})
# Create AudioProcessor instance (reuses karaoke_gen code)
# Use model configuration from job if provided, otherwise use defaults
job_log.info("Creating AudioProcessor instance...")
if job.clean_instrumental_model:
job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}")
if job.backing_vocals_models:
job_log.info(f" Using backing vocals models: {job.backing_vocals_models}")
if job.other_stems_models:
job_log.info(f" Using other stems models: {job.other_stems_models}")
audio_processor = create_audio_processor(
temp_dir,
clean_instrumental_model=job.clean_instrumental_model,
backing_vocals_models=job.backing_vocals_models,
other_stems_models=job.other_stems_models
)
# Format artist-title for file naming (matches CLI behavior)
artist_title = f"{job.artist} - {job.title}"
# Run audio separation (calls Modal API internally)
# This returns a dict with paths to all separated stems
job_log.info("Starting audio separation (this may take 5-10 minutes)...")
job_log.info(" Stage 1: Clean instrumental separation (MDX models)")
job_log.info(" Stage 2: Backing vocals separation (Demucs model)")
logger.info(f"Job {job_id}: Calling audio_processor.process_audio_separation()")
separation_result = audio_processor.process_audio_separation(
audio_file=audio_path,
artist_title=artist_title,
track_output_dir=temp_dir
)
job_log.info("Audio separation complete!")
job_log.info(f" Generated {len(separation_result)} stem files")
logger.info(f"Job {job_id}: Audio separation complete, organizing results")
# Update progress using state_data (don't change status during parallel processing)
job_manager.update_state_data(job_id, 'audio_progress', {
'stage': 'audio_complete',
'progress': 45,
'message': 'Audio separation complete, uploading stems'
})
# Upload all stems to GCS
await upload_separation_results(job_id, separation_result, storage, job_manager)
logger.info(f"Job {job_id}: All stems uploaded successfully")
# Mark audio processing complete
# This will check if lyrics are also complete and transition to next stage if so
job_manager.mark_audio_complete(job_id)
return True
except Exception as e:
logger.error(f"Job {job_id}: Audio separation failed: {e}", exc_info=True)
+157 -153
View File
@@ -26,7 +26,7 @@ from backend.models.job import JobStatus
from karaoke_gen.utils import sanitize_filename
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.workers.worker_logging import create_job_logger, setup_job_logging
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
from backend.workers.style_helper import load_style_config
# Import from karaoke_gen package
@@ -127,158 +127,162 @@ async def process_lyrics_transcription(job_id: str) -> bool:
temp_dir = None
try:
job = job_manager.get_job(job_id)
if not job:
logger.error(f"Job {job_id} not found")
job_log.error(f"Job {job_id} not found in Firestore!")
return False
# Create temporary working directory
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_lyrics_{job_id}_")
job_log.info(f"Created temp directory: {temp_dir}")
job_log.info(f"Starting lyrics transcription for {job.artist} - {job.title}")
job_log.info(f"Log capture enabled for: {', '.join(LYRICS_WORKER_LOGGERS)}")
logger.info(f"Starting lyrics transcription for job {job_id}")
# Log environment configuration
job_log.info("Checking API configuration...")
if os.environ.get("GENIUS_API_TOKEN"):
job_log.info(" Genius API: configured")
else:
job_log.warning(" Genius API: NOT configured")
if os.environ.get("SPOTIFY_COOKIE_SP_DC"):
job_log.info(" Spotify: configured")
else:
job_log.warning(" Spotify: NOT configured")
if os.environ.get("RAPIDAPI_KEY"):
job_log.info(" Musixmatch (RapidAPI): configured")
else:
job_log.warning(" Musixmatch (RapidAPI): NOT configured")
# Ensure required environment variables are set
if not os.environ.get("AUDIOSHAKE_API_TOKEN"):
job_log.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
logger.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
# Download audio file from GCS (waits for audio worker if URL job)
job_log.info("Downloading audio file from GCS...")
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager)
if not audio_path:
raise Exception("Failed to download audio file")
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
# Update progress using state_data (don't change status during parallel processing)
# The status is managed at a higher level - workers just track their progress
job_manager.update_state_data(job_id, 'lyrics_progress', {
'stage': 'transcribing',
'progress': 10,
'message': 'Starting lyrics transcription via AudioShake'
})
# Load style configuration (downloads assets from GCS if available)
# This is needed for max_line_length and other style settings in LyricsTranscriber
job_log.info("Loading style configuration for lyrics processing...")
style_config = await load_style_config(job, storage, temp_dir)
style_params_json_path = style_config.get_style_params_path() if style_config.has_custom_styles() else None
if style_params_json_path:
job_log.info(f"Using custom style params: {style_params_json_path}")
# Log the contents of the style params for debugging
try:
with open(style_params_json_path, 'r') as f:
style_content = json.load(f)
job_log.info(f"Style params sections: {list(style_content.keys())}")
if 'karaoke' in style_content:
karaoke_style = style_content['karaoke']
job_log.info(f" karaoke.background_image: {karaoke_style.get('background_image', 'NOT SET')}")
job_log.info(f" karaoke.font_path: {karaoke_style.get('font_path', 'NOT SET')}")
except Exception as e:
job_log.warning(f"Could not read style params for logging: {e}")
else:
job_log.info("No custom style params found, using defaults")
# Download user-provided lyrics file if available
lyrics_file_path = None
if hasattr(job, 'lyrics_file_gcs_path') and job.lyrics_file_gcs_path:
lyrics_file_path = os.path.join(temp_dir, "user_lyrics.txt")
job_log.info(f"Downloading user-provided lyrics file: {job.lyrics_file_gcs_path}")
storage.download_file(job.lyrics_file_gcs_path, lyrics_file_path)
job_log.info(f"User lyrics file downloaded to: {lyrics_file_path}")
# Get lyrics configuration from job
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
if subtitle_offset != 0:
job_log.info(f"Subtitle offset: {subtitle_offset}ms")
# Create LyricsProcessor instance (reuses karaoke_gen code)
job_log.info("Creating LyricsProcessor instance...")
job_log.info(f" style_params_json: {style_params_json_path}")
job_log.info(f" lyrics_file: {lyrics_file_path}")
job_log.info(f" subtitle_offset_ms: {subtitle_offset}")
lyrics_processor = create_lyrics_processor(
style_params_json=style_params_json_path,
lyrics_file=lyrics_file_path,
subtitle_offset_ms=subtitle_offset
)
# Run transcription + correction
# This will:
# 1. Fetch reference lyrics from Genius/Spotify/Musixmatch/LRCLib
# 2. Transcribe audio via AudioShake API
# 3. Run automatic correction with lyrics_transcriber
# 4. Generate corrections JSON (no interactive review)
# 5. Skip video generation (render_video=False)
# Use lyrics_artist/lyrics_title overrides if provided, else fall back to job artist/title
lyrics_search_artist = getattr(job, 'lyrics_artist', None) or job.artist
lyrics_search_title = getattr(job, 'lyrics_title', None) or job.title
job_log.info("Starting LyricsTranscriber processing...")
job_log.info(f" Artist: {job.artist}")
job_log.info(f" Title: {job.title}")
if lyrics_search_artist != job.artist:
job_log.info(f" Lyrics search artist override: {lyrics_search_artist}")
if lyrics_search_title != job.title:
job_log.info(f" Lyrics search title override: {lyrics_search_title}")
logger.info(f"Job {job_id}: Calling lyrics_processor.transcribe_lyrics()")
# Run transcription in thread pool to avoid blocking the event loop
# (transcribe_lyrics is synchronous and takes 1-2 minutes)
# Note: We pass the original artist/title for file naming, but the LyricsProcessor
# will use lyrics_artist/lyrics_title for the search (set via constructor args)
result = await asyncio.to_thread(
lyrics_processor.transcribe_lyrics,
input_audio_wav=audio_path,
artist=job.artist, # Original artist for file naming
title=job.title, # Original title for file naming
track_output_dir=temp_dir,
lyrics_artist=lyrics_search_artist, # Override for lyrics search
lyrics_title=lyrics_search_title # Override for lyrics search
)
job_log.info("Transcription processing complete")
logger.info(f"Job {job_id}: Transcription complete, uploading results")
# Upload lyrics results to GCS
job_log.info("Uploading lyrics results to GCS...")
await upload_lyrics_results(job_id, temp_dir, result, storage, job_manager, job_log)
job_log.info("All lyrics data uploaded successfully")
logger.info(f"Job {job_id}: All lyrics data uploaded successfully")
# Update progress using state_data (don't change status during parallel processing)
job_manager.update_state_data(job_id, 'lyrics_progress', {
'stage': 'lyrics_complete',
'progress': 45,
'message': 'Lyrics transcription complete'
})
# Mark lyrics processing complete
# This will check if audio is also complete and transition to next stage if so
job_log.info("Lyrics worker complete, checking if audio is also done...")
job_manager.mark_lyrics_complete(job_id)
return True
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
# This ensures logs from third-party libraries (lyrics_transcriber) are only captured
# by this job's handler, not handlers from other concurrent jobs
with job_logging_context(job_id):
job = job_manager.get_job(job_id)
if not job:
logger.error(f"Job {job_id} not found")
job_log.error(f"Job {job_id} not found in Firestore!")
return False
# Create temporary working directory
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_lyrics_{job_id}_")
job_log.info(f"Created temp directory: {temp_dir}")
job_log.info(f"Starting lyrics transcription for {job.artist} - {job.title}")
job_log.info(f"Log capture enabled for: {', '.join(LYRICS_WORKER_LOGGERS)}")
logger.info(f"Starting lyrics transcription for job {job_id}")
# Log environment configuration
job_log.info("Checking API configuration...")
if os.environ.get("GENIUS_API_TOKEN"):
job_log.info(" Genius API: configured")
else:
job_log.warning(" Genius API: NOT configured")
if os.environ.get("SPOTIFY_COOKIE_SP_DC"):
job_log.info(" Spotify: configured")
else:
job_log.warning(" Spotify: NOT configured")
if os.environ.get("RAPIDAPI_KEY"):
job_log.info(" Musixmatch (RapidAPI): configured")
else:
job_log.warning(" Musixmatch (RapidAPI): NOT configured")
# Ensure required environment variables are set
if not os.environ.get("AUDIOSHAKE_API_TOKEN"):
job_log.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
logger.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
# Download audio file from GCS (waits for audio worker if URL job)
job_log.info("Downloading audio file from GCS...")
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager)
if not audio_path:
raise Exception("Failed to download audio file")
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
# Update progress using state_data (don't change status during parallel processing)
# The status is managed at a higher level - workers just track their progress
job_manager.update_state_data(job_id, 'lyrics_progress', {
'stage': 'transcribing',
'progress': 10,
'message': 'Starting lyrics transcription via AudioShake'
})
# Load style configuration (downloads assets from GCS if available)
# This is needed for max_line_length and other style settings in LyricsTranscriber
job_log.info("Loading style configuration for lyrics processing...")
style_config = await load_style_config(job, storage, temp_dir)
style_params_json_path = style_config.get_style_params_path() if style_config.has_custom_styles() else None
if style_params_json_path:
job_log.info(f"Using custom style params: {style_params_json_path}")
# Log the contents of the style params for debugging
try:
with open(style_params_json_path, 'r') as f:
style_content = json.load(f)
job_log.info(f"Style params sections: {list(style_content.keys())}")
if 'karaoke' in style_content:
karaoke_style = style_content['karaoke']
job_log.info(f" karaoke.background_image: {karaoke_style.get('background_image', 'NOT SET')}")
job_log.info(f" karaoke.font_path: {karaoke_style.get('font_path', 'NOT SET')}")
except Exception as e:
job_log.warning(f"Could not read style params for logging: {e}")
else:
job_log.info("No custom style params found, using defaults")
# Download user-provided lyrics file if available
lyrics_file_path = None
if hasattr(job, 'lyrics_file_gcs_path') and job.lyrics_file_gcs_path:
lyrics_file_path = os.path.join(temp_dir, "user_lyrics.txt")
job_log.info(f"Downloading user-provided lyrics file: {job.lyrics_file_gcs_path}")
storage.download_file(job.lyrics_file_gcs_path, lyrics_file_path)
job_log.info(f"User lyrics file downloaded to: {lyrics_file_path}")
# Get lyrics configuration from job
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
if subtitle_offset != 0:
job_log.info(f"Subtitle offset: {subtitle_offset}ms")
# Create LyricsProcessor instance (reuses karaoke_gen code)
job_log.info("Creating LyricsProcessor instance...")
job_log.info(f" style_params_json: {style_params_json_path}")
job_log.info(f" lyrics_file: {lyrics_file_path}")
job_log.info(f" subtitle_offset_ms: {subtitle_offset}")
lyrics_processor = create_lyrics_processor(
style_params_json=style_params_json_path,
lyrics_file=lyrics_file_path,
subtitle_offset_ms=subtitle_offset
)
# Run transcription + correction
# This will:
# 1. Fetch reference lyrics from Genius/Spotify/Musixmatch/LRCLib
# 2. Transcribe audio via AudioShake API
# 3. Run automatic correction with lyrics_transcriber
# 4. Generate corrections JSON (no interactive review)
# 5. Skip video generation (render_video=False)
# Use lyrics_artist/lyrics_title overrides if provided, else fall back to job artist/title
lyrics_search_artist = getattr(job, 'lyrics_artist', None) or job.artist
lyrics_search_title = getattr(job, 'lyrics_title', None) or job.title
job_log.info("Starting LyricsTranscriber processing...")
job_log.info(f" Artist: {job.artist}")
job_log.info(f" Title: {job.title}")
if lyrics_search_artist != job.artist:
job_log.info(f" Lyrics search artist override: {lyrics_search_artist}")
if lyrics_search_title != job.title:
job_log.info(f" Lyrics search title override: {lyrics_search_title}")
logger.info(f"Job {job_id}: Calling lyrics_processor.transcribe_lyrics()")
# Run transcription in thread pool to avoid blocking the event loop
# (transcribe_lyrics is synchronous and takes 1-2 minutes)
# Note: We pass the original artist/title for file naming, but the LyricsProcessor
# will use lyrics_artist/lyrics_title for the search (set via constructor args)
result = await asyncio.to_thread(
lyrics_processor.transcribe_lyrics,
input_audio_wav=audio_path,
artist=job.artist, # Original artist for file naming
title=job.title, # Original title for file naming
track_output_dir=temp_dir,
lyrics_artist=lyrics_search_artist, # Override for lyrics search
lyrics_title=lyrics_search_title # Override for lyrics search
)
job_log.info("Transcription processing complete")
logger.info(f"Job {job_id}: Transcription complete, uploading results")
# Upload lyrics results to GCS
job_log.info("Uploading lyrics results to GCS...")
await upload_lyrics_results(job_id, temp_dir, result, storage, job_manager, job_log)
job_log.info("All lyrics data uploaded successfully")
logger.info(f"Job {job_id}: All lyrics data uploaded successfully")
# Update progress using state_data (don't change status during parallel processing)
job_manager.update_state_data(job_id, 'lyrics_progress', {
'stage': 'lyrics_complete',
'progress': 45,
'message': 'Lyrics transcription complete'
})
# Mark lyrics processing complete
# This will check if audio is also complete and transition to next stage if so
job_log.info("Lyrics worker complete, checking if audio is also done...")
job_manager.mark_lyrics_complete(job_id)
return True
except Exception as e:
job_log.error(f"Lyrics transcription failed: {str(e)}", exc_info=True)
+189 -185
View File
@@ -26,7 +26,7 @@ from backend.models.job import JobStatus
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.config import get_settings
from backend.workers.worker_logging import create_job_logger, setup_job_logging
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
# Import from lyrics_transcriber (submodule)
from lyrics_transcriber.output.generator import OutputGenerator
@@ -85,194 +85,198 @@ async def process_render_video(job_id: str) -> bool:
logger.info(f"Job {job_id}: Starting video render (post-review)")
try:
# Transition to RENDERING_VIDEO
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.RENDERING_VIDEO,
progress=75,
message="Rendering karaoke video with corrected lyrics"
)
with tempfile.TemporaryDirectory() as temp_dir:
job_log.info(f"Created temp directory: {temp_dir}")
# 1. Download corrected corrections data
corrections_path = os.path.join(temp_dir, "corrections.json")
# Try updated corrections first (from human review), fall back to original
corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json"
corrections_gcs_original = f"jobs/{job_id}/lyrics/corrections.json"
# Get GCS path from file_urls
corrections_url = job.file_urls.get('lyrics', {}).get('corrections_updated')
if not corrections_url:
corrections_url = job.file_urls.get('lyrics', {}).get('corrections')
if not corrections_url:
# Try direct GCS paths
if storage.file_exists(corrections_gcs_updated):
corrections_gcs = corrections_gcs_updated
elif storage.file_exists(corrections_gcs_original):
corrections_gcs = corrections_gcs_original
else:
raise FileNotFoundError(f"No corrections file found for job {job_id}")
else:
# Extract GCS path from URL
corrections_gcs = _extract_gcs_path(corrections_url)
# 2. Load the ORIGINAL corrections (has full structure)
original_corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
original_corrections_path = os.path.join(temp_dir, "corrections_original.json")
job_log.info(f"Downloading original corrections from {original_corrections_gcs}")
logger.info(f"Job {job_id}: Downloading original corrections from {original_corrections_gcs}")
storage.download_file(original_corrections_gcs, original_corrections_path)
with open(original_corrections_path, 'r', encoding='utf-8') as f:
original_data = json.load(f)
# 3. Check if there are updated corrections (from review UI)
# The frontend sends only partial data: {corrections, corrected_segments}
updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json"
if storage.file_exists(updated_corrections_gcs):
job_log.info("Found updated corrections from review, merging")
logger.info(f"Job {job_id}: Found updated corrections, merging")
updated_path = os.path.join(temp_dir, "corrections_updated.json")
storage.download_file(updated_corrections_gcs, updated_path)
with open(updated_path, 'r', encoding='utf-8') as f:
updated_data = json.load(f)
# Merge: update the original with the user's corrections
if 'corrections' in updated_data:
original_data['corrections'] = updated_data['corrections']
if 'corrected_segments' in updated_data:
original_data['corrected_segments'] = updated_data['corrected_segments']
job_log.info("Merged user corrections into original data")
logger.info(f"Job {job_id}: Merged user corrections into original data")
# 4. Convert to CorrectionResult
correction_result = CorrectionResult.from_dict(original_data)
job_log.info(f"Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
logger.info(f"Job {job_id}: Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
# 3. Download audio file
audio_path = os.path.join(temp_dir, "audio.flac")
audio_gcs_path = job.input_media_gcs_path
if not audio_gcs_path:
raise FileNotFoundError(f"No input audio path for job {job_id}")
job_log.info(f"Downloading audio from {audio_gcs_path}")
logger.info(f"Job {job_id}: Downloading audio from {audio_gcs_path}")
storage.download_file(audio_gcs_path, audio_path)
job_log.info(f"Audio downloaded: {os.path.getsize(audio_path)} bytes")
# 4. Get or create styles using the unified style loader
job_log.info("Loading style configuration...")
job_log.info(f" job.style_params_gcs_path: {job.style_params_gcs_path}")
job_log.info(f" job.style_assets: {list(job.style_assets.keys()) if job.style_assets else 'None'}")
styles_path, style_data = load_styles_from_gcs(
style_params_gcs_path=job.style_params_gcs_path,
style_assets=job.style_assets,
temp_dir=temp_dir,
download_func=storage.download_file,
logger=job_log,
)
# 5. Configure OutputGenerator
output_dir = os.path.join(temp_dir, "output")
cache_dir = os.path.join(temp_dir, "cache")
os.makedirs(output_dir, exist_ok=True)
os.makedirs(cache_dir, exist_ok=True)
job_log.info(f"Using styles from: {styles_path}")
# Get subtitle offset from job (user-specified timing adjustment)
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
if subtitle_offset != 0:
job_log.info(f"Applying subtitle offset: {subtitle_offset}ms")
config = OutputConfig(
output_dir=output_dir,
cache_dir=cache_dir,
output_styles_json=styles_path,
render_video=True,
generate_cdg=False, # CDG optional, generated separately
generate_plain_text=True,
generate_lrc=True,
video_resolution="4k",
subtitle_offset_ms=subtitle_offset
)
job_log.info(f"OutputConfig: output_styles_json={config.output_styles_json}, render_video={config.render_video}")
output_generator = OutputGenerator(config, logger)
# 6. Generate outputs (video, LRC, ASS, etc.)
output_prefix = f"{job.artist or 'Unknown'} - {job.title or 'Unknown'}"
job_log.info(f"Generating outputs with prefix '{output_prefix}'")
logger.info(f"Job {job_id}: Generating outputs with prefix '{output_prefix}'")
outputs = output_generator.generate_outputs(
transcription_corrected=correction_result,
lyrics_results={}, # Reference lyrics already in correction_result
output_prefix=output_prefix,
audio_filepath=audio_path,
artist=job.artist,
title=job.title
)
# 7. Upload video to GCS
if outputs.video and os.path.exists(outputs.video):
video_size = os.path.getsize(outputs.video)
job_log.info(f"Video generated: {video_size} bytes")
video_gcs_path = f"jobs/{job_id}/videos/with_vocals.mkv"
video_url = storage.upload_file(outputs.video, video_gcs_path)
job_manager.update_file_url(job_id, 'videos', 'with_vocals', video_url)
job_log.info(f"Uploaded with_vocals.mkv to GCS")
logger.info(f"Job {job_id}: Uploaded with_vocals.mkv ({video_size} bytes)")
else:
job_log.error("Video generation failed - no output file produced!")
raise Exception("Video generation failed - no output file produced")
# 8. Upload LRC file
if outputs.lrc and os.path.exists(outputs.lrc):
lrc_gcs_path = f"jobs/{job_id}/lyrics/karaoke.lrc"
lrc_url = storage.upload_file(outputs.lrc, lrc_gcs_path)
job_manager.update_file_url(job_id, 'lyrics', 'lrc', lrc_url)
job_log.info("Uploaded karaoke.lrc")
logger.info(f"Job {job_id}: Uploaded karaoke.lrc")
# 9. Upload ASS subtitle file
if outputs.ass and os.path.exists(outputs.ass):
ass_gcs_path = f"jobs/{job_id}/lyrics/karaoke.ass"
ass_url = storage.upload_file(outputs.ass, ass_gcs_path)
job_manager.update_file_url(job_id, 'lyrics', 'ass', ass_url)
job_log.info("Uploaded karaoke.ass")
logger.info(f"Job {job_id}: Uploaded karaoke.ass")
# 10. Upload corrected text files
if outputs.corrected_txt and os.path.exists(outputs.corrected_txt):
txt_gcs_path = f"jobs/{job_id}/lyrics/corrected.txt"
txt_url = storage.upload_file(outputs.corrected_txt, txt_gcs_path)
job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', txt_url)
job_log.info("Uploaded corrected.txt")
logger.info(f"Job {job_id}: Uploaded corrected.txt")
# 11. Transition to awaiting instrumental selection
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
# This ensures logs from third-party libraries (lyrics_transcriber.output) are only
# captured by this job's handler, not handlers from other concurrent jobs
with job_logging_context(job_id):
# Transition to RENDERING_VIDEO
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.AWAITING_INSTRUMENTAL_SELECTION,
progress=80,
message="Video rendered - select your instrumental"
new_status=JobStatus.RENDERING_VIDEO,
progress=75,
message="Rendering karaoke video with corrected lyrics"
)
job_log.info("=== RENDER VIDEO WORKER COMPLETE ===")
logger.info(f"Job {job_id}: Video render complete, awaiting instrumental selection")
return True
with tempfile.TemporaryDirectory() as temp_dir:
job_log.info(f"Created temp directory: {temp_dir}")
# 1. Download corrected corrections data
corrections_path = os.path.join(temp_dir, "corrections.json")
# Try updated corrections first (from human review), fall back to original
corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json"
corrections_gcs_original = f"jobs/{job_id}/lyrics/corrections.json"
# Get GCS path from file_urls
corrections_url = job.file_urls.get('lyrics', {}).get('corrections_updated')
if not corrections_url:
corrections_url = job.file_urls.get('lyrics', {}).get('corrections')
if not corrections_url:
# Try direct GCS paths
if storage.file_exists(corrections_gcs_updated):
corrections_gcs = corrections_gcs_updated
elif storage.file_exists(corrections_gcs_original):
corrections_gcs = corrections_gcs_original
else:
raise FileNotFoundError(f"No corrections file found for job {job_id}")
else:
# Extract GCS path from URL
corrections_gcs = _extract_gcs_path(corrections_url)
# 2. Load the ORIGINAL corrections (has full structure)
original_corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
original_corrections_path = os.path.join(temp_dir, "corrections_original.json")
job_log.info(f"Downloading original corrections from {original_corrections_gcs}")
logger.info(f"Job {job_id}: Downloading original corrections from {original_corrections_gcs}")
storage.download_file(original_corrections_gcs, original_corrections_path)
with open(original_corrections_path, 'r', encoding='utf-8') as f:
original_data = json.load(f)
# 3. Check if there are updated corrections (from review UI)
# The frontend sends only partial data: {corrections, corrected_segments}
updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json"
if storage.file_exists(updated_corrections_gcs):
job_log.info("Found updated corrections from review, merging")
logger.info(f"Job {job_id}: Found updated corrections, merging")
updated_path = os.path.join(temp_dir, "corrections_updated.json")
storage.download_file(updated_corrections_gcs, updated_path)
with open(updated_path, 'r', encoding='utf-8') as f:
updated_data = json.load(f)
# Merge: update the original with the user's corrections
if 'corrections' in updated_data:
original_data['corrections'] = updated_data['corrections']
if 'corrected_segments' in updated_data:
original_data['corrected_segments'] = updated_data['corrected_segments']
job_log.info("Merged user corrections into original data")
logger.info(f"Job {job_id}: Merged user corrections into original data")
# 4. Convert to CorrectionResult
correction_result = CorrectionResult.from_dict(original_data)
job_log.info(f"Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
logger.info(f"Job {job_id}: Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
# 3. Download audio file
audio_path = os.path.join(temp_dir, "audio.flac")
audio_gcs_path = job.input_media_gcs_path
if not audio_gcs_path:
raise FileNotFoundError(f"No input audio path for job {job_id}")
job_log.info(f"Downloading audio from {audio_gcs_path}")
logger.info(f"Job {job_id}: Downloading audio from {audio_gcs_path}")
storage.download_file(audio_gcs_path, audio_path)
job_log.info(f"Audio downloaded: {os.path.getsize(audio_path)} bytes")
# 4. Get or create styles using the unified style loader
job_log.info("Loading style configuration...")
job_log.info(f" job.style_params_gcs_path: {job.style_params_gcs_path}")
job_log.info(f" job.style_assets: {list(job.style_assets.keys()) if job.style_assets else 'None'}")
styles_path, style_data = load_styles_from_gcs(
style_params_gcs_path=job.style_params_gcs_path,
style_assets=job.style_assets,
temp_dir=temp_dir,
download_func=storage.download_file,
logger=job_log,
)
# 5. Configure OutputGenerator
output_dir = os.path.join(temp_dir, "output")
cache_dir = os.path.join(temp_dir, "cache")
os.makedirs(output_dir, exist_ok=True)
os.makedirs(cache_dir, exist_ok=True)
job_log.info(f"Using styles from: {styles_path}")
# Get subtitle offset from job (user-specified timing adjustment)
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
if subtitle_offset != 0:
job_log.info(f"Applying subtitle offset: {subtitle_offset}ms")
config = OutputConfig(
output_dir=output_dir,
cache_dir=cache_dir,
output_styles_json=styles_path,
render_video=True,
generate_cdg=False, # CDG optional, generated separately
generate_plain_text=True,
generate_lrc=True,
video_resolution="4k",
subtitle_offset_ms=subtitle_offset
)
job_log.info(f"OutputConfig: output_styles_json={config.output_styles_json}, render_video={config.render_video}")
output_generator = OutputGenerator(config, logger)
# 6. Generate outputs (video, LRC, ASS, etc.)
output_prefix = f"{job.artist or 'Unknown'} - {job.title or 'Unknown'}"
job_log.info(f"Generating outputs with prefix '{output_prefix}'")
logger.info(f"Job {job_id}: Generating outputs with prefix '{output_prefix}'")
outputs = output_generator.generate_outputs(
transcription_corrected=correction_result,
lyrics_results={}, # Reference lyrics already in correction_result
output_prefix=output_prefix,
audio_filepath=audio_path,
artist=job.artist,
title=job.title
)
# 7. Upload video to GCS
if outputs.video and os.path.exists(outputs.video):
video_size = os.path.getsize(outputs.video)
job_log.info(f"Video generated: {video_size} bytes")
video_gcs_path = f"jobs/{job_id}/videos/with_vocals.mkv"
video_url = storage.upload_file(outputs.video, video_gcs_path)
job_manager.update_file_url(job_id, 'videos', 'with_vocals', video_url)
job_log.info(f"Uploaded with_vocals.mkv to GCS")
logger.info(f"Job {job_id}: Uploaded with_vocals.mkv ({video_size} bytes)")
else:
job_log.error("Video generation failed - no output file produced!")
raise Exception("Video generation failed - no output file produced")
# 8. Upload LRC file
if outputs.lrc and os.path.exists(outputs.lrc):
lrc_gcs_path = f"jobs/{job_id}/lyrics/karaoke.lrc"
lrc_url = storage.upload_file(outputs.lrc, lrc_gcs_path)
job_manager.update_file_url(job_id, 'lyrics', 'lrc', lrc_url)
job_log.info("Uploaded karaoke.lrc")
logger.info(f"Job {job_id}: Uploaded karaoke.lrc")
# 9. Upload ASS subtitle file
if outputs.ass and os.path.exists(outputs.ass):
ass_gcs_path = f"jobs/{job_id}/lyrics/karaoke.ass"
ass_url = storage.upload_file(outputs.ass, ass_gcs_path)
job_manager.update_file_url(job_id, 'lyrics', 'ass', ass_url)
job_log.info("Uploaded karaoke.ass")
logger.info(f"Job {job_id}: Uploaded karaoke.ass")
# 10. Upload corrected text files
if outputs.corrected_txt and os.path.exists(outputs.corrected_txt):
txt_gcs_path = f"jobs/{job_id}/lyrics/corrected.txt"
txt_url = storage.upload_file(outputs.corrected_txt, txt_gcs_path)
job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', txt_url)
job_log.info("Uploaded corrected.txt")
logger.info(f"Job {job_id}: Uploaded corrected.txt")
# 11. Transition to awaiting instrumental selection
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.AWAITING_INSTRUMENTAL_SELECTION,
progress=80,
message="Video rendered - select your instrumental"
)
job_log.info("=== RENDER VIDEO WORKER COMPLETE ===")
logger.info(f"Job {job_id}: Video render complete, awaiting instrumental selection")
return True
except Exception as e:
job_log.error(f"Video render failed: {e}")
+93 -89
View File
@@ -35,7 +35,7 @@ from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.config import get_settings
from backend.workers.style_helper import load_style_config, StyleConfig
from backend.workers.worker_logging import create_job_logger, setup_job_logging
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
# Import from karaoke_gen package
from karaoke_gen.video_generator import VideoGenerator
@@ -88,94 +88,98 @@ async def generate_screens(job_id: str) -> bool:
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_screens_{job_id}_")
try:
job_log.info(f"Starting screen generation for {job.artist} - {job.title}")
logger.info(f"Starting screen generation for job {job_id}")
# Transition to GENERATING_SCREENS state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.GENERATING_SCREENS,
progress=50,
message="Generating title and end screens"
)
# Log style assets info
style_assets = getattr(job, 'style_assets', {}) or {}
job_log.info(f"Style assets from job: {list(style_assets.keys()) if style_assets else 'None'}")
if style_assets:
for key, path in style_assets.items():
job_log.info(f" {key}: {path}")
# Load style configuration (downloads assets from GCS if available)
job_log.info("Loading style configuration from GCS...")
style_config = await load_style_config(job, storage, temp_dir)
if style_config.has_custom_styles():
job_log.info("Using CUSTOM style configuration")
logger.info(f"Job {job_id}: Using custom style configuration")
else:
job_log.warning("Using DEFAULT style configuration (no custom styles found)")
logger.info(f"Job {job_id}: Using default style configuration")
# Initialize video generator
video_generator = _create_video_generator(temp_dir)
# Generate title screen with style config
job_log.info("Generating title screen...")
title_screen_path = await _generate_title_screen(
job_id=job_id,
job=job,
video_generator=video_generator,
style_config=style_config,
temp_dir=temp_dir,
job_log=job_log
)
if not title_screen_path:
raise Exception("Title screen generation failed")
job_log.info(f"Title screen generated: {title_screen_path}")
# Generate end screen with style config
job_log.info("Generating end screen...")
end_screen_path = await _generate_end_screen(
job_id=job_id,
job=job,
video_generator=video_generator,
style_config=style_config,
temp_dir=temp_dir,
job_log=job_log
)
if not end_screen_path:
raise Exception("End screen generation failed")
job_log.info(f"End screen generated: {end_screen_path}")
# Upload screens to GCS
await _upload_screens(
job_id=job_id,
job_manager=job_manager,
storage=storage,
title_screen_path=title_screen_path,
end_screen_path=end_screen_path
)
# Apply countdown padding if needed
# (This is handled automatically by checking lyrics metadata)
await _apply_countdown_padding_if_needed(job_id, job_manager, job)
# Transition to AWAITING_REVIEW
# Human must review lyrics before video can be rendered
logger.info(f"Job {job_id}: Screens generated, awaiting lyrics review")
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.AWAITING_REVIEW,
progress=55,
message="Ready for lyrics review. Please review and correct lyrics."
)
# TODO: Send notification to user that selection is needed
# await notify_user(job_id, "instrumental_selection_ready")
return True
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
# This ensures logs from third-party libraries (karaoke_gen.video_generator) are only
# captured by this job's handler, not handlers from other concurrent jobs
with job_logging_context(job_id):
job_log.info(f"Starting screen generation for {job.artist} - {job.title}")
logger.info(f"Starting screen generation for job {job_id}")
# Transition to GENERATING_SCREENS state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.GENERATING_SCREENS,
progress=50,
message="Generating title and end screens"
)
# Log style assets info
style_assets = getattr(job, 'style_assets', {}) or {}
job_log.info(f"Style assets from job: {list(style_assets.keys()) if style_assets else 'None'}")
if style_assets:
for key, path in style_assets.items():
job_log.info(f" {key}: {path}")
# Load style configuration (downloads assets from GCS if available)
job_log.info("Loading style configuration from GCS...")
style_config = await load_style_config(job, storage, temp_dir)
if style_config.has_custom_styles():
job_log.info("Using CUSTOM style configuration")
logger.info(f"Job {job_id}: Using custom style configuration")
else:
job_log.warning("Using DEFAULT style configuration (no custom styles found)")
logger.info(f"Job {job_id}: Using default style configuration")
# Initialize video generator
video_generator = _create_video_generator(temp_dir)
# Generate title screen with style config
job_log.info("Generating title screen...")
title_screen_path = await _generate_title_screen(
job_id=job_id,
job=job,
video_generator=video_generator,
style_config=style_config,
temp_dir=temp_dir,
job_log=job_log
)
if not title_screen_path:
raise Exception("Title screen generation failed")
job_log.info(f"Title screen generated: {title_screen_path}")
# Generate end screen with style config
job_log.info("Generating end screen...")
end_screen_path = await _generate_end_screen(
job_id=job_id,
job=job,
video_generator=video_generator,
style_config=style_config,
temp_dir=temp_dir,
job_log=job_log
)
if not end_screen_path:
raise Exception("End screen generation failed")
job_log.info(f"End screen generated: {end_screen_path}")
# Upload screens to GCS
await _upload_screens(
job_id=job_id,
job_manager=job_manager,
storage=storage,
title_screen_path=title_screen_path,
end_screen_path=end_screen_path
)
# Apply countdown padding if needed
# (This is handled automatically by checking lyrics metadata)
await _apply_countdown_padding_if_needed(job_id, job_manager, job)
# Transition to AWAITING_REVIEW
# Human must review lyrics before video can be rendered
logger.info(f"Job {job_id}: Screens generated, awaiting lyrics review")
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.AWAITING_REVIEW,
progress=55,
message="Ready for lyrics review. Please review and correct lyrics."
)
# TODO: Send notification to user that selection is needed
# await notify_user(job_id, "instrumental_selection_ready")
return True
except Exception as e:
logger.error(f"Job {job_id}: Screen generation failed: {e}", exc_info=True)
+161 -157
View File
@@ -31,7 +31,7 @@ from backend.services.rclone_service import get_rclone_service
from backend.services.youtube_service import get_youtube_service
from backend.config import get_settings
from backend.workers.style_helper import load_style_config
from backend.workers.worker_logging import create_job_logger, setup_job_logging
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
# Import from karaoke_gen package - reuse existing implementation
from karaoke_gen.karaoke_finalise.karaoke_finalise import KaraokeFinalise
@@ -105,162 +105,166 @@ async def generate_video(job_id: str) -> bool:
job_log.warning("YouTube credentials not available - upload will be skipped")
try:
job_log.info(f"Starting video finalization for {job.artist} - {job.title}")
logger.info(f"Starting video generation for job {job_id}")
# Transition to GENERATING_VIDEO state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.GENERATING_VIDEO,
progress=70,
message="Preparing files for video generation"
)
# Download and set up files in the format KaraokeFinalise expects
base_name = f"{job.artist} - {job.title}"
job_log.info("Downloading files from GCS (this may take a few minutes for large files)...")
await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log)
job_log.info("All files downloaded successfully")
# Load style config for CDG styles
job_log.info("Loading CDG style configuration...")
style_config = await load_style_config(job, storage, temp_dir)
cdg_styles = style_config.get_cdg_styles()
if cdg_styles:
job_log.info("CDG styles loaded from custom configuration")
else:
job_log.info("Using default CDG styles")
# Change to working directory (KaraokeFinalise works in cwd)
os.chdir(temp_dir)
# Get the selected instrumental file path
# Batch 3: If user provided existing instrumental, use that; otherwise use AI-separated
instrumental_selection = job.state_data['instrumental_selection']
existing_instrumental_path = getattr(job, 'existing_instrumental_gcs_path', None)
if existing_instrumental_path:
# User provided existing instrumental
ext = Path(existing_instrumental_path).suffix.lower()
instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental User){ext}")
instrumental_source = "user-provided"
else:
# Use AI-separated instrumental
instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing"
instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac")
instrumental_source = instrumental_selection
# Transition to encoding state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.ENCODING,
progress=75,
message="Encoding videos (15-20 min)"
)
# Log finalization parameters
job_log.info("Creating KaraokeFinalise with parameters:")
job_log.info(f" enable_cdg: {getattr(job, 'enable_cdg', False)}")
job_log.info(f" enable_txt: {getattr(job, 'enable_txt', False)}")
job_log.info(f" brand_prefix: {getattr(job, 'brand_prefix', None)}")
job_log.info(f" discord_webhook: {'configured' if getattr(job, 'discord_webhook_url', None) else 'not configured'}")
job_log.info(f" instrumental source: {instrumental_source}")
if existing_instrumental_path:
job_log.info(f" using user-provided instrumental (selection was: {instrumental_selection})")
# Create KaraokeFinalise with ALL the parameters from the job
# This reuses all existing functionality!
# Set up YouTube description file if template is provided
youtube_desc_path = None
if youtube_credentials and getattr(job, 'youtube_description_template', None):
youtube_desc_path = os.path.join(temp_dir, "youtube_description.txt")
with open(youtube_desc_path, 'w') as f:
f.write(job.youtube_description_template)
job_log.info("YouTube description template written to temp file")
finalise = KaraokeFinalise(
logger=logger,
log_level=logging.INFO,
dry_run=False,
instrumental_format="flac",
# CDG/TXT generation
enable_cdg=getattr(job, 'enable_cdg', False),
enable_txt=getattr(job, 'enable_txt', False),
cdg_styles=cdg_styles,
# Brand code and organization (server-side mode uses rclone)
brand_prefix=getattr(job, 'brand_prefix', None),
organised_dir=None, # Not used in cloud - files stay in GCS
organised_dir_rclone_root=getattr(job, 'organised_dir_rclone_root', None),
public_share_dir=None, # Not used in cloud
# Notifications
discord_webhook_url=getattr(job, 'discord_webhook_url', None),
# YouTube upload (server-side with pre-loaded credentials)
youtube_client_secrets_file=None, # Not used with pre-stored credentials
youtube_description_file=youtube_desc_path,
user_youtube_credentials=youtube_credentials, # Pre-loaded from Secret Manager
rclone_destination=None,
email_template_file=None,
# Server-side optimizations
non_interactive=True,
server_side_mode=True,
selected_instrumental_file=instrumental_file,
)
# Call process() - this does ALL the work:
# - Encodes to 4 video formats
# - Generates CDG/TXT packages if enabled
# - Posts Discord notification if configured
# - Handles brand code generation
job_log.info("Starting KaraokeFinalise.process() - this may take 15-20 minutes...")
logger.info(f"Job {job_id}: Starting KaraokeFinalise.process()")
result = finalise.process()
job_log.info("KaraokeFinalise.process() complete!")
if result.get('brand_code'):
job_log.info(f"Brand code: {result.get('brand_code')}")
logger.info(f"Job {job_id}: KaraokeFinalise.process() complete")
logger.info(f"Job {job_id}: Brand code: {result.get('brand_code')}")
# Native API distribution uploads (used by remote CLI instead of rclone)
await _handle_native_distribution(
job_id=job_id,
job=job,
job_log=job_log,
job_manager=job_manager,
temp_dir=temp_dir,
result=result,
)
# Upload generated files to GCS
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.PACKAGING,
progress=95,
message="Uploading final files"
)
await _upload_results(job_id, job_manager, storage, temp_dir, result)
# Mark job as complete
logger.info(f"Job {job_id}: Video generation complete")
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.COMPLETE,
progress=100,
message="Karaoke generation complete!"
)
# Store result metadata in job
job_manager.update_job(job_id, {
'state_data': {
**job.state_data,
'brand_code': result.get('brand_code'),
'youtube_url': result.get('youtube_url'),
}
})
return True
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
# This ensures logs from third-party libraries (karaoke_gen.karaoke_finalise) are only
# captured by this job's handler, not handlers from other concurrent jobs
with job_logging_context(job_id):
job_log.info(f"Starting video finalization for {job.artist} - {job.title}")
logger.info(f"Starting video generation for job {job_id}")
# Transition to GENERATING_VIDEO state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.GENERATING_VIDEO,
progress=70,
message="Preparing files for video generation"
)
# Download and set up files in the format KaraokeFinalise expects
base_name = f"{job.artist} - {job.title}"
job_log.info("Downloading files from GCS (this may take a few minutes for large files)...")
await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log)
job_log.info("All files downloaded successfully")
# Load style config for CDG styles
job_log.info("Loading CDG style configuration...")
style_config = await load_style_config(job, storage, temp_dir)
cdg_styles = style_config.get_cdg_styles()
if cdg_styles:
job_log.info("CDG styles loaded from custom configuration")
else:
job_log.info("Using default CDG styles")
# Change to working directory (KaraokeFinalise works in cwd)
os.chdir(temp_dir)
# Get the selected instrumental file path
# Batch 3: If user provided existing instrumental, use that; otherwise use AI-separated
instrumental_selection = job.state_data['instrumental_selection']
existing_instrumental_path = getattr(job, 'existing_instrumental_gcs_path', None)
if existing_instrumental_path:
# User provided existing instrumental
ext = Path(existing_instrumental_path).suffix.lower()
instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental User){ext}")
instrumental_source = "user-provided"
else:
# Use AI-separated instrumental
instrumental_suffix = "Clean" if instrumental_selection == 'clean' else "Backing"
instrumental_file = os.path.join(temp_dir, f"{base_name} (Instrumental {instrumental_suffix}).flac")
instrumental_source = instrumental_selection
# Transition to encoding state
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.ENCODING,
progress=75,
message="Encoding videos (15-20 min)"
)
# Log finalization parameters
job_log.info("Creating KaraokeFinalise with parameters:")
job_log.info(f" enable_cdg: {getattr(job, 'enable_cdg', False)}")
job_log.info(f" enable_txt: {getattr(job, 'enable_txt', False)}")
job_log.info(f" brand_prefix: {getattr(job, 'brand_prefix', None)}")
job_log.info(f" discord_webhook: {'configured' if getattr(job, 'discord_webhook_url', None) else 'not configured'}")
job_log.info(f" instrumental source: {instrumental_source}")
if existing_instrumental_path:
job_log.info(f" using user-provided instrumental (selection was: {instrumental_selection})")
# Create KaraokeFinalise with ALL the parameters from the job
# This reuses all existing functionality!
# Set up YouTube description file if template is provided
youtube_desc_path = None
if youtube_credentials and getattr(job, 'youtube_description_template', None):
youtube_desc_path = os.path.join(temp_dir, "youtube_description.txt")
with open(youtube_desc_path, 'w') as f:
f.write(job.youtube_description_template)
job_log.info("YouTube description template written to temp file")
finalise = KaraokeFinalise(
logger=logger,
log_level=logging.INFO,
dry_run=False,
instrumental_format="flac",
# CDG/TXT generation
enable_cdg=getattr(job, 'enable_cdg', False),
enable_txt=getattr(job, 'enable_txt', False),
cdg_styles=cdg_styles,
# Brand code and organization (server-side mode uses rclone)
brand_prefix=getattr(job, 'brand_prefix', None),
organised_dir=None, # Not used in cloud - files stay in GCS
organised_dir_rclone_root=getattr(job, 'organised_dir_rclone_root', None),
public_share_dir=None, # Not used in cloud
# Notifications
discord_webhook_url=getattr(job, 'discord_webhook_url', None),
# YouTube upload (server-side with pre-loaded credentials)
youtube_client_secrets_file=None, # Not used with pre-stored credentials
youtube_description_file=youtube_desc_path,
user_youtube_credentials=youtube_credentials, # Pre-loaded from Secret Manager
rclone_destination=None,
email_template_file=None,
# Server-side optimizations
non_interactive=True,
server_side_mode=True,
selected_instrumental_file=instrumental_file,
)
# Call process() - this does ALL the work:
# - Encodes to 4 video formats
# - Generates CDG/TXT packages if enabled
# - Posts Discord notification if configured
# - Handles brand code generation
job_log.info("Starting KaraokeFinalise.process() - this may take 15-20 minutes...")
logger.info(f"Job {job_id}: Starting KaraokeFinalise.process()")
result = finalise.process()
job_log.info("KaraokeFinalise.process() complete!")
if result.get('brand_code'):
job_log.info(f"Brand code: {result.get('brand_code')}")
logger.info(f"Job {job_id}: KaraokeFinalise.process() complete")
logger.info(f"Job {job_id}: Brand code: {result.get('brand_code')}")
# Native API distribution uploads (used by remote CLI instead of rclone)
await _handle_native_distribution(
job_id=job_id,
job=job,
job_log=job_log,
job_manager=job_manager,
temp_dir=temp_dir,
result=result,
)
# Upload generated files to GCS
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.PACKAGING,
progress=95,
message="Uploading final files"
)
await _upload_results(job_id, job_manager, storage, temp_dir, result)
# Mark job as complete
logger.info(f"Job {job_id}: Video generation complete")
job_manager.transition_to_state(
job_id=job_id,
new_status=JobStatus.COMPLETE,
progress=100,
message="Karaoke generation complete!"
)
# Store result metadata in job
job_manager.update_job(job_id, {
'state_data': {
**job.state_data,
'brand_code': result.get('brand_code'),
'youtube_url': result.get('youtube_url'),
}
})
return True
except Exception as e:
logger.error(f"Job {job_id}: Video generation failed: {e}", exc_info=True)
+54 -1
View File
@@ -7,11 +7,48 @@ for remote debugging via the CLI.
Two approaches are provided:
1. JobLogger - A custom logger class for explicit logging in workers
2. JobLogHandler - A logging.Handler that captures logs from any logger (including dependencies)
IMPORTANT: Uses contextvars to ensure log isolation between concurrent jobs.
When multiple jobs run in parallel on the same Cloud Run instance, each job's
worker logs are correctly filtered to only include logs from that job.
"""
import contextvars
import logging
from contextlib import contextmanager
from typing import Optional, Set
# Context variable to track the current job being processed
# This is used to filter logs when multiple jobs run concurrently
_current_job_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
'current_job_id', default=None
)
@contextmanager
def job_logging_context(job_id: str):
"""
Context manager to set the current job ID for log filtering.
When multiple jobs run concurrently on the same Cloud Run instance,
this ensures that each job's handler only captures logs from its own
processing thread/context.
Usage:
with job_logging_context(job_id):
# All logs emitted here will be associated with this job_id
process_job(job_id)
Args:
job_id: The job ID to set as the current context
"""
token = _current_job_id.set(job_id)
try:
yield
finally:
_current_job_id.reset(token)
class JobLogHandler(logging.Handler):
"""
A logging handler that captures log records and stores them in Firestore.
@@ -53,6 +90,15 @@ class JobLogHandler(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
"""Process a log record."""
try:
# CRITICAL: Filter by current job context to prevent log mixing
# When multiple jobs run concurrently, each attaches handlers to
# the same global loggers. Without this check, logs from Job A
# would be captured by Job B's handler and vice versa.
current_job = _current_job_id.get()
if current_job is not None and current_job != self.job_id:
# This log is from a different job's context, skip it
return
# Format the log message
message = self.format(record)
@@ -182,6 +228,10 @@ def setup_job_logging(job_id: str, worker_name: str, *logger_names: str) -> JobL
This adds a JobLogHandler to capture logs from specified loggers
(including dependency loggers like lyrics_transcriber).
IMPORTANT: When using this function, wrap your job processing code in
`job_logging_context(job_id)` to ensure proper log isolation when multiple
jobs run concurrently on the same Cloud Run instance.
Args:
job_id: Job ID
worker_name: Worker name
@@ -199,7 +249,10 @@ def setup_job_logging(job_id: str, worker_name: str, *logger_names: str) -> JobL
"lyrics_transcriber", # Capture LyricsTranscriber logs
)
# ... do work ...
# IMPORTANT: Use job_logging_context for proper isolation
with job_logging_context(job_id):
# ... do work ...
pass
# Optional: remove handler when done
for name in logger_names:
@@ -704,96 +704,17 @@ karaoke-gen-remote \
---
## Future Architecture: Shared Pipeline
## Future Architecture
Once feature parity is achieved, the codebase should be refactored toward a **shared pipeline architecture** where both local and remote execution use the same abstractions.
For the comprehensive architecture plan covering both **horizontal scalability** (Cloud Tasks, Cloud Run Jobs) and **software architecture** (shared pipeline, SOLID, DRY), see:
### Current State (Divergent Paths)
**[SCALABLE-ARCHITECTURE-PLAN.md](./SCALABLE-ARCHITECTURE-PLAN.md)**
```
LOCAL CLI CLOUD BACKEND
───────── ─────────────
KaraokeGen.process() API Routes
│ │
├─► AudioProcessor ├─► audio_worker.py
│ └── Modal API or local │ └── Modal API
│ │
├─► LyricsProcessor ├─► lyrics_worker.py
│ └── Orchestrates everything │ └── Transcription only
│ including video generation │
│ ├─► screens_worker.py
│ │
│ ├─► render_video_worker.py
│ │ └── OutputGenerator directly
│ │
└─► KaraokeFinalise └─► video_worker.py
└── Encoding, distribution └── KaraokeFinalise
```
**Problems:**
- Video generation called differently (via LyricsProcessor vs OutputGenerator directly)
- LyricsProcessor does too many things (fetching, transcription, video, file management)
- Testing requires mocking different things for local vs remote
- Bug fixes may need to be applied in multiple places
### Target State (Shared Pipeline)
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ SHARED PIPELINE ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ AudioInput → Separation → Transcription → Review → Render → Finalize │
│ │ │ │ │ │ │ │
│ ▼ ▼ ▼ ▼ ▼ ▼ │
│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │
│ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │
│ │ API │ │ API │ │ API │ │ API │ │ API │ │ API │ │
│ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ │
│ │ │ │ │ │ │ │
│ ┌────┴───────────┴────────────┴────────────┴──────────┴──────────┴────┐ │
│ │ EXECUTION LAYER │ │
│ │ │ │
│ │ Local Mode: │ Remote Mode: │ │
│ │ - Direct calls │ - HTTP to backend │ │
│ │ - Local GPU/CPU │ - Workers + Modal │ │
│ │ - Blocking │ - Async + polling │ │
│ └──────────────────────┴───────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
**Benefits:**
- Single source of truth for each pipeline stage
- Each stage independently testable
- Same business logic regardless of execution mode
- Easier to add new stages or modify existing ones
- DRY - fix bugs once, works everywhere
### Implementation Plan for Shared Pipeline
This refactor should be done in a separate branch after the v0.71.0 release:
1. **Extract stage interfaces**
```python
class PipelineStage(Protocol):
async def execute(self, context: PipelineContext) -> StageResult:
...
```
2. **Create execution adapters**
```python
class LocalExecutor:
"""Runs stages directly in-process"""
class RemoteExecutor:
"""Runs stages via backend API/workers"""
```
3. **Refactor incrementally**
- Start with one stage (e.g., Separation)
- Prove the pattern works
- Migrate other stages
Key goals:
- **100+ concurrent jobs** with no performance degradation
- **Dedicated resources per job** (no CPU/memory contention)
- **Shared pipeline architecture** - single source of truth for business logic
- **DRY codebase** - bug fixes apply to both local and remote execution
---
@@ -1003,9 +924,10 @@ DEFAULT_DISCORD_WEBHOOK_URL=https://discord.com/...
5. **Edit Lyrics Mode** (Batch 9)
- Allow fixing lyrics on completed jobs
### Long-Term (Shared Pipeline Refactor)
1. Create new branch for shared pipeline architecture
2. Extract stage interfaces
3. Implement execution adapters
4. Migrate stages incrementally
5. Achieve true code sharing between local and remote
### Long-Term (Scalable Architecture)
See **[SCALABLE-ARCHITECTURE-PLAN.md](./SCALABLE-ARCHITECTURE-PLAN.md)** for the comprehensive plan covering:
1. Cloud Tasks integration for horizontal scalability
2. Cloud Run Jobs for long-running video encoding
3. Shared pipeline architecture for code reuse
4. Implementation roadmap and migration strategy
@@ -0,0 +1,656 @@
# Scalable Architecture Plan
**Last Updated:** 2025-12-11
This document outlines the plan for evolving karaoke-gen's cloud backend to support **100+ concurrent jobs** with no performance degradation, while also achieving a clean, maintainable codebase with proper separation of concerns.
---
## Goals
### Infrastructure Goals
1. **Horizontal scalability** - Each job gets dedicated resources (no resource contention)
2. **Maximum processing speed** - Same speed for 1 job or 100 concurrent jobs
3. **Reliability** - Automatic retries, no lost jobs from container restarts
4. **Cost efficiency** - Scale to zero when idle, pay only for active processing
### Software Architecture Goals
1. **SOLID principles** - Single responsibility, clean interfaces
2. **DRY** - Share code between local CLI and remote backend
3. **Testability** - Each component independently testable
4. **Maintainability** - Bug fixes apply everywhere, easy to add features
---
## Current Architecture
### What Works Well
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ CURRENT ARCHITECTURE (v0.71.x) │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ ✅ Heavy work already offloaded to external services: │
│ │
│ Audio Separation ──────────────────▶ Modal API (GPU, ~5-8 min) │
│ Lyrics Transcription ──────────────▶ AudioShake API (~2-3 min) │
│ │
│ ✅ State persistence in Firestore (survives restarts) │
│ ✅ Files in GCS (no local storage dependencies) │
│ ✅ Internal HTTP endpoints exist for worker triggers │
│ ✅ Workers are self-contained (only need job_id) │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Current Limitations
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ CURRENT FLOW (PROBLEMATIC) │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ POST /api/jobs/upload │
│ │ │
│ ├──▶ Save job to Firestore ✅ │
│ │ │
│ └──▶ BackgroundTasks.add_task(trigger_workers) │
│ │ │
│ └──▶ HTTP POST /api/internal/workers/audio │
│ │ │
│ └──▶ BackgroundTasks.add_task(process_audio) ⚠️ │
│ │ │
│ └── Runs IN THE SAME CONTAINER │
│ as the API request handler │
│ │
│ PROBLEMS: │
│ ⚠️ Container may be killed before background tasks complete │
│ ⚠️ No guaranteed delivery - if container crashes, job is lost │
│ ⚠️ Multiple concurrent jobs share CPU/memory (resource contention) │
│ ⚠️ Long-running video encoding (15-20 min) vulnerable to timeouts │
│ ⚠️ FFmpeg renders compete for resources when jobs run in parallel │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
---
## Target Architecture: Horizontally Scalable Infrastructure
### Option A: Google Cloud Tasks (Recommended for Workers)
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ CLOUD TASKS ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ POST /api/jobs/upload │
│ │ │
│ ├──▶ Save job to Firestore │
│ │ │
│ └──▶ Enqueue Cloud Tasks: │
│ │ │
│ ├──▶ audio-worker-queue ──────────────────────────────┐ │
│ │ {job_id, target: /internal/workers/audio} │ │
│ │ │ │
│ └──▶ lyrics-worker-queue ─────────────────────────────┤ │
│ {job_id, target: /internal/workers/lyrics} │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────────────────────┐ │
│ │ Cloud Tasks Queue │ │
│ │ │ │
│ │ • Guaranteed delivery (retries on failure) │ │
│ │ • Rate limiting / concurrency control │ │
│ │ • Task deduplication │ │
│ │ • Configurable timeouts (up to 30 min per task) │ │
│ └────────────────────────────────────────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────────────────────┐ │
│ │ Cloud Run (Auto-Scaled Worker Instances) │ │
│ │ │ │
│ │ Instance 1 Instance 2 Instance 3 ... │ │
│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │
│ │ │ Job A │ │ Job B │ │ Job C │ │ │
│ │ │ Audio │ │ Audio │ │ Lyrics │ │ │
│ │ └──────────┘ └──────────┘ └──────────┘ │ │
│ │ │ │
│ │ Each task gets a DEDICATED container instance │ │
│ │ • Full CPU/memory for that task alone │ │
│ │ • No resource contention with other jobs │ │
│ │ • Container lifecycle tied to task completion │ │
│ └────────────────────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
**Benefits:**
- ✅ Each job gets dedicated resources (no contention)
- ✅ Automatic retries on failure
- ✅ Queue-based backpressure (won't overwhelm external APIs)
- ✅ Container lifecycle matches task lifecycle
- ✅ Cloud Run auto-scales based on queue depth
- ✅ Scales to 1000s of concurrent jobs
**Migration Effort:** LOW - Internal HTTP endpoints already exist
### Option B: Cloud Run Jobs (For Long-Running Tasks)
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ CLOUD RUN JOBS ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ For tasks that exceed 30-minute Cloud Tasks timeout (video encoding): │
│ │
│ POST /api/jobs/{id}/select-instrumental │
│ │ │
│ └──▶ Start Cloud Run Job execution: │
│ │ │
│ └──▶ gcloud run jobs execute video-worker-job \ │
│ --args job_id=$JOB_ID │
│ │
│ ┌────────────────────────────────────────────────────────────────────┐ │
│ │ Cloud Run Job Execution │ │
│ │ │ │
│ │ • Container runs until completion (up to 24 hours) │ │
│ │ • Clean separation from API service │ │
│ │ • Can request higher CPU/memory than service │ │
│ │ • Ideal for video encoding (15-20 min) │ │
│ │ • Progress written to Firestore for CLI monitoring │ │
│ └────────────────────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
**Use Cases:**
- Video rendering (15-20 min of FFmpeg encoding)
- Final video generation with multiple output formats
- Any task that might exceed 30 minutes
### Option C: Pub/Sub + Cloud Run (Event-Driven)
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ PUB/SUB EVENT-DRIVEN ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ Job State Changes ──────────▶ Pub/Sub Topics │
│ │ │
│ job.created ───────────────────────┼──▶ audio-worker subscription │
│ │ └──▶ /internal/workers/audio │
│ │ │
│ job.created ───────────────────────┼──▶ lyrics-worker subscription │
│ │ └──▶ /internal/workers/lyrics │
│ │ │
│ audio.complete + lyrics.complete ──┼──▶ screens-worker subscription │
│ │ └──▶ /internal/workers/screens │
│ │ │
│ review.complete ───────────────────┼──▶ render-worker subscription │
│ │ └──▶ /internal/workers/render │
│ │ │
│ instrumental.selected ─────────────┼──▶ video-worker subscription │
│ └──▶ /internal/workers/video │
│ │
│ Benefits: │
│ • True decoupling (workers don't know about each other) │
│ • Dead-letter queues for failed messages │
│ • Can add new worker types without changing existing code │
│ • Natural fit for event-sourced architecture │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Recommended Hybrid Approach
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ RECOMMENDED: HYBRID ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ Cloud Tasks (fast workers < 30 min): │
│ ├── audio-worker-queue (5-8 min via Modal API) │
│ ├── lyrics-worker-queue (2-3 min via AudioShake) │
│ ├── screens-worker-queue (30 sec FFmpeg) │
│ └── render-worker-queue (10-15 min LyricsTranscriber) │
│ │
│ Cloud Run Jobs (long workers > 30 min): │
│ └── video-encoding-job (15-20 min FFmpeg multi-format encoding) │
│ │
│ Both use the SAME internal HTTP endpoints - minimal code changes! │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
---
## Target Architecture: Shared Pipeline (Software)
### Current State (Divergent Paths)
```
LOCAL CLI CLOUD BACKEND
───────── ─────────────
KaraokeGen.process() API Routes
│ │
├─► AudioProcessor ├─► audio_worker.py
│ └── Modal API or local │ └── Modal API
│ │
├─► LyricsProcessor ├─► lyrics_worker.py
│ └── Orchestrates everything │ └── Transcription only
│ including video generation │
│ ├─► screens_worker.py
│ │
│ ├─► render_video_worker.py
│ │ └── OutputGenerator directly
│ │
└─► KaraokeFinalise └─► video_worker.py
└── Encoding, distribution └── KaraokeFinalise
```
**Problems:**
- Video generation called differently (via LyricsProcessor vs OutputGenerator directly)
- LyricsProcessor does too many things (fetching, transcription, video, file management)
- Testing requires mocking different things for local vs remote
- Bug fixes may need to be applied in multiple places
### Target State (Shared Pipeline)
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ SHARED PIPELINE ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ AudioInput → Separation → Transcription → Review → Render → Finalize │
│ │ │ │ │ │ │ │
│ ▼ ▼ ▼ ▼ ▼ ▼ │
│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │
│ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │ Stage │ │
│ │ API │ │ API │ │ API │ │ API │ │ API │ │ API │ │
│ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ └────┬───┘ │
│ │ │ │ │ │ │ │
│ ┌────┴───────────┴────────────┴────────────┴──────────┴──────────┴────┐ │
│ │ EXECUTION LAYER │ │
│ │ │ │
│ │ Local Mode: │ Remote Mode: │ │
│ │ - Direct calls │ - HTTP to backend │ │
│ │ - Local GPU/CPU │ - Cloud Tasks queue │ │
│ │ - Blocking │ - Async + polling │ │
│ └──────────────────────┴───────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Stage Interface Design
```python
# karaoke_gen/pipeline/base.py
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Protocol
@dataclass
class PipelineContext:
"""Shared context passed between pipeline stages."""
job_id: str
artist: str
title: str
temp_dir: str
style_config: dict
# ... other shared state
@dataclass
class StageResult:
"""Result from a pipeline stage."""
success: bool
outputs: dict # Stage-specific outputs
error: Optional[str] = None
class PipelineStage(Protocol):
"""Interface for all pipeline stages."""
@property
def name(self) -> str:
"""Stage identifier."""
...
async def execute(self, context: PipelineContext) -> StageResult:
"""Execute the stage."""
...
async def validate_inputs(self, context: PipelineContext) -> bool:
"""Verify required inputs exist."""
...
```
### Execution Adapters
```python
# karaoke_gen/pipeline/executors/local.py
class LocalExecutor:
"""Runs stages directly in-process (for CLI)."""
async def run_stage(self, stage: PipelineStage, context: PipelineContext) -> StageResult:
# Validate inputs
if not await stage.validate_inputs(context):
return StageResult(success=False, outputs={}, error="Missing inputs")
# Run directly
return await stage.execute(context)
# karaoke_gen/pipeline/executors/remote.py
class RemoteExecutor:
"""Runs stages via backend API/Cloud Tasks (for remote CLI)."""
async def run_stage(self, stage: PipelineStage, context: PipelineContext) -> StageResult:
# Enqueue to Cloud Tasks
task = cloud_tasks_client.create_task(
queue=f"{stage.name}-worker-queue",
http_request={
"url": f"{self.backend_url}/api/internal/workers/{stage.name}",
"body": json.dumps({"job_id": context.job_id}),
}
)
# Poll for completion
while True:
status = await self.get_job_status(context.job_id)
if status.stage_complete(stage.name):
return StageResult(success=True, outputs=status.stage_outputs)
await asyncio.sleep(5)
```
---
## Combined Architecture Vision
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ COMPLETE TARGET ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ PIPELINE STAGES (SHARED) │ │
│ │ │ │
│ │ SeparationStage TranscriptionStage RenderStage FinalizeStage │ │
│ │ │ │ │ │ │ │
│ │ Single source of truth for business logic │ │
│ │ Used by both local CLI and cloud workers │ │
│ └─────────────────────────────────────────────────────────────────────┘ │
│ │ │
│ ┌───────────┴───────────┐ │
│ │ │ │
│ ┌──────────────────────▼───────┐ ┌──────────▼──────────────────────┐ │
│ │ LOCAL EXECUTOR │ │ REMOTE EXECUTOR │ │
│ │ │ │ │ │
│ │ karaoke-gen CLI │ │ karaoke-gen-remote CLI │ │
│ │ • Direct stage calls │ │ • HTTP to backend │ │
│ │ • Local GPU/CPU │ │ • Progress polling │ │
│ │ • Blocking execution │ │ │ │
│ │ • Browser review UI │ │ │ │
│ └──────────────────────────────┘ └──────────┬──────────────────────┘ │
│ │ │
│ ▼ │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ CLOUD INFRASTRUCTURE │ │
│ │ │ │
│ │ Cloud Run Service (API) │ │
│ │ ├── Job submission, status, review endpoints │ │
│ │ └── Enqueues tasks to Cloud Tasks │ │
│ │ │ │
│ │ Cloud Tasks Queues (Guaranteed Delivery) │ │
│ │ ├── audio-worker-queue ──▶ Cloud Run (auto-scaled) │ │
│ │ ├── lyrics-worker-queue ──▶ Cloud Run (auto-scaled) │ │
│ │ ├── screens-worker-queue ──▶ Cloud Run (auto-scaled) │ │
│ │ └── render-worker-queue ──▶ Cloud Run (auto-scaled) │ │
│ │ │ │
│ │ Cloud Run Jobs (Long-Running) │ │
│ │ └── video-encoding-job ──▶ Dedicated high-CPU instance │ │
│ │ │ │
│ │ Each task gets DEDICATED RESOURCES: │ │
│ │ • 2-4 vCPU, 4-8GB RAM per task │ │
│ │ • No contention with other jobs │ │
│ │ • Same speed whether 1 job or 100 jobs │ │
│ └─────────────────────────────────────────────────────────────────────┘ │
│ │
│ External Services (Unchanged): │
│ ├── Modal API (GPU audio separation) │
│ ├── AudioShake API (transcription) │
│ ├── Firestore (job state) │
│ └── GCS (file storage) │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
---
## Implementation Plan
### Phase 1: Infrastructure Scaling (Priority: HIGH)
**Goal:** Enable 100+ concurrent jobs with no performance degradation
**Changes:**
1. **Add Cloud Tasks infrastructure (Pulumi)**
```python
# infrastructure/__main__.py
# Create Cloud Tasks queues
audio_queue = gcp.cloudtasks.Queue("audio-worker-queue",
location=region,
rate_limits={
"max_dispatches_per_second": 10, # Rate limit to protect Modal API
"max_concurrent_dispatches": 50, # Max parallel workers
},
retry_config={
"max_attempts": 3,
"min_backoff": "10s",
"max_backoff": "300s",
}
)
# Similar for lyrics, screens, render queues
```
2. **Update WorkerService to use Cloud Tasks**
```python
# backend/services/worker_service.py
async def trigger_worker(self, worker_type: str, job_id: str) -> bool:
if self.use_cloud_tasks:
# Production: enqueue to Cloud Tasks
return await self._enqueue_cloud_task(worker_type, job_id)
else:
# Development: direct HTTP call (existing behavior)
return await self._trigger_http(worker_type, job_id)
```
3. **Add Cloud Run Job for video encoding**
```yaml
# cloudbuild-video-job.yaml
- name: gcr.io/cloud-builders/gcloud
args:
- run
- jobs
- create
- video-encoding-job
- --image=${_IMAGE}
- --cpu=4
- --memory=8Gi
- --max-retries=2
- --task-timeout=3600 # 1 hour max
```
4. **Update internal endpoints for Cloud Tasks**
- Add request validation for Cloud Tasks headers
- Extend timeout handling for long-running tasks
- Add idempotency for safe retries
**Estimated Effort:** 2-3 days
**Risk:** Low (existing endpoints work unchanged)
### Phase 2: Shared Pipeline Architecture (Priority: MEDIUM)
**Goal:** Single source of truth for business logic, DRY codebase
**Changes:**
1. **Extract stage interfaces**
```
karaoke_gen/pipeline/
├── __init__.py
├── base.py # PipelineStage, PipelineContext, StageResult
├── context.py # Context management
├── stages/
│ ├── __init__.py
│ ├── separation.py # Audio separation stage
│ ├── transcription.py # Lyrics transcription stage
│ ├── screens.py # Title/end screen generation
│ ├── render.py # Video rendering with lyrics
│ └── finalize.py # Final encoding and distribution
└── executors/
├── __init__.py
├── local.py # Direct execution for CLI
└── remote.py # Cloud Tasks execution for remote CLI
```
2. **Migrate stages incrementally**
- Start with SeparationStage (simplest, well-isolated)
- Prove pattern works with both executors
- Migrate remaining stages one at a time
- Keep old code paths working during migration
3. **Update CLIs to use pipeline**
```python
# karaoke_gen/utils/gen_cli.py (local)
executor = LocalExecutor()
pipeline = Pipeline([
SeparationStage(),
TranscriptionStage(),
ReviewStage(),
RenderStage(),
FinalizeStage(),
])
await pipeline.run(executor, context)
# karaoke_gen/utils/remote_cli.py (remote)
executor = RemoteExecutor(backend_url)
# Same pipeline, different executor
await pipeline.run(executor, context)
```
**Estimated Effort:** 1-2 weeks
**Risk:** Medium (requires careful refactoring)
### Phase 3: Advanced Features (Priority: LOW)
1. **Event-driven architecture with Pub/Sub** (optional)
2. **Workflow orchestration with Cloud Workflows** (optional)
3. **Cost optimization with preemptible VMs** (for video encoding)
4. **Multi-region deployment** for lower latency
---
## Migration Strategy
### Zero-Downtime Migration
```
Week 1: Deploy Cloud Tasks infrastructure (no code changes)
└── Queues exist but aren't used yet
Week 2: Add feature flag for Cloud Tasks
└── ENABLE_CLOUD_TASKS=false (default)
└── Test with flag enabled in staging
Week 3: Enable Cloud Tasks in production
└── ENABLE_CLOUD_TASKS=true
└── Monitor for issues
Week 4: Remove old BackgroundTasks code paths
└── Clean up feature flag
└── Update documentation
```
### Rollback Plan
If issues occur after enabling Cloud Tasks:
1. Set `ENABLE_CLOUD_TASKS=false`
2. Redeploy (takes ~2 minutes)
3. New jobs use BackgroundTasks (old behavior)
4. In-flight Cloud Tasks jobs continue to completion
---
## Resource Configuration
### Cloud Tasks Queues
| Queue | Max Concurrent | Rate Limit | Timeout | Retries |
|-------|---------------|------------|---------|---------|
| audio-worker | 50 | 10/sec | 15 min | 3 |
| lyrics-worker | 50 | 10/sec | 10 min | 3 |
| screens-worker | 100 | 50/sec | 2 min | 3 |
| render-worker | 20 | 5/sec | 30 min | 2 |
### Cloud Run Worker Instances
| Worker | CPU | Memory | Timeout | Notes |
|--------|-----|--------|---------|-------|
| audio | 2 | 2Gi | 15 min | Mostly I/O (Modal API) |
| lyrics | 2 | 2Gi | 10 min | Mostly I/O (AudioShake) |
| screens | 2 | 2Gi | 2 min | FFmpeg (quick) |
| render | 4 | 4Gi | 30 min | LyricsTranscriber + FFmpeg |
| video-job | 4 | 8Gi | 60 min | Heavy FFmpeg encoding |
### Cost Estimate (100 concurrent jobs)
```
Cloud Tasks: ~$0.40/million tasks ≈ $0.01/day
Cloud Run: ~$0.00002/vCPU-sec ≈ $5-10/day (100 jobs)
Cloud Run Jobs: ~$0.00002/vCPU-sec ≈ $2-5/day (100 jobs)
Modal (GPU): ~$2.50/hour ≈ $20-40/day (100 jobs)
AudioShake: Usage-based ≈ $10-20/day (100 jobs)
Total: ~$40-80/day for 100 concurrent jobs
```
---
## Success Criteria
### Infrastructure
- [ ] 100 concurrent jobs complete without failures
- [ ] Processing time per job unchanged (same as 1 job)
- [ ] No resource contention (CPU/memory per job consistent)
- [ ] Automatic retry on transient failures
- [ ] Zero lost jobs from container restarts
### Software Architecture
- [ ] Single stage implementation used by both CLIs
- [ ] Bug fixes apply to both local and remote execution
- [ ] Each stage independently testable
- [ ] Clear interfaces between stages
- [ ] 80%+ code coverage on pipeline module
---
## Related Documents
- [ARCHITECTURE.md](../01-reference/ARCHITECTURE.md) - Current system architecture
- [BACKEND-FEATURE-PARITY-PLAN.md](./BACKEND-FEATURE-PARITY-PLAN.md) - Feature parity roadmap
- [WORKER-IMPLEMENTATION-PLAN.md](./WORKER-IMPLEMENTATION-PLAN.md) - Worker details
---
## Changelog
### 2025-12-11: Initial Plan
- Documented current architecture limitations
- Proposed Cloud Tasks + Cloud Run Jobs hybrid
- Outlined shared pipeline architecture
- Created implementation roadmap
+10 -8
View File
@@ -121,16 +121,18 @@ class FileHandler:
},
}
# Handle cookies if provided
# Handle cookies if provided - use safe tempfile pattern to avoid leaks
cookie_file_path = None
if cookies_str:
# Write cookies to a temporary file
cookie_file = tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False)
try:
cookie_file.write(cookies_str)
cookie_file.close()
ydl_opts['cookiefile'] = cookie_file.name
# Use context manager to safely write cookies file
with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as cookie_file:
cookie_file.write(cookies_str)
cookie_file_path = cookie_file.name
ydl_opts['cookiefile'] = cookie_file_path
except Exception as e:
self.logger.warning(f"Failed to write cookies file: {e}")
cookie_file_path = None
try:
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
@@ -177,9 +179,9 @@ class FileHandler:
return None
finally:
# Clean up cookie file if we created one
if cookies_str and 'cookie_file' in locals():
if cookie_file_path is not None:
try:
os.unlink(cookie_file.name)
os.unlink(cookie_file_path)
except Exception:
pass
+10 -3
View File
@@ -252,7 +252,6 @@ class RemoteKaraokeClient:
url: str,
artist: Optional[str] = None,
title: Optional[str] = None,
style_params_path: Optional[str] = None,
enable_cdg: bool = True,
enable_txt: bool = True,
brand_prefix: Optional[str] = None,
@@ -275,11 +274,14 @@ class RemoteKaraokeClient:
The backend will download the audio from the URL and process it.
Artist and title will be auto-detected from the URL if not provided.
Note: Custom style configuration is not supported for URL-based jobs.
If you need custom styles, download the audio locally first and use
the regular file upload flow with submit_job().
Args:
url: YouTube or other video URL to download audio from
artist: Artist name (optional - auto-detected if not provided)
title: Song title (optional - auto-detected if not provided)
style_params_path: Path to style_params.json (optional)
enable_cdg: Generate CDG+MP3 package
enable_txt: Generate TXT+MP3 package
brand_prefix: Brand code prefix (e.g., "NOMAD")
@@ -2045,11 +2047,16 @@ def main():
# Submit job - different endpoint for URL vs file
if is_url_input:
# URL-based job submission
# Note: style_params_path is not supported for URL-based jobs
# If custom styles are needed, download the audio locally first
if args.style_params_json:
logger.warning("Custom styles (--style_params_json) are not supported for URL-based jobs. "
"Download the audio locally first and use file upload for custom styles.")
result = client.submit_job_from_url(
url=input_media,
artist=artist,
title=title,
style_params_path=args.style_params_json,
enable_cdg=args.enable_cdg,
enable_txt=args.enable_txt,
brand_prefix=args.brand_prefix,
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.71.26"
version = "0.71.29"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"