diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a4e2dff2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,83 @@ +# .dockerignore - Reduce Docker build context size +# This is critical for fast builds as it reduces upload time and cache invalidation + +# Git +.git/ +.gitignore +.gitmodules + +# Node.js / Frontend (not needed for backend image) +node_modules/ +frontend/ +frontend-react/ +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Python virtual environments (use Docker's Python instead) +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +.venv/ +venv/ +backend/venv/ +ENV/ +env/ +*.egg-info/ +.eggs/ +dist/ +build/ +*.egg + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store + +# Testing (not needed in production image) +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ +coverage.xml +*.cover +tests/ +backend/tests/ + +# Documentation (not needed in Docker image) +docs/ +*.md +!README.md + +# Infrastructure (not needed for Docker build) +infrastructure/ + +# CI/CD configs +.github/ + +# Cloud Build configs (keep for reference but not in image) +cloudbuild*.yaml + +# Local development files +lyrics_transcriber_local/ +*.local +.env +.env.* +*.bak +*.tmp +Thumbs.db + +# Large test/sample files +tests/data/ +*.mp3 +*.wav +*.mp4 +*.mkv +*.flac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4620d7fe..f15b9cbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -644,6 +644,12 @@ jobs: permissions: contents: read id-token: write + env: + REGISTRY: us-central1-docker.pkg.dev + PROJECT_ID: nomadkaraoke + REPOSITORY: karaoke-repo + BASE_IMAGE_NAME: karaoke-backend-base + APP_IMAGE_NAME: karaoke-backend steps: - name: Checkout code uses: actions/checkout@v4 @@ -689,21 +695,37 @@ jobs: with: version: "latest" - - name: Check if base image needs rebuild + - name: Configure Docker for Artifact Registry + if: steps.version_check.outputs.needs_deploy == 'true' + run: gcloud auth configure-docker ${{ env.REGISTRY }} --quiet + + # ============================================ + # Docker Build with BuildKit + Registry Cache + # This replaces expensive Cloud Build with free GitHub Actions + # Registry cache persists between builds for fast rebuilds + # ============================================ + + - name: Set up Docker Buildx + if: steps.version_check.outputs.needs_deploy == 'true' + uses: docker/setup-buildx-action@v3 + + - name: Calculate base image hash + if: steps.version_check.outputs.needs_deploy == 'true' + id: base-hash + run: | + # Hash both poetry.lock AND Dockerfile.base - rebuild if either changes + CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1) + echo "hash=$CURRENT_HASH" >> $GITHUB_OUTPUT + echo "📋 Base content hash: $CURRENT_HASH" + + - name: Check if base image exists and is current if: steps.version_check.outputs.needs_deploy == 'true' id: check-base run: | - echo "🔍 Checking if base image needs rebuild..." + BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest" - # Hash both poetry.lock AND Dockerfile.base - rebuild if either changes - CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1) - echo "Current combined hash (poetry.lock + Dockerfile.base): $CURRENT_HASH" - - gcloud auth configure-docker us-central1-docker.pkg.dev --quiet - - BASE_IMAGE="us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest" + # Try to get the existing base image's hash label EXISTING_HASH="" - if docker pull $BASE_IMAGE 2>/dev/null; then EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "base.content.hash"}}' 2>/dev/null || echo "") # Fallback to old label name for backwards compatibility @@ -712,10 +734,10 @@ jobs: fi echo "Existing base image hash: $EXISTING_HASH" else - echo "⚠️ No existing base image found" + echo "⚠️ No existing base image found - will build" fi - if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then + if [ "${{ steps.base-hash.outputs.hash }}" != "$EXISTING_HASH" ]; then echo "🔄 Base image needs rebuild (poetry.lock or Dockerfile.base changed)" echo "needs_rebuild=true" >> $GITHUB_OUTPUT else @@ -723,88 +745,83 @@ jobs: echo "needs_rebuild=false" >> $GITHUB_OUTPUT fi - - name: Rebuild base image (if needed) + - name: Free disk space (for base image build) if: steps.version_check.outputs.needs_deploy == 'true' && steps.check-base.outputs.needs_rebuild == 'true' - run: | - echo "🏗️ Rebuilding base image with updated dependencies..." + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + haskell: true + dotnet: true + large-packages: true + docker-images: false + swap-storage: false - BASE_BUILD_ID=$(gcloud builds submit \ - --config=cloudbuild-base.yaml \ - --project=nomadkaraoke \ - --format='value(id)' \ - --async \ - .) + - name: Build and push base image (if needed) + if: steps.version_check.outputs.needs_deploy == 'true' && steps.check-base.outputs.needs_rebuild == 'true' + uses: docker/build-push-action@v6 + with: + context: . + file: backend/Dockerfile.base + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest + ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:${{ github.sha }} + # Registry cache for ALL layers (mode=max) - persists between builds + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:cache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:cache,mode=max + build-args: | + BUILD_DATE=${{ github.run_id }} + BUILD_ID=${{ github.sha }} + BASE_CONTENT_HASH=${{ steps.base-hash.outputs.hash }} + # Provenance disabled for smaller image and faster push + provenance: false - echo "Base Build ID: $BASE_BUILD_ID" - echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BASE_BUILD_ID?project=nomadkaraoke" + - name: Build and push app image + if: steps.version_check.outputs.needs_deploy == 'true' + uses: docker/build-push-action@v6 + with: + context: . + file: backend/Dockerfile + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:latest + ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }} + ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:v${{ steps.version_check.outputs.version }} + # Registry cache for app layers + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,mode=max + provenance: false - echo "Waiting for base image build to complete..." - while true; do - BUILD_STATUS=$(gcloud builds describe $BASE_BUILD_ID --project=nomadkaraoke --format='value(status)') - echo "Base build status: $BUILD_STATUS" + # ============================================ + # Deploy to Cloud Run + # ============================================ - if [ "$BUILD_STATUS" = "SUCCESS" ]; then - echo "✅ Base image build completed successfully" - break - elif [ "$BUILD_STATUS" = "FAILURE" ] || [ "$BUILD_STATUS" = "CANCELLED" ] || [ "$BUILD_STATUS" = "TIMEOUT" ]; then - echo "❌ Base image build failed with status: $BUILD_STATUS" - echo "" - echo "==================== BUILD LOGS ====================" - gcloud logging read "resource.type=build AND resource.labels.build_id=$BASE_BUILD_ID" \ - --project=nomadkaraoke \ - --format="value(textPayload)" \ - --limit=200 \ - --order=asc || echo "Failed to retrieve logs from Cloud Logging" - echo "====================================================" - gcloud builds log $BASE_BUILD_ID --project=nomadkaraoke 2>&1 || true - exit 1 - fi - - sleep 30 - done - - - name: Trigger Cloud Build + - name: Deploy to Cloud Run if: steps.version_check.outputs.needs_deploy == 'true' run: | - echo "🚀 Triggering Cloud Build for deployment..." - echo "Commit: ${{ github.sha }}" - echo "Version: ${{ steps.version_check.outputs.version }}" + VERSION="${{ steps.version_check.outputs.version }}" + IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}" - BUILD_ID=$(gcloud builds submit \ - --config=cloudbuild.yaml \ - --project=nomadkaraoke \ - --format='value(id)' \ - --async \ - .) + echo "🚀 Deploying to Cloud Run..." + echo "Image: $IMAGE" + echo "Version: $VERSION" - echo "Build ID: $BUILD_ID" - echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke" - - echo "Waiting for build to complete..." - while true; do - BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)') - echo "Build status: $BUILD_STATUS" - - if [ "$BUILD_STATUS" = "SUCCESS" ]; then - echo "✅ Build completed successfully" - break - elif [ "$BUILD_STATUS" = "FAILURE" ] || [ "$BUILD_STATUS" = "CANCELLED" ] || [ "$BUILD_STATUS" = "TIMEOUT" ]; then - echo "❌ Build failed with status: $BUILD_STATUS" - echo "" - echo "==================== BUILD LOGS ====================" - gcloud logging read "resource.type=build AND resource.labels.build_id=$BUILD_ID" \ - --project=nomadkaraoke \ - --format="value(textPayload)" \ - --limit=200 \ - --order=asc || echo "Failed to retrieve logs from Cloud Logging" - echo "====================================================" - echo "" - gcloud builds log $BUILD_ID --project=nomadkaraoke 2>&1 || true - exit 1 - fi - - sleep 30 - done + gcloud run deploy karaoke-backend \ + --image $IMAGE \ + --region us-central1 \ + --platform managed \ + --allow-unauthenticated \ + --memory 16Gi \ + --cpu 8 \ + --cpu-boost \ + --execution-environment gen2 \ + --timeout 1800 \ + --concurrency 1 \ + --max-instances 50 \ + --min-instances 0 \ + --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD" \ + --set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest" - name: Verify deployment and version if: steps.version_check.outputs.needs_deploy == 'true' @@ -855,6 +872,10 @@ jobs: echo "Commit: ${{ github.sha }}" echo "Branch: ${{ github.ref_name }}" echo "Run: ${{ github.run_number }}" + echo "" + echo "📊 Cost savings: This build used FREE GitHub Actions instead of Cloud Build" + echo " Previously: ~\$0.064/min with N1_HIGHCPU_32" + echo " Now: \$0 (public repo) with registry-cached builds" - name: Skipped deployment summary if: steps.version_check.outputs.needs_deploy == 'false' diff --git a/docs/03-deployment/GITHUB-ACTIONS-DOCKER-BUILDS.md b/docs/03-deployment/GITHUB-ACTIONS-DOCKER-BUILDS.md new file mode 100644 index 00000000..9386cbd5 --- /dev/null +++ b/docs/03-deployment/GITHUB-ACTIONS-DOCKER-BUILDS.md @@ -0,0 +1,197 @@ +# GitHub Actions Docker Builds - Cost Optimization + +**Date:** 2025-12-28 +**Problem:** Cloud Build costing $100+/month with slow builds +**Solution:** Move Docker builds to GitHub Actions with registry caching + +--- + +## Overview + +This document describes the migration from Google Cloud Build to GitHub Actions for Docker image builds. This change eliminates Cloud Build costs while improving build speeds through better caching. + +## Cost Comparison + +| Approach | Monthly Cost | Build Speed | Caching | +|----------|-------------|-------------|---------| +| **Cloud Build (N1_HIGHCPU_32)** | ~$100+ | 5-15 min | Limited | +| **GitHub Actions + Registry Cache** | **$0** | 2-5 min | Excellent | + +### Why Cloud Build Was Expensive + +1. **N1_HIGHCPU_32 pricing**: ~$0.064/build-minute (21x more than default) +2. **No persistent layer cache**: Each build re-downloaded dependencies +3. **Cold start overhead**: Spinning up build VMs added latency +4. **Large dependency tree**: torch, transformers, etc. = ~500 packages + +### Why GitHub Actions Is Free + +- **Public repositories**: Unlimited build minutes on GitHub-hosted runners +- **Registry cache**: Layers cached in Artifact Registry, persist between builds +- **BuildKit mode=max**: Caches ALL intermediate layers, not just final + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ NEW BUILD PIPELINE │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ GitHub Actions (Free) │ +│ ├── docker/setup-buildx-action (BuildKit) │ +│ ├── google-github-actions/auth (Workload Identity) │ +│ └── docker/build-push-action │ +│ ├── cache-from: type=registry,ref=...karaoke-backend:cache │ +│ └── cache-to: type=registry,ref=...karaoke-backend:cache,mode=max│ +│ │ +│ Artifact Registry │ +│ ├── karaoke-backend:latest (production image) │ +│ ├── karaoke-backend:cache (build cache - ALL layers) │ +│ ├── karaoke-backend-base:latest (base image with deps) │ +│ └── karaoke-backend-base:cache (base cache) │ +│ │ +│ Build Times (Expected): │ +│ ├── First build (cold): ~8-10 min │ +│ ├── Code-only changes: ~2-3 min (cache hit) │ +│ └── Dependency changes: ~6-8 min (partial cache hit) │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## How It Works + +### Two-Stage Build Pattern + +We maintain the existing two-stage build pattern for efficiency: + +1. **Base Image** (`karaoke-backend-base`) + - Contains Python 3.11, ffmpeg, system dependencies + - All pip dependencies pre-installed + - Only rebuilt when `poetry.lock` or `Dockerfile.base` changes + - Hash-based change detection via image labels + +2. **App Image** (`karaoke-backend`) + - Built FROM the base image + - Contains only application code + - Fast rebuilds (~2 min) for code-only changes + +### Registry Cache + +The key optimization is using Artifact Registry as a persistent layer cache: + +```yaml +- uses: docker/build-push-action@v6 + with: + cache-from: type=registry,ref=.../karaoke-backend:cache + cache-to: type=registry,ref=.../karaoke-backend:cache,mode=max +``` + +**Benefits:** +- `mode=max` caches ALL layers, not just the final image +- Cache persists between CI runs (unlike GitHub Actions cache which has 10GB limit) +- Layers only re-built when inputs change + +--- + +## Files Changed + +### New Files + +- **`.dockerignore`** - Reduces build context from ~1GB to ~10MB + +### Modified Files + +- **`.github/workflows/ci.yml`** - `deploy-backend` job rewritten to use: + - `docker/setup-buildx-action@v3` + - `docker/build-push-action@v6` + - Registry caching to Artifact Registry + +### Kept (as fallback) + +- **`cloudbuild.yaml`** - Still works for manual triggers +- **`cloudbuild-base.yaml`** - Still works for manual base image builds + +--- + +## Manual Builds (Fallback) + +If GitHub Actions fails, you can still use Cloud Build: + +```bash +# Build base image (only if dependencies changed) +gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke + +# Build and deploy app image +gcloud builds submit --config=cloudbuild.yaml --project=nomadkaraoke +``` + +--- + +## Monitoring + +### Check Build Cache Usage + +View cache images in Artifact Registry: + +```bash +gcloud artifacts docker images list \ + us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo \ + --include-tags +``` + +You should see `*:cache` tags for both base and app images. + +### Compare Build Times + +- **GitHub Actions**: Check workflow run times in Actions tab +- **Previous Cloud Build**: ~5-15 min per build +- **Expected with cache**: ~2-3 min for code changes + +--- + +## Troubleshooting + +### Cache Miss (Slow Build) + +If builds are slow despite no dependency changes: + +1. Check if cache image exists: + ```bash + docker pull us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:cache + ``` + +2. Force cache rebuild by pushing to `main` with a small change + +### Disk Space Issues + +GitHub Actions runners have limited disk. If base image builds fail: + +1. The workflow includes `jlumbroso/free-disk-space` action +2. It removes Android SDK, Haskell, .NET, etc. to free ~20GB + +### Authentication Issues + +If `docker push` fails with permission errors: + +1. Check Workload Identity is configured correctly +2. Verify service account has `roles/artifactregistry.writer` + +--- + +## Future Improvements + +1. **Parallel base/app builds**: If base is unchanged, start app build immediately +2. **GitHub Actions cache fallback**: Use GHA cache when registry is slow +3. **Build metrics**: Track build times and cache hit rates + +--- + +## References + +- [Docker Build with GitHub Actions](https://docs.docker.com/build/ci/github-actions/) +- [Registry Cache Backend](https://docs.docker.com/build/cache/backends/registry/) +- [Cache Management](https://docs.docker.com/build/ci/github-actions/cache/) diff --git a/pyproject.toml b/pyproject.toml index 6929bdcb..3cbe0a96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.76.20" +version = "0.76.21" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"