Optimize CI/CD: faster builds with base image and Cloud Build

- Add .gcloudignore to reduce upload from 43K files to 317 files (99% less)
- Create Dockerfile.base with pre-built dependencies (~15 min, only when deps change)
- Update Dockerfile to use base image (app builds now ~1-2 min)
- Upgrade Cloud Build to N1_HIGHCPU_32 for faster compilation
- Move Docker build from GitHub Actions to Cloud Build (faster GCP network)
- Fix signed URL generation for Cloud Run (use IAM signBlob API)
- Update submit-job.sh to poll for review completion instead of waiting for Enter
- Sync Cloud Run deploy settings (16Gi RAM, 8 CPU, gen2, proper secrets)
This commit is contained in:
Andrew Beveridge committed 2025-12-08 03:00:22 -05:00
1 parent 0b266decb3
commit 58ccc4324e
8 files changed
+295 -118

No files matched your search

+73
View File
@@ -0,0 +1,73 @@
# .gcloudignore - Exclude files from Cloud Build uploads
# This dramatically reduces upload time (from 218MB to ~5-10MB)
# Git
.git/
.gitignore
.gitmodules
# Node.js / Frontend
node_modules/
frontend-react/node_modules/
frontend-react/dist/
frontend-react/.vite/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
.venv/
venv/
ENV/
env/
*.egg-info/
.eggs/
dist/
build/
*.egg
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
.DS_Store
# Testing
.pytest_cache/
.coverage
htmlcov/
.tox/
.nox/
coverage.xml
*.cover
# Documentation (not needed for Docker build)
docs/
# Local development files
lyrics_transcriber_local/
*.local
.env
.env.*
# Large test files
tests/data/*.mp3
tests/data/*.wav
tests/data/*.mp4
tests/data/*.mkv
# Infrastructure (not needed for Docker build)
infrastructure/
# Misc
*.bak
*.tmp
Thumbs.db
+27 -41
View File
@@ -131,7 +131,7 @@ jobs:
echo "✅ Basic syntax validation passed" echo "✅ Basic syntax validation passed"
deploy: deploy:
name: Deploy to Cloud Run name: Deploy via Cloud Build
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [unit-tests, emulator-tests, lint] needs: [unit-tests, emulator-tests, lint]
# Only deploy on push to the branch (not on PRs) # Only deploy on push to the branch (not on PRs)
@@ -159,49 +159,36 @@ jobs:
with: with:
version: 'latest' version: 'latest'
- name: Configure Docker for Artifact Registry - name: Trigger Cloud Build
run: | run: |
gcloud auth configure-docker us-central1-docker.pkg.dev echo "🚀 Triggering Cloud Build for deployment..."
echo "Commit: ${{ github.sha }}"
- name: Build and push Docker images
run: |
# Build backend image (context must be repo root for pyproject.toml and karaoke_gen/)
docker build -t us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest \
-t us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:${{ github.sha }} \
-f backend/Dockerfile .
# Push both tags # Submit build and wait for completion
docker push us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest BUILD_ID=$(gcloud builds submit \
docker push us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:${{ github.sha }} --config=cloudbuild.yaml \
--project=nomadkaraoke \
--substitutions=_COMMIT_SHA=${{ github.sha }} \
--format='value(id)' \
--async \
.)
echo "✅ Images pushed to Artifact Registry" echo "Build ID: $BUILD_ID"
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke"
- name: Deploy to Cloud Run
run: |
gcloud run deploy karaoke-backend \
--image us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest \
--platform managed \
--region us-central1 \
--allow-unauthenticated \
--set-env-vars "GOOGLE_CLOUD_PROJECT=nomadkaraoke" \
--set-env-vars "GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke" \
--set-env-vars "FIRESTORE_COLLECTION=jobs" \
--set-env-vars "ENVIRONMENT=production" \
--set-env-vars "ADMIN_TOKENS=${{ secrets.ADMIN_TOKENS }}" \
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest" \
--set-secrets "GENIUS_API_TOKEN=genius-api-key:latest" \
--set-secrets "SPOTIFY_COOKIE=spotify-cookie:latest" \
--set-secrets "RAPIDAPI_KEY=rapidapi-key:latest" \
--memory 16Gi \
--cpu 8 \
--cpu-boost \
--execution-environment gen2 \
--timeout 1800 \
--concurrency 4 \
--max-instances 10 \
--min-instances 0
echo "✅ Deployed to Cloud Run" # Wait for build to complete
echo "Waiting for build to complete..."
gcloud builds log $BUILD_ID --project=nomadkaraoke --stream
# Check build status
BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)')
if [ "$BUILD_STATUS" != "SUCCESS" ]; then
echo "❌ Build failed with status: $BUILD_STATUS"
exit 1
fi
echo "✅ Build completed successfully"
- name: Verify deployment - name: Verify deployment
run: | run: |
@@ -226,4 +213,3 @@ jobs:
echo "Commit: ${{ github.sha }}" echo "Commit: ${{ github.sha }}"
echo "Branch: ${{ github.ref_name }}" echo "Branch: ${{ github.ref_name }}"
echo "Run: ${{ github.run_number }}" echo "Run: ${{ github.run_number }}"
+15 -36
View File
@@ -1,43 +1,22 @@
# Use Python 3.11 slim image # Fast application image - uses pre-built base with all dependencies
FROM python:3.11-slim #
# The base image (karaoke-backend-base) contains:
# - Python 3.11, ffmpeg, sox, system deps
# - All pip dependencies from requirements.txt
# - The karaoke_gen package
#
# This Dockerfile only copies the backend code, making builds very fast (~30s).
#
# If the base image doesn't exist yet, build it first:
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
# Install system dependencies required by karaoke-gen # Use pre-built base image with all dependencies
RUN apt-get update && apt-get install -y \ FROM us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
ffmpeg \
libsndfile1 \
libsox-dev \
sox \
build-essential \
&& rm -rf /var/lib/apt/lists/*
# Set working directory # Set working directory (should already exist from base)
WORKDIR /app WORKDIR /app
# ============================================ # Copy backend code (this is the only thing that changes frequently)
# OPTIMIZATION: Install dependencies BEFORE copying code
# This allows Docker to cache the dependency layer
# ============================================
# Copy ONLY dependency files first
COPY pyproject.toml /app/
COPY README.md /app/
COPY LICENSE /app/
COPY backend/requirements.txt /app/backend/requirements.txt
# Install Python dependencies (this layer will be cached unless requirements change)
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir -r backend/requirements.txt
# Now copy the karaoke_gen package and install it
# This includes karaoke_gen dependencies
COPY karaoke_gen /app/karaoke_gen
RUN pip install --no-cache-dir -e .
# ============================================
# Copy application code LAST (changes frequently)
# ============================================
# Copy backend code (this invalidates cache only for code changes)
COPY backend /app/backend COPY backend /app/backend
# Set environment variables # Set environment variables
+41
View File
@@ -0,0 +1,41 @@
# Base image with all Python dependencies pre-installed
# This image is rebuilt only when requirements.txt changes
#
# Build with: gcloud builds submit --config=cloudbuild-base.yaml
# Or manually: docker build -f backend/Dockerfile.base -t karaoke-backend-base .
FROM python:3.11-slim
# Install system dependencies required by karaoke-gen
RUN apt-get update && apt-get install -y \
ffmpeg \
libsndfile1 \
libsox-dev \
sox \
build-essential \
&& rm -rf /var/lib/apt/lists/*
# Set working directory
WORKDIR /app
# Copy dependency files
COPY pyproject.toml /app/
COPY README.md /app/
COPY LICENSE /app/
COPY backend/requirements.txt /app/backend/requirements.txt
# Install Python dependencies (the slow part - cached in base image)
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir -r backend/requirements.txt
# Copy and install karaoke_gen package (dependencies cached above)
COPY karaoke_gen /app/karaoke_gen
RUN pip install --no-cache-dir -e .
# Label with build info
ARG BUILD_DATE
ARG BUILD_ID
LABEL org.opencontainers.image.created="${BUILD_DATE}"
LABEL org.opencontainers.image.revision="${BUILD_ID}"
LABEL org.opencontainers.image.title="karaoke-backend-base"
LABEL org.opencontainers.image.description="Base image with Python dependencies for karaoke-backend"
+35 -6
View File
@@ -59,14 +59,43 @@ class StorageService:
raise raise
def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str: def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str:
"""Generate a signed URL for downloading a file.""" """Generate a signed URL for downloading a file.
In Cloud Run, this uses the IAM signBlob API since we don't have
a private key available. Requires the service account to have
roles/iam.serviceAccountTokenCreator on itself.
"""
import google.auth
from google.auth.transport import requests
try: try:
blob = self.bucket.blob(blob_path) blob = self.bucket.blob(blob_path)
url = blob.generate_signed_url(
version="v4", # Get default credentials and refresh to ensure we have a valid token
expiration=timedelta(minutes=expiration_minutes), credentials, project = google.auth.default()
method="GET"
) # Check if we're using compute credentials (Cloud Run/GCE)
# These need to use IAM signBlob via service_account_email + access_token
if hasattr(credentials, 'service_account_email'):
# Refresh credentials to get a valid access token
auth_request = requests.Request()
credentials.refresh(auth_request)
url = blob.generate_signed_url(
version="v4",
expiration=timedelta(minutes=expiration_minutes),
method="GET",
service_account_email=credentials.service_account_email,
access_token=credentials.token
)
else:
# Local development with service account key or user credentials
url = blob.generate_signed_url(
version="v4",
expiration=timedelta(minutes=expiration_minutes),
method="GET"
)
logger.info(f"Generated signed URL for {blob_path}") logger.info(f"Generated signed URL for {blob_path}")
return url return url
except Exception as e: except Exception as e:
+45
View File
@@ -0,0 +1,45 @@
# Cloud Build configuration for base image
#
# This builds the base image with all Python dependencies pre-installed.
# Run this manually when requirements.txt changes:
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
#
# The base image is used by the main Dockerfile for fast app builds.
steps:
# Build base image with all dependencies
- name: 'gcr.io/cloud-builders/docker'
args:
- 'build'
- '--build-arg'
- 'BUILD_DATE=$BUILD_ID'
- '--build-arg'
- 'BUILD_ID=$BUILD_ID'
- '-t'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest'
- '-t'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID'
- '-f'
- 'backend/Dockerfile.base'
- '.'
# Push base image
- name: 'gcr.io/cloud-builders/docker'
args:
- 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest'
- name: 'gcr.io/cloud-builders/docker'
args:
- 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID'
images:
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID'
options:
machineType: 'N1_HIGHCPU_32'
diskSizeGb: 100
timeout: 1800s
+33 -13
View File
@@ -1,13 +1,24 @@
# Cloud Build configuration for karaoke-backend
#
# This builds and deploys the application using the pre-built base image.
# Build time: ~1-2 minutes (vs ~8 minutes without base image)
#
# Prerequisites:
# - Base image must exist: gcloud builds submit --config=cloudbuild-base.yaml
#
# Trigger manually:
# gcloud builds submit --config=cloudbuild.yaml --project=nomadkaraoke
steps: steps:
# Pull the previous image for caching # Pull the previous app image for layer caching
- name: 'gcr.io/cloud-builders/docker' - name: 'gcr.io/cloud-builders/docker'
entrypoint: 'bash' entrypoint: 'bash'
args: args:
- '-c' - '-c'
- | - |
docker pull us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest || exit 0 docker pull us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest || exit 0
# Build with cache # Build app image (fast - just copies backend code onto base image)
- name: 'gcr.io/cloud-builders/docker' - name: 'gcr.io/cloud-builders/docker'
args: args:
- 'build' - 'build'
@@ -20,19 +31,19 @@ steps:
- '-f' - '-f'
- 'backend/Dockerfile' - 'backend/Dockerfile'
- '.' - '.'
# Push images to Artifact Registry (must happen before deploy) # Push images to Artifact Registry
- name: 'gcr.io/cloud-builders/docker' - name: 'gcr.io/cloud-builders/docker'
args: args:
- 'push' - 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:$BUILD_ID' - 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:$BUILD_ID'
- name: 'gcr.io/cloud-builders/docker' - name: 'gcr.io/cloud-builders/docker'
args: args:
- 'push' - 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest' - 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
# Deploy to Cloud Run (after images are pushed) # Deploy to Cloud Run
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
entrypoint: gcloud entrypoint: gcloud
args: args:
@@ -45,24 +56,33 @@ steps:
- 'us-central1' - 'us-central1'
- '--platform' - '--platform'
- 'managed' - 'managed'
- '--allow-unauthenticated'
- '--memory' - '--memory'
- '4Gi' - '16Gi'
- '--cpu' - '--cpu'
- '2' - '8'
- '--cpu-boost'
- '--execution-environment'
- 'gen2'
- '--timeout' - '--timeout'
- '900' - '1800'
- '--concurrency'
- '4'
- '--max-instances'
- '10'
- '--min-instances'
- '0'
- '--set-env-vars' - '--set-env-vars'
- 'GOOGLE_CLOUD_PROJECT=$PROJECT_ID,GCS_BUCKET_NAME=karaoke-gen-storage-$PROJECT_ID,FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run' - 'GOOGLE_CLOUD_PROJECT=$PROJECT_ID,GCS_BUCKET_NAME=karaoke-gen-storage-$PROJECT_ID,FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run'
- '--set-secrets' - '--set-secrets'
- 'AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest' - 'AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest'
images: images:
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest' - 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:$BUILD_ID' - 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:$BUILD_ID'
options: options:
machineType: 'E2_HIGHCPU_8' machineType: 'N1_HIGHCPU_32'
diskSizeGb: 100 diskSizeGb: 100
timeout: 1200s timeout: 1200s
+26 -22
View File
@@ -218,29 +218,33 @@ handle_review() {
open_review_ui "$job_id" "$auth_token" open_review_ui "$job_id" "$auth_token"
echo -e "${YELLOW}Press Enter when you have completed the review...${NC}" echo -e "${YELLOW}Waiting for review to be completed in the browser...${NC}"
read -r echo -e "${CYAN}(Polling for status change every ${POLL_INTERVAL}s)${NC}"
# After review, trigger complete-review endpoint
echo -e "Completing review..."
local response
response=$(curl -s -X POST \
-H "Authorization: Bearer $auth_token" \
-H "Content-Type: application/json" \
"$SERVICE_URL/api/jobs/$job_id/complete-review")
local status
status=$(echo "$response" | jq -r '.status // "error"')
if [ "$status" == "success" ]; then
echo -e "${GREEN}✓ Review completed${NC}"
else
echo -e "${RED}Warning: Could not complete review:${NC}"
echo "$response" | jq .
echo ""
echo "The job may have already been completed or there was an error."
fi
echo "" echo ""
# Poll until job status changes from awaiting_review/in_review
while true; do
local job_response
job_response=$(get_job "$job_id" "$auth_token")
local current_status
current_status=$(echo "$job_response" | jq -r '.status // "unknown"')
case "$current_status" in
awaiting_review|in_review)
# Still in review, keep waiting
printf "\r\033[K Status: %s - still waiting for review completion..." "$current_status"
sleep "$POLL_INTERVAL"
;;
*)
# Status changed, review is complete
printf "\r\033[K"
echo -e "${GREEN}✓ Review completed (status: $current_status)${NC}"
echo ""
return 0
;;
esac
done
} }
# Handle instrumental selection # Handle instrumental selection