Optimize CI/CD: faster builds with base image and Cloud Build

- Add .gcloudignore to reduce upload from 43K files to 317 files (99% less)
- Create Dockerfile.base with pre-built dependencies (~15 min, only when deps change)
- Update Dockerfile to use base image (app builds now ~1-2 min)
- Upgrade Cloud Build to N1_HIGHCPU_32 for faster compilation
- Move Docker build from GitHub Actions to Cloud Build (faster GCP network)
- Fix signed URL generation for Cloud Run (use IAM signBlob API)
- Update submit-job.sh to poll for review completion instead of waiting for Enter
- Sync Cloud Run deploy settings (16Gi RAM, 8 CPU, gen2, proper secrets)
This commit is contained in:
Andrew Beveridge committed 2025-12-08 03:00:22 -05:00
1 parent 0b266decb3
commit 58ccc4324e
8 files changed
+284 -107

No files matched your search

+73
View File
@@ -0,0 +1,73 @@
# .gcloudignore - Exclude files from Cloud Build uploads
# This dramatically reduces upload time (from 218MB to ~5-10MB)
# Git
.git/
.gitignore
.gitmodules
# Node.js / Frontend
node_modules/
frontend-react/node_modules/
frontend-react/dist/
frontend-react/.vite/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
.venv/
venv/
ENV/
env/
*.egg-info/
.eggs/
dist/
build/
*.egg
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
.DS_Store
# Testing
.pytest_cache/
.coverage
htmlcov/
.tox/
.nox/
coverage.xml
*.cover
# Documentation (not needed for Docker build)
docs/
# Local development files
lyrics_transcriber_local/
*.local
.env
.env.*
# Large test files
tests/data/*.mp3
tests/data/*.wav
tests/data/*.mp4
tests/data/*.mkv
# Infrastructure (not needed for Docker build)
infrastructure/
# Misc
*.bak
*.tmp
Thumbs.db
+25 -39
View File
@@ -131,7 +131,7 @@ jobs:
echo "✅ Basic syntax validation passed"
deploy:
name: Deploy to Cloud Run
name: Deploy via Cloud Build
runs-on: ubuntu-latest
needs: [unit-tests, emulator-tests, lint]
# Only deploy on push to the branch (not on PRs)
@@ -159,49 +159,36 @@ jobs:
with:
version: 'latest'
- name: Configure Docker for Artifact Registry
- name: Trigger Cloud Build
run: |
gcloud auth configure-docker us-central1-docker.pkg.dev
echo "🚀 Triggering Cloud Build for deployment..."
echo "Commit: ${{ github.sha }}"
- name: Build and push Docker images
run: |
# Build backend image (context must be repo root for pyproject.toml and karaoke_gen/)
docker build -t us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest \
-t us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:${{ github.sha }} \
-f backend/Dockerfile .
# Submit build and wait for completion
BUILD_ID=$(gcloud builds submit \
--config=cloudbuild.yaml \
--project=nomadkaraoke \
--substitutions=_COMMIT_SHA=${{ github.sha }} \
--format='value(id)' \
--async \
.)
# Push both tags
docker push us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest
docker push us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:${{ github.sha }}
echo "Build ID: $BUILD_ID"
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke"
echo "✅ Images pushed to Artifact Registry"
# Wait for build to complete
echo "Waiting for build to complete..."
gcloud builds log $BUILD_ID --project=nomadkaraoke --stream
- name: Deploy to Cloud Run
run: |
gcloud run deploy karaoke-backend \
--image us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest \
--platform managed \
--region us-central1 \
--allow-unauthenticated \
--set-env-vars "GOOGLE_CLOUD_PROJECT=nomadkaraoke" \
--set-env-vars "GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke" \
--set-env-vars "FIRESTORE_COLLECTION=jobs" \
--set-env-vars "ENVIRONMENT=production" \
--set-env-vars "ADMIN_TOKENS=${{ secrets.ADMIN_TOKENS }}" \
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest" \
--set-secrets "GENIUS_API_TOKEN=genius-api-key:latest" \
--set-secrets "SPOTIFY_COOKIE=spotify-cookie:latest" \
--set-secrets "RAPIDAPI_KEY=rapidapi-key:latest" \
--memory 16Gi \
--cpu 8 \
--cpu-boost \
--execution-environment gen2 \
--timeout 1800 \
--concurrency 4 \
--max-instances 10 \
--min-instances 0
# Check build status
BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)')
echo "✅ Deployed to Cloud Run"
if [ "$BUILD_STATUS" != "SUCCESS" ]; then
echo "❌ Build failed with status: $BUILD_STATUS"
exit 1
fi
echo "✅ Build completed successfully"
- name: Verify deployment
run: |
@@ -226,4 +213,3 @@ jobs:
echo "Commit: ${{ github.sha }}"
echo "Branch: ${{ github.ref_name }}"
echo "Run: ${{ github.run_number }}"
+15 -36
View File
@@ -1,43 +1,22 @@
# Use Python 3.11 slim image
FROM python:3.11-slim
# Fast application image - uses pre-built base with all dependencies
#
# The base image (karaoke-backend-base) contains:
# - Python 3.11, ffmpeg, sox, system deps
# - All pip dependencies from requirements.txt
# - The karaoke_gen package
#
# This Dockerfile only copies the backend code, making builds very fast (~30s).
#
# If the base image doesn't exist yet, build it first:
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
# Install system dependencies required by karaoke-gen
RUN apt-get update && apt-get install -y \
ffmpeg \
libsndfile1 \
libsox-dev \
sox \
build-essential \
&& rm -rf /var/lib/apt/lists/*
# Use pre-built base image with all dependencies
FROM us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
# Set working directory
# Set working directory (should already exist from base)
WORKDIR /app
# ============================================
# OPTIMIZATION: Install dependencies BEFORE copying code
# This allows Docker to cache the dependency layer
# ============================================
# Copy ONLY dependency files first
COPY pyproject.toml /app/
COPY README.md /app/
COPY LICENSE /app/
COPY backend/requirements.txt /app/backend/requirements.txt
# Install Python dependencies (this layer will be cached unless requirements change)
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir -r backend/requirements.txt
# Now copy the karaoke_gen package and install it
# This includes karaoke_gen dependencies
COPY karaoke_gen /app/karaoke_gen
RUN pip install --no-cache-dir -e .
# ============================================
# Copy application code LAST (changes frequently)
# ============================================
# Copy backend code (this invalidates cache only for code changes)
# Copy backend code (this is the only thing that changes frequently)
COPY backend /app/backend
# Set environment variables
+41
View File
@@ -0,0 +1,41 @@
# Base image with all Python dependencies pre-installed
# This image is rebuilt only when requirements.txt changes
#
# Build with: gcloud builds submit --config=cloudbuild-base.yaml
# Or manually: docker build -f backend/Dockerfile.base -t karaoke-backend-base .
FROM python:3.11-slim
# Install system dependencies required by karaoke-gen
RUN apt-get update && apt-get install -y \
ffmpeg \
libsndfile1 \
libsox-dev \
sox \
build-essential \
&& rm -rf /var/lib/apt/lists/*
# Set working directory
WORKDIR /app
# Copy dependency files
COPY pyproject.toml /app/
COPY README.md /app/
COPY LICENSE /app/
COPY backend/requirements.txt /app/backend/requirements.txt
# Install Python dependencies (the slow part - cached in base image)
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir -r backend/requirements.txt
# Copy and install karaoke_gen package (dependencies cached above)
COPY karaoke_gen /app/karaoke_gen
RUN pip install --no-cache-dir -e .
# Label with build info
ARG BUILD_DATE
ARG BUILD_ID
LABEL org.opencontainers.image.created="${BUILD_DATE}"
LABEL org.opencontainers.image.revision="${BUILD_ID}"
LABEL org.opencontainers.image.title="karaoke-backend-base"
LABEL org.opencontainers.image.description="Base image with Python dependencies for karaoke-backend"
+30 -1
View File
@@ -59,14 +59,43 @@ class StorageService:
raise
def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str:
"""Generate a signed URL for downloading a file."""
"""Generate a signed URL for downloading a file.
In Cloud Run, this uses the IAM signBlob API since we don't have
a private key available. Requires the service account to have
roles/iam.serviceAccountTokenCreator on itself.
"""
import google.auth
from google.auth.transport import requests
try:
blob = self.bucket.blob(blob_path)
# Get default credentials and refresh to ensure we have a valid token
credentials, project = google.auth.default()
# Check if we're using compute credentials (Cloud Run/GCE)
# These need to use IAM signBlob via service_account_email + access_token
if hasattr(credentials, 'service_account_email'):
# Refresh credentials to get a valid access token
auth_request = requests.Request()
credentials.refresh(auth_request)
url = blob.generate_signed_url(
version="v4",
expiration=timedelta(minutes=expiration_minutes),
method="GET",
service_account_email=credentials.service_account_email,
access_token=credentials.token
)
else:
# Local development with service account key or user credentials
url = blob.generate_signed_url(
version="v4",
expiration=timedelta(minutes=expiration_minutes),
method="GET"
)
logger.info(f"Generated signed URL for {blob_path}")
return url
except Exception as e:
+45
View File
@@ -0,0 +1,45 @@
# Cloud Build configuration for base image
#
# This builds the base image with all Python dependencies pre-installed.
# Run this manually when requirements.txt changes:
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
#
# The base image is used by the main Dockerfile for fast app builds.
steps:
# Build base image with all dependencies
- name: 'gcr.io/cloud-builders/docker'
args:
- 'build'
- '--build-arg'
- 'BUILD_DATE=$BUILD_ID'
- '--build-arg'
- 'BUILD_ID=$BUILD_ID'
- '-t'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest'
- '-t'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID'
- '-f'
- 'backend/Dockerfile.base'
- '.'
# Push base image
- name: 'gcr.io/cloud-builders/docker'
args:
- 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest'
- name: 'gcr.io/cloud-builders/docker'
args:
- 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID'
images:
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID'
options:
machineType: 'N1_HIGHCPU_32'
diskSizeGb: 100
timeout: 1800s
+30 -10
View File
@@ -1,5 +1,16 @@
# Cloud Build configuration for karaoke-backend
#
# This builds and deploys the application using the pre-built base image.
# Build time: ~1-2 minutes (vs ~8 minutes without base image)
#
# Prerequisites:
# - Base image must exist: gcloud builds submit --config=cloudbuild-base.yaml
#
# Trigger manually:
# gcloud builds submit --config=cloudbuild.yaml --project=nomadkaraoke
steps:
# Pull the previous image for caching
# Pull the previous app image for layer caching
- name: 'gcr.io/cloud-builders/docker'
entrypoint: 'bash'
args:
@@ -7,7 +18,7 @@ steps:
- |
docker pull us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest || exit 0
# Build with cache
# Build app image (fast - just copies backend code onto base image)
- name: 'gcr.io/cloud-builders/docker'
args:
- 'build'
@@ -21,7 +32,7 @@ steps:
- 'backend/Dockerfile'
- '.'
# Push images to Artifact Registry (must happen before deploy)
# Push images to Artifact Registry
- name: 'gcr.io/cloud-builders/docker'
args:
- 'push'
@@ -32,7 +43,7 @@ steps:
- 'push'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
# Deploy to Cloud Run (after images are pushed)
# Deploy to Cloud Run
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
entrypoint: gcloud
args:
@@ -45,24 +56,33 @@ steps:
- 'us-central1'
- '--platform'
- 'managed'
- '--allow-unauthenticated'
- '--memory'
- '4Gi'
- '16Gi'
- '--cpu'
- '2'
- '8'
- '--cpu-boost'
- '--execution-environment'
- 'gen2'
- '--timeout'
- '900'
- '1800'
- '--concurrency'
- '4'
- '--max-instances'
- '10'
- '--min-instances'
- '0'
- '--set-env-vars'
- 'GOOGLE_CLOUD_PROJECT=$PROJECT_ID,GCS_BUCKET_NAME=karaoke-gen-storage-$PROJECT_ID,FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run'
- '--set-secrets'
- 'AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest'
- 'AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest'
images:
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:$BUILD_ID'
options:
machineType: 'E2_HIGHCPU_8'
machineType: 'N1_HIGHCPU_32'
diskSizeGb: 100
timeout: 1200s
+25 -21
View File
@@ -218,29 +218,33 @@ handle_review() {
open_review_ui "$job_id" "$auth_token"
echo -e "${YELLOW}Press Enter when you have completed the review...${NC}"
read -r
# After review, trigger complete-review endpoint
echo -e "Completing review..."
local response
response=$(curl -s -X POST \
-H "Authorization: Bearer $auth_token" \
-H "Content-Type: application/json" \
"$SERVICE_URL/api/jobs/$job_id/complete-review")
local status
status=$(echo "$response" | jq -r '.status // "error"')
if [ "$status" == "success" ]; then
echo -e "${GREEN}✓ Review completed${NC}"
else
echo -e "${RED}Warning: Could not complete review:${NC}"
echo "$response" | jq .
echo -e "${YELLOW}Waiting for review to be completed in the browser...${NC}"
echo -e "${CYAN}(Polling for status change every ${POLL_INTERVAL}s)${NC}"
echo ""
echo "The job may have already been completed or there was an error."
fi
# Poll until job status changes from awaiting_review/in_review
while true; do
local job_response
job_response=$(get_job "$job_id" "$auth_token")
local current_status
current_status=$(echo "$job_response" | jq -r '.status // "unknown"')
case "$current_status" in
awaiting_review|in_review)
# Still in review, keep waiting
printf "\r\033[K Status: %s - still waiting for review completion..." "$current_status"
sleep "$POLL_INTERVAL"
;;
*)
# Status changed, review is complete
printf "\r\033[K"
echo -e "${GREEN}✓ Review completed (status: $current_status)${NC}"
echo ""
return 0
;;
esac
done
}
# Handle instrumental selection