Files
karaoke-gen/.github/workflows/ci.yml
T
Andrew Beveridge ☄️andClaude Opus 4.6 9ccad7c6d3 feat: service health status modal with blue-green deploy details (#645)
## Summary
- Clickable footer opens a modal showing all service health, versions,
deploy times, and PR links
- Progressive disclosure: basic info for everyone, blue-green encoder
details for admins
- New aggregated `/api/health/system-status` endpoint (single fetch
instead of 4 separate calls)
- CI captures git SHA and PR info, passes to frontend and backend builds

## Changes
- **Backend**: New `/api/health/system-status` endpoint with optional
admin auth for blue-green Firestore config
- **Backend**: Build metadata env vars (COMMIT_SHA, PR_NUMBER, PR_TITLE,
STARTUP_TIME) in version module
- **Frontend**: New `SystemStatusModal` component with service cards and
expandable blue-green section
- **Frontend**: Footer version line is now clickable to open the modal
- **Frontend**: Build metadata env vars passed through next.config.mjs
- **CI**: Both frontend and backend deploy jobs capture and pass git SHA
+ PR info

## Testing
- [x] 3 unit tests for system-status endpoint (public, offline, admin
blue-green)
- [x] TypeScript compilation clean (no new errors)
- [x] CodeRabbit CLI reviewed

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-31 15:42:58 -04:00

1757 lines
69 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
types: [opened, synchronize, reopened, labeled]
workflow_dispatch:
permissions:
contents: write
id-token: write
concurrency:
group: "ci-${{ github.ref }}"
cancel-in-progress: true
jobs:
# ============================================
# PATH FILTERING - Determine what changed
# Only needed for PRs (main branch skips tests)
# ============================================
changes:
name: "Detect Changes"
runs-on: ubuntu-latest
if: github.ref != 'refs/heads/main'
outputs:
frontend: ${{ steps.filter.outputs.frontend }}
backend: ${{ steps.filter.outputs.backend }}
package: ${{ steps.filter.outputs.package }}
infra: ${{ steps.filter.outputs.infra }}
deps: ${{ steps.filter.outputs.deps }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
frontend:
- 'frontend/**'
backend:
- 'backend/**'
package:
- 'karaoke_gen/**'
- 'lyrics_transcriber_temp/**'
- 'tests/**'
infra:
- 'infrastructure/**'
deps:
- 'poetry.lock'
- name: Summary
run: |
echo "### Change Detection Results" >> $GITHUB_STEP_SUMMARY
echo "| Area | Changed |" >> $GITHUB_STEP_SUMMARY
echo "|------|---------|" >> $GITHUB_STEP_SUMMARY
echo "| Frontend | ${{ steps.filter.outputs.frontend }} |" >> $GITHUB_STEP_SUMMARY
echo "| Backend | ${{ steps.filter.outputs.backend }} |" >> $GITHUB_STEP_SUMMARY
echo "| Package | ${{ steps.filter.outputs.package }} |" >> $GITHUB_STEP_SUMMARY
echo "| Infrastructure | ${{ steps.filter.outputs.infra }} |" >> $GITHUB_STEP_SUMMARY
echo "| Dependencies | ${{ steps.filter.outputs.deps }} |" >> $GITHUB_STEP_SUMMARY
# ============================================
# FRONTEND TESTS (PRs only - main skips tests)
# ============================================
frontend-lint:
name: "Frontend - Lint (Rules of Hooks)"
runs-on: ubuntu-latest
needs: changes
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
steps:
- uses: actions/checkout@v4
- name: Setup Node.js with caching
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
- name: Run ESLint
working-directory: frontend
run: npm run lint
frontend-unit-tests:
name: "Frontend - Unit Tests"
runs-on: ubuntu-latest
needs: changes
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
steps:
- uses: actions/checkout@v4
- name: Setup Node.js with caching
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
- name: Run unit tests with coverage
working-directory: frontend
run: npm run test:ci
- name: Upload coverage report
if: always()
uses: actions/upload-artifact@v4
with:
name: frontend-coverage
path: frontend/coverage/
retention-days: 7
frontend-e2e-smoke:
name: "Frontend - E2E Smoke (~2 min)"
runs-on: ubuntu-latest
needs: changes
# Quick smoke tests run on every PR with frontend changes
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
steps:
- uses: actions/checkout@v4
- name: Setup Node.js with caching
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
- name: Install Playwright browsers
working-directory: frontend
run: npx playwright install --with-deps chromium
- name: Run smoke tests only
working-directory: frontend
run: npx playwright test smoke.spec.ts --project=chromium --reporter=line
- name: Upload Playwright report
if: failure()
uses: actions/upload-artifact@v4
with:
name: playwright-smoke-report
path: frontend/playwright-report/
retention-days: 7
frontend-e2e-full:
name: "Frontend - E2E Full"
runs-on: [self-hosted, linux, gcp]
needs: changes
# Full E2E suite only runs when PR has 'e2e' label (on-demand)
if: >-
github.ref != 'refs/heads/main' &&
needs.changes.outputs.frontend == 'true' &&
contains(github.event.pull_request.labels.*.name, 'e2e')
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- uses: actions/checkout@v4
- name: Setup Node.js with caching
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
- name: Install Playwright browsers
working-directory: frontend
run: npx playwright install --with-deps chromium
- name: Run full E2E tests
working-directory: frontend
run: npx playwright test --reporter=list -x
- name: Upload Playwright report
if: failure()
uses: actions/upload-artifact@v4
with:
name: playwright-report
path: frontend/playwright-report/
retention-days: 7
# Happy Path Production Test - runs the comprehensive real-user test against prod
frontend-e2e-happy-path:
name: "Frontend - E2E Happy Path (Production)"
runs-on: ubuntu-latest
needs: changes
# Only runs when PR has 'e2e-happy-path' label
if: >-
github.ref != 'refs/heads/main' &&
contains(github.event.pull_request.labels.*.name, 'e2e-happy-path')
timeout-minutes: 90
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
- name: Install Playwright browsers
working-directory: frontend
run: npx playwright install --with-deps chromium
- name: Create test-results directory
working-directory: frontend
run: mkdir -p test-results
- name: Refresh E2E session token
id: refresh_token
run: |
RESPONSE=$(curl -sf -X POST \
"https://api.nomadkaraoke.com/api/admin/users/e2e-test-runner@nomadkaraoke.com/impersonate" \
-H "X-Admin-Token: ${{ secrets.E2E_ADMIN_TOKEN }}" \
-H "Content-Type: application/json")
TOKEN=$(echo "$RESPONSE" | python3 -c 'import json,sys; print(json.load(sys.stdin)["session_token"])')
echo "::add-mask::$TOKEN"
echo "token=$TOKEN" >> $GITHUB_OUTPUT
- name: Run Happy Path Production Test
working-directory: frontend
env:
TESTMAIL_API_KEY: ${{ secrets.TESTMAIL_API_KEY }}
TESTMAIL_NAMESPACE: ${{ secrets.TESTMAIL_NAMESPACE }}
E2E_TEST_TOKEN: ${{ steps.refresh_token.outputs.token }}
E2E_ADMIN_TOKEN: ${{ secrets.E2E_ADMIN_TOKEN }}
E2E_BYPASS_KEY: ${{ secrets.E2E_BYPASS_KEY }}
run: |
echo "Starting E2E Happy Path Test against Production"
echo "Time: $(date)"
npx playwright test happy-path-real-user.spec.ts \
--config=playwright.production.config.ts \
--project=chromium \
--reporter=list \
--timeout=3600000
- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-happy-path-results
path: |
frontend/test-results/
frontend/playwright-report/
retention-days: 30
frontend-build:
name: "Frontend - Build"
runs-on: ubuntu-latest
needs: changes
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
steps:
- uses: actions/checkout@v4
- name: Setup Node.js with caching
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
- name: Build frontend
working-directory: frontend
env:
NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com
run: npm run build
# ============================================
# PACKAGE TESTS (PRs only - main skips tests)
# ============================================
package-unit-tests:
name: "Package - Unit Tests (70% coverage)"
runs-on: [self-hosted, linux, gcp]
needs: changes
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
restore-keys: |
venv-package-${{ runner.os }}-py3.13-
- name: Install dependencies
run: poetry install --with dev
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
- name: Cache spaCy model
uses: actions/cache@v4
with:
path: ~/.cache/spacy
key: spacy-en_core_web_sm-${{ runner.os }}
- name: Install spaCy language models
run: poetry run python -m spacy download en_core_web_sm
- name: Run unit tests
run: |
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=69
- name: Upload coverage reports
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: unittests
name: codecov-umbrella
package-integration-tests:
name: "Package - Integration Tests"
runs-on: [self-hosted, linux, gcp]
needs: changes
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
restore-keys: |
venv-package-${{ runner.os }}-py3.13-
- name: Install dependencies
run: poetry install --with dev
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
- name: Cache spaCy model
uses: actions/cache@v4
with:
path: ~/.cache/spacy
key: spacy-en_core_web_sm-${{ runner.os }}
- name: Install spaCy language models
run: poetry run python -m spacy download en_core_web_sm
- name: Run integration tests
run: poetry run pytest tests/integration/ -v
package-build-test:
name: "Package - Build & Install (Test)"
runs-on: [self-hosted, linux, gcp]
needs: [changes, package-unit-tests, package-integration-tests]
if: >-
github.ref != 'refs/heads/main' && always() &&
(needs.package-unit-tests.result == 'success' || needs.package-unit-tests.result == 'skipped') &&
(needs.package-integration-tests.result == 'success' || needs.package-integration-tests.result == 'skipped') &&
(needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Build package
run: poetry build
- name: Test package installation
run: |
pip install dist/*.whl
python -m karaoke_gen.utils.gen_cli --help
# ============================================
# BACKEND TESTS (PRs only - main skips tests)
# ============================================
backend-unit-tests:
name: "Backend - Unit Tests"
runs-on: [self-hosted, linux, gcp]
needs: changes
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-backend-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
restore-keys: |
venv-backend-${{ runner.os }}-py3.13-
- name: Install dependencies
run: poetry install
- name: Run all backend unit tests
env:
GOOGLE_CLOUD_PROJECT: test-project
GCS_BUCKET_NAME: test-bucket
FIRESTORE_COLLECTION: jobs
ENVIRONMENT: test
run: |
poetry run pytest \
backend/tests/ \
--ignore=backend/tests/emulator \
-v --tb=short
backend-emulator-tests:
name: "Backend - Emulator Integration Tests"
runs-on: [self-hosted, linux, gcp]
needs: changes
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-backend-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
restore-keys: |
venv-backend-${{ runner.os }}-py3.13-
- name: Set up Java 21 (required by Firestore emulator)
uses: actions/setup-java@v4
with:
distribution: "temurin"
java-version: "21"
- name: Install Firestore emulator
run: sudo apt-get install -y google-cloud-cli-firestore-emulator
- name: Install dependencies
run: poetry install
- name: Wait for Docker to be available
run: |
# Self-hosted runners may still be initializing Docker on boot.
# Wait up to 60s for the docker command and daemon to be ready.
for i in $(seq 1 12); do
if command -v docker &>/dev/null && docker info &>/dev/null 2>&1; then
echo "Docker is ready"
exit 0
fi
echo "Waiting for Docker... (attempt $i/12)"
sleep 5
done
echo "ERROR: Docker not available after 60s"
echo "PATH=$PATH"
which docker 2>&1 || echo "docker binary not found"
exit 1
- name: Pull GCS emulator image
run: docker pull fsouza/fake-gcs-server:latest
- name: Run emulator integration tests
run: |
chmod +x scripts/*.sh
./scripts/run-emulator-tests.sh
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: backend-test-results
path: backend/htmlcov/
retention-days: 7
backend-lint:
name: "Backend - Code Quality"
runs-on: ubuntu-latest
needs: changes
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true')
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Run validation (syntax check only)
run: |
python -m py_compile backend/main.py backend/config.py
echo "Basic syntax validation passed"
# ============================================
# CI GATE - Required status check for PRs
# Only runs on PRs, not on main
# ============================================
ci-gate:
name: "CI Gate"
runs-on: ubuntu-latest
# frontend-e2e-smoke IS required - fast smoke tests catch critical regressions
# frontend-lint IS required - catches React hooks violations that cause runtime crashes
# frontend-e2e-happy-path is required ONLY when 'e2e-happy-path' label is present
# frontend-e2e-full is optional (requires 'e2e' label) so not enforced here
needs: [frontend-lint, frontend-unit-tests, frontend-e2e-smoke, frontend-build, package-unit-tests, package-integration-tests, backend-unit-tests, backend-emulator-tests, backend-lint, frontend-e2e-happy-path]
if: github.ref != 'refs/heads/main' && always()
steps:
- name: Check results
run: |
# Check all always-required jobs
results=(
"${{ needs.frontend-lint.result }}"
"${{ needs.frontend-unit-tests.result }}"
"${{ needs.frontend-e2e-smoke.result }}"
"${{ needs.frontend-build.result }}"
"${{ needs.package-unit-tests.result }}"
"${{ needs.package-integration-tests.result }}"
"${{ needs.backend-unit-tests.result }}"
"${{ needs.backend-emulator-tests.result }}"
"${{ needs.backend-lint.result }}"
)
for result in "${results[@]}"; do
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
echo "❌ CI failed"
exit 1
fi
done
# E2E Happy Path: enforced when 'e2e-happy-path' label is on the PR
e2e_result="${{ needs.frontend-e2e-happy-path.result }}"
has_label="${{ contains(github.event.pull_request.labels.*.name, 'e2e-happy-path') }}"
if [[ "$has_label" == "true" ]]; then
if [[ "$e2e_result" != "success" ]]; then
echo "❌ E2E Happy Path is required (label present) but result is: $e2e_result"
exit 1
fi
echo "✅ E2E Happy Path passed (required by label)"
else
echo "ℹ️ E2E Happy Path skipped (no label)"
fi
echo "✅ CI passed"
# ============================================
# DEPLOYMENTS (main branch only)
# Tests already passed on PR, so we skip them here
# ============================================
package-build:
name: "Package - Build"
runs-on: [self-hosted, linux, gcp]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- name: Check disk space
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 70 ]; then
echo "Disk usage high, running cleanup..."
docker system prune -af || true
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
restore-keys: |
venv-package-${{ runner.os }}-py3.13-
- name: Build package
run: poetry build
- name: Upload build artifacts
uses: actions/upload-artifact@v4
with:
name: package-dist
path: dist/
retention-days: 1
package-publish:
name: "Deploy - Publish to PyPI"
runs-on: ubuntu-latest
needs: [package-build]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: package-dist
path: dist/
- name: Extract version
id: extract_version
run: |
VERSION=$(poetry version --short)
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "📋 Extracted version: $VERSION"
- name: Check if tag exists
id: check_tag
run: |
VERSION="${{ steps.extract_version.outputs.version }}"
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
echo "tag_exists=true" >> $GITHUB_OUTPUT
echo "🏷️ Tag v$VERSION already exists, skipping release"
else
echo "tag_exists=false" >> $GITHUB_OUTPUT
echo "✨ Tag v$VERSION does not exist, proceeding with release"
fi
- name: Publish to PyPI
if: steps.check_tag.outputs.tag_exists == 'false'
run: |
echo "📦 Publishing the following files to PyPI:"
ls -la dist/
echo "🚀 Publishing to PyPI..."
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
poetry publish
- name: Create Git Tag
if: steps.check_tag.outputs.tag_exists == 'false'
run: |
git config user.name github-actions
git config user.email github-actions@github.com
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}"
- name: Create GitHub Release
if: steps.check_tag.outputs.tag_exists == 'false'
uses: softprops/action-gh-release@v1
with:
tag_name: "v${{ steps.extract_version.outputs.version }}"
name: "v${{ steps.extract_version.outputs.version }}"
body: |
## Changes in v${{ steps.extract_version.outputs.version }}
This release was automatically created from the main branch.
### Installation
```bash
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
```
### Assets
- Python package published to PyPI
- Frontend deployed to Cloudflare Pages at gen.nomadkaraoke.com
For detailed changes, see the commit history.
draft: false
prerelease: false
files: |
dist/*.whl
dist/*.tar.gz
deploy-frontend:
name: "Deploy - Frontend (Cloudflare Pages)"
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node.js with caching
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
# ===== Build Unified Frontend (Next.js) =====
# NOTE: Lyrics review and instrumental review UIs were consolidated into
# the main Next.js frontend (PR #267, Jan 2025). The unified frontend
# serves all three UIs from a single codebase at:
# - / (main app)
# - /app/jobs/{id}/lyrics (lyrics review)
# - /app/jobs/{id}/instrumental (instrumental review)
- name: Capture build metadata
id: frontend-meta
run: |
echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
PR_NUM=$(git log -1 --format=%s | grep -oP '\(#\K[0-9]+' | head -1 || echo "")
if [ -n "$PR_NUM" ]; then
PR_TITLE=$(gh pr view "$PR_NUM" --json title -q .title 2>/dev/null || echo "")
else
PR_NUM=""
PR_TITLE=""
fi
echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT
echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Install frontend dependencies
working-directory: frontend
run: npm ci --legacy-peer-deps
# ===== Consumer Build (gen.nomadkaraoke.com) =====
- name: Build frontend (consumer)
working-directory: frontend
env:
NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com
NEXT_PUBLIC_COMMIT_SHA: ${{ steps.frontend-meta.outputs.sha }}
NEXT_PUBLIC_PR_NUMBER: ${{ steps.frontend-meta.outputs.pr_number }}
NEXT_PUBLIC_PR_TITLE: ${{ steps.frontend-meta.outputs.pr_title }}
NEXT_PUBLIC_BUILD_TIME: ${{ github.event.head_commit.timestamp }}
run: npm run build
- name: Deploy consumer to Cloudflare Pages
uses: cloudflare/wrangler-action@v3
with:
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
command: pages deploy frontend/out --project-name=karaoke-gen --branch=main
# ===== Tenant Portal Build (vocalstar.nomadkaraoke.com, singa.nomadkaraoke.com, etc.) =====
# Separate build with NEXT_PUBLIC_TENANT_PORTAL=true which tree-shakes
# consumer marketing content and renders TenantLandingPage at root.
# The Cloudflare Pages Function (frontend/functions/) injects tenant config at the edge.
- name: Build frontend (tenant portal)
working-directory: frontend
env:
NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com
NEXT_PUBLIC_TENANT_PORTAL: "true"
NEXT_PUBLIC_COMMIT_SHA: ${{ steps.frontend-meta.outputs.sha }}
NEXT_PUBLIC_PR_NUMBER: ${{ steps.frontend-meta.outputs.pr_number }}
NEXT_PUBLIC_PR_TITLE: ${{ steps.frontend-meta.outputs.pr_title }}
NEXT_PUBLIC_BUILD_TIME: ${{ github.event.head_commit.timestamp }}
run: npm run build
- name: Deploy tenant portal to Cloudflare Pages
uses: cloudflare/wrangler-action@v3
with:
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
workingDirectory: frontend
command: pages deploy out --project-name=karaoke-gen-tenant --branch=main
# NOTE: buy-site was consolidated into the main frontend (PR #122, Dec 2024)
# Landing page and payment flow are now at gen.nomadkaraoke.com (root)
# buy.nomadkaraoke.com subdomain has been deleted
deploy-backend:
name: "Deploy - Backend (Cloud Run)"
needs: [package-build]
runs-on: [self-hosted, linux, gcp, docker-build]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
permissions:
contents: read
id-token: write
outputs:
base_rebuilt: ${{ steps.check-base.outputs.needs_rebuild }}
env:
REGISTRY: us-central1-docker.pkg.dev
PROJECT_ID: nomadkaraoke
REPOSITORY: karaoke-repo
BASE_IMAGE_NAME: karaoke-backend-base
APP_IMAGE_NAME: karaoke-backend
# GPU image for audio worker (L4 in us-east4)
GPU_REGISTRY: us-east4-docker.pkg.dev
GPU_REPOSITORY: karaoke-backend-gpu
GPU_BASE_IMAGE_NAME: karaoke-backend-gpu-base
GPU_APP_IMAGE_NAME: karaoke-backend
steps:
- name: Check disk space (aggressive for Docker builds)
run: |
echo "Checking disk space..."
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "Current disk usage: ${USAGE}%"
if [ "$USAGE" -gt 50 ]; then
echo "Disk usage above 50%, cleaning unused images..."
docker system prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After image cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 70 ]; then
echo "Still above 70%, clearing build cache..."
docker builder prune -af || true
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
echo "After build cache cleanup: ${USAGE}%"
fi
if [ "$USAGE" -gt 85 ]; then
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
exit 1
fi
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: false
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Extract version
id: version_check
run: |
VERSION=$(poetry version --short)
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "📋 Current version: $VERSION"
echo "🚀 Deploying on every push to main"
- name: Capture build metadata
id: build-meta
run: |
echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
# For squash-merge commits, PR number is in the commit message as (#NNN)
PR_NUM=$(git log -1 --format=%s | grep -oP '\(#\K[0-9]+' | head -1 || echo "")
if [ -n "$PR_NUM" ]; then
PR_TITLE=$(gh pr view "$PR_NUM" --json title -q .title 2>/dev/null || echo "")
else
PR_NUM=""
PR_TITLE=""
fi
echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT
echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Authenticate to Google Cloud (Workload Identity)
id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
with:
version: "latest"
# ============================================
# Blue-Green Deploy to GCE Encoding Worker
#
# This pattern ensures zero-downtime deployments:
# 1. Upload wheel + startup.sh + version manifest to GCS
# 2. Read blue-green config from Firestore
# 3. Start secondary VM, wait for health + version match
# 4. Deep health check: submit a real encode test
# 5. Swap primary/secondary in Firestore (atomic transaction)
# 6. Drain and stop old primary
#
# See infrastructure/encoding-worker/README.md for details
# ============================================
- name: Download wheel artifact
uses: actions/download-artifact@v4
with:
name: package-dist
path: dist/
- name: Deploy to GCS
run: |
VERSION="${{ steps.version_check.outputs.version }}"
BUCKET="gs://karaoke-gen-storage-${{ env.PROJECT_ID }}"
echo "Deploying encoding worker v${VERSION}..."
# 1. Find the wheel file
WHEEL_FILE=$(ls dist/*.whl | head -1)
if [ -z "$WHEEL_FILE" ]; then
echo "::error::No wheel file found in dist/"
exit 1
fi
echo "Found wheel: $WHEEL_FILE"
# 2. Upload wheel to FIXED path (atomic via temp file)
# This eliminates version sorting - always use karaoke_gen-current.whl
echo "Uploading wheel to fixed path..."
gsutil cp "$WHEEL_FILE" "${BUCKET}/wheels/karaoke_gen-current.whl.tmp"
gsutil mv "${BUCKET}/wheels/karaoke_gen-current.whl.tmp" \
"${BUCKET}/wheels/karaoke_gen-current.whl"
# Also keep versioned copy for rollback/debugging
gsutil cp "$WHEEL_FILE" "${BUCKET}/wheels/"
# 3. Upload startup.sh (allows logic changes without image rebuild)
echo "Uploading startup.sh..."
gsutil cp infrastructure/encoding-worker/startup.sh \
"${BUCKET}/encoding-worker/startup.sh"
# 4. Write version manifest for verification
echo "Writing version manifest..."
echo "${VERSION}" | gsutil cp - "${BUCKET}/encoding-worker/version.txt"
echo "GCS deployment complete"
- name: Install Firestore client
run: pip install google-cloud-firestore
- name: Read Firestore config
id: config
run: |
python3 << 'PYEOF'
import json, os
from google.cloud import firestore
db = firestore.Client(project="nomadkaraoke")
doc = db.collection("config").document("encoding-worker").get()
if not doc.exists:
print("::error::Encoding worker config not found in Firestore")
raise SystemExit(1)
data = doc.to_dict()
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
for key in ["primary_vm", "primary_ip", "primary_version",
"secondary_vm", "secondary_ip"]:
f.write(f"{key}={data.get(key, '')}\n")
f.write(f"deploy_in_progress={str(data.get('deploy_in_progress', False)).lower()}\n")
print(f"Config: primary={data['primary_vm']}, secondary={data['secondary_vm']}")
PYEOF
- name: Wait for concurrent deploy
if: steps.config.outputs.deploy_in_progress == 'true'
run: |
echo "Another deploy is in progress, waiting..."
python3 << 'PYEOF'
import time
from google.cloud import firestore
db = firestore.Client(project="nomadkaraoke")
for i in range(20):
time.sleep(30)
doc = db.collection("config").document("encoding-worker").get()
if not doc.to_dict().get("deploy_in_progress", False):
print("Concurrent deploy finished, proceeding")
break
else:
print("::error::Timed out waiting for concurrent deploy (10 min)")
raise SystemExit(1)
PYEOF
- name: Set deploy in progress
run: |
python3 << 'PYEOF'
from datetime import datetime, timezone
from google.cloud import firestore
db = firestore.Client(project="nomadkaraoke")
db.collection("config").document("encoding-worker").update({
"deploy_in_progress": True,
"deploy_in_progress_since": datetime.now(timezone.utc).isoformat(),
})
print("Set deploy_in_progress=true")
PYEOF
- name: Start secondary VM
run: |
SECONDARY_VM="${{ steps.config.outputs.secondary_vm }}"
echo "Starting secondary VM: $SECONDARY_VM"
gcloud compute instances start "$SECONDARY_VM" \
--zone=us-central1-c --project=nomadkaraoke || true
for i in $(seq 1 12); do
STATUS=$(gcloud compute instances describe "$SECONDARY_VM" \
--zone=us-central1-c --project=nomadkaraoke \
--format='value(status)')
echo "VM status: $STATUS (attempt $i/12)"
if [ "$STATUS" = "RUNNING" ]; then
break
fi
sleep 10
done
- name: Wait for service health
id: health
continue-on-error: true
run: |
SECONDARY_IP="${{ steps.config.outputs.secondary_ip }}"
EXPECTED_VERSION="${{ steps.version_check.outputs.version }}"
HEALTH_URL="http://${SECONDARY_IP}:8080/health"
echo "Waiting for service at $HEALTH_URL..."
for i in $(seq 1 18); do
RESPONSE=$(curl -sf "$HEALTH_URL" 2>/dev/null || echo "")
if [ -n "$RESPONSE" ]; then
WHEEL_VERSION=$(echo "$RESPONSE" | jq -r '.wheel_version // empty')
echo "Health: version=$WHEEL_VERSION"
if [ "$WHEEL_VERSION" = "$EXPECTED_VERSION" ]; then
echo "Version matches! Service is healthy."
echo "healthy=true" >> $GITHUB_OUTPUT
exit 0
fi
fi
echo "Attempt $i/18 - waiting 10s..."
sleep 10
done
echo "::error::Service did not become healthy within 3 minutes"
echo "healthy=false" >> $GITHUB_OUTPUT
exit 1
- name: Deep health check - real encode test
if: steps.health.outputs.healthy == 'true'
id: encode-test
continue-on-error: true
run: |
SECONDARY_IP="${{ steps.config.outputs.secondary_ip }}"
API_KEY=$(gcloud secrets versions access latest \
--secret=encoding-worker-api-key --project=nomadkaraoke)
DEPLOY_TEST_PATH="gs://karaoke-gen-storage-nomadkaraoke/deploy-test"
echo "Uploading test assets to GCS..."
gsutil cp infrastructure/encoding-worker/test-assets/test-preview.ass \
"${DEPLOY_TEST_PATH}/test-preview.ass"
gsutil cp infrastructure/encoding-worker/test-assets/test-audio.flac \
"${DEPLOY_TEST_PATH}/test-audio.flac"
echo "Submitting test encode to $SECONDARY_IP..."
HTTP_CODE=$(curl -s -o /tmp/encode-response.json -w '%{http_code}' \
-X POST "http://${SECONDARY_IP}:8080/encode-preview" \
-H "X-API-Key: $API_KEY" \
-H "Content-Type: application/json" \
-d "{
\"job_id\": \"deploy-health-check\",
\"ass_gcs_path\": \"${DEPLOY_TEST_PATH}/test-preview.ass\",
\"audio_gcs_path\": \"${DEPLOY_TEST_PATH}/test-audio.flac\",
\"output_gcs_path\": \"${DEPLOY_TEST_PATH}/output-preview.mp4\"
}") || HTTP_CODE="000"
RESPONSE=$(cat /tmp/encode-response.json 2>/dev/null || echo "{}")
echo "Encode response (HTTP $HTTP_CODE): $RESPONSE"
if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "202" ]; then
echo "::error::Encode request failed with HTTP $HTTP_CODE"
echo "passed=false" >> $GITHUB_OUTPUT
exit 1
fi
STATUS=$(echo "$RESPONSE" | jq -r '.status // "unknown"')
if [ "$STATUS" != "accepted" ] && [ "$STATUS" != "cached" ] && [ "$STATUS" != "in_progress" ]; then
echo "::error::Encode test failed with status: $STATUS"
echo "passed=false" >> $GITHUB_OUTPUT
exit 1
fi
for i in $(seq 1 12); do
sleep 5
JOB_STATUS=$(curl -s \
"http://${SECONDARY_IP}:8080/status/deploy-health-check" \
-H "X-API-Key: $API_KEY" 2>/dev/null | jq -r '.status // "unknown"' 2>/dev/null) || JOB_STATUS="unknown"
echo "Encode status: $JOB_STATUS (attempt $i/12)"
if [ "$JOB_STATUS" = "complete" ]; then
echo "Encode test passed!"
echo "passed=true" >> $GITHUB_OUTPUT
gsutil rm -f gs://karaoke-gen-storage-nomadkaraoke/deploy-test/** || true
exit 0
elif [ "$JOB_STATUS" = "failed" ]; then
echo "::error::Encode test failed"
echo "passed=false" >> $GITHUB_OUTPUT
exit 1
fi
done
echo "::error::Encode test timed out after 60s"
echo "passed=false" >> $GITHUB_OUTPUT
exit 1
- name: Rollback on failure
if: steps.health.outputs.healthy != 'true' || steps.encode-test.outputs.passed != 'true'
run: |
SECONDARY_VM="${{ steps.config.outputs.secondary_vm }}"
if [ -n "$SECONDARY_VM" ]; then
echo "Stopping secondary VM $SECONDARY_VM due to failed deploy"
gcloud compute instances stop "$SECONDARY_VM" \
--zone=us-central1-c --project=nomadkaraoke || true
fi
python3 -c "
from google.cloud import firestore
db = firestore.Client(project='nomadkaraoke')
db.collection('config').document('encoding-worker').update({
'deploy_in_progress': False,
'deploy_in_progress_since': None,
})
print('Cleared deploy_in_progress flag')
" || true
- name: Swap primary in Firestore
if: steps.encode-test.outputs.passed == 'true'
env:
NEW_PRIMARY: ${{ steps.config.outputs.secondary_vm }}
NEW_PRIMARY_IP: ${{ steps.config.outputs.secondary_ip }}
NEW_VERSION: ${{ steps.version_check.outputs.version }}
OLD_PRIMARY: ${{ steps.config.outputs.primary_vm }}
OLD_PRIMARY_IP: ${{ steps.config.outputs.primary_ip }}
OLD_VERSION: ${{ steps.config.outputs.primary_version }}
run: |
echo "Swapping: $OLD_PRIMARY -> secondary, $NEW_PRIMARY -> primary"
python3 << 'PYEOF'
import os
from datetime import datetime, timezone
from google.cloud import firestore
db = firestore.Client(project="nomadkaraoke")
doc_ref = db.collection("config").document("encoding-worker")
now = datetime.now(timezone.utc).isoformat()
@firestore.transactional
def swap(transaction):
transaction.update(doc_ref, {
"primary_vm": os.environ["NEW_PRIMARY"],
"primary_ip": os.environ["NEW_PRIMARY_IP"],
"primary_version": os.environ["NEW_VERSION"],
"primary_deployed_at": now,
"secondary_vm": os.environ["OLD_PRIMARY"],
"secondary_ip": os.environ["OLD_PRIMARY_IP"],
"secondary_version": os.environ["OLD_VERSION"],
"secondary_deployed_at": now,
"last_swap_at": now,
"deploy_in_progress": False,
"deploy_in_progress_since": None,
})
transaction = db.transaction()
swap(transaction)
print(f"Swapped: {os.environ['NEW_PRIMARY']} is now primary")
PYEOF
- name: Drain and stop old primary
if: steps.encode-test.outputs.passed == 'true'
run: |
OLD_PRIMARY="${{ steps.config.outputs.primary_vm }}"
OLD_PRIMARY_IP="${{ steps.config.outputs.primary_ip }}"
API_KEY=$(gcloud secrets versions access latest \
--secret=encoding-worker-api-key --project=nomadkaraoke)
ACTIVE_JOBS=$(curl -sf "http://${OLD_PRIMARY_IP}:8080/health" \
| jq -r '.active_jobs // 0' 2>/dev/null || echo "0")
if [ "$ACTIVE_JOBS" -gt 0 ]; then
echo "Old primary has $ACTIVE_JOBS active jobs, waiting for drain..."
DRAIN_TIMEOUT=600
ELAPSED=0
while [ $ELAPSED -lt $DRAIN_TIMEOUT ]; do
sleep 15
ELAPSED=$((ELAPSED + 15))
ACTIVE_JOBS=$(curl -sf "http://${OLD_PRIMARY_IP}:8080/health" \
| jq -r '.active_jobs // 0' 2>/dev/null || echo "0")
echo "Active jobs: $ACTIVE_JOBS (${ELAPSED}s elapsed)"
if [ "$ACTIVE_JOBS" -eq 0 ]; then
break
fi
done
fi
echo "Stopping old primary: $OLD_PRIMARY"
gcloud compute instances stop "$OLD_PRIMARY" \
--zone=us-central1-c --project=nomadkaraoke || true
- name: Configure Docker for Artifact Registry
run: |
gcloud auth configure-docker ${{ env.REGISTRY }} --quiet
gcloud auth configure-docker ${{ env.GPU_REGISTRY }} --quiet
# ============================================
# Docker Build with BuildKit + Registry Cache
# This replaces expensive Cloud Build with free GitHub Actions
# Registry cache persists between builds for fast rebuilds
# ============================================
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Calculate base image hash
id: base-hash
run: |
# Hash both poetry.lock AND Dockerfile.base - rebuild if either changes
CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
echo "hash=$CURRENT_HASH" >> $GITHUB_OUTPUT
echo "📋 Base content hash: $CURRENT_HASH"
- name: Check if base image exists and is current
id: check-base
run: |
BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest"
# Fetch hash label directly from registry (10KB config blob, not full image)
# This avoids issues with stale local Docker cache on GitHub runners
EXISTING_HASH=""
# Get the manifest (small JSON, ~1KB) - includes config digest
MANIFEST=$(docker manifest inspect $BASE_IMAGE 2>/dev/null || echo "")
CONFIG_DIGEST=$(echo "$MANIFEST" | jq -r '.config.digest // ""' 2>/dev/null || echo "")
if [ -n "$CONFIG_DIGEST" ] && [ "$CONFIG_DIGEST" != "null" ] && [ "$CONFIG_DIGEST" != "" ]; then
echo "📋 Found manifest, config digest: $CONFIG_DIGEST"
# Fetch just the config blob (~10KB) to get labels - much faster than pulling full image
TOKEN=$(gcloud auth print-access-token)
CONFIG_JSON=$(curl -sL -H "Authorization: Bearer $TOKEN" \
"https://${{ env.REGISTRY }}/v2/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}/blobs/$CONFIG_DIGEST" 2>/dev/null)
EXISTING_HASH=$(echo "$CONFIG_JSON" | jq -r '.config.Labels["base.content.hash"] // .config.Labels["poetry.lock.hash"] // ""' 2>/dev/null || echo "")
echo "📋 Registry base image hash: $EXISTING_HASH"
else
echo "⚠️ No existing base image found in registry - will build"
fi
if [ "${{ steps.base-hash.outputs.hash }}" != "$EXISTING_HASH" ]; then
echo "🔄 Base image needs rebuild (poetry.lock or Dockerfile.base changed)"
echo " Current: ${{ steps.base-hash.outputs.hash }}"
echo " Registry: $EXISTING_HASH"
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
else
echo "✅ Base image is up to date"
echo "needs_rebuild=false" >> $GITHUB_OUTPUT
fi
- name: Build and push base image (if needed)
if: steps.check-base.outputs.needs_rebuild == 'true'
timeout-minutes: 20
uses: docker/build-push-action@v6
with:
context: .
file: backend/Dockerfile.base
push: true
tags: |
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:${{ github.sha }}
# Registry cache for ALL layers (mode=max) - persists between builds
# ignore-error=true allows build to continue if cache doesn't exist yet
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:cache,ignore-error=true
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:cache,mode=max,ignore-error=true
build-args: |
BUILD_DATE=${{ github.run_id }}
BUILD_ID=${{ github.sha }}
BASE_CONTENT_HASH=${{ steps.base-hash.outputs.hash }}
# Provenance disabled for smaller image and faster push
provenance: false
- name: Ensure local base image is current (if not rebuilt)
if: steps.check-base.outputs.needs_rebuild == 'false'
run: |
BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest"
# Get the config digest from registry manifest (identifies the image content)
# This is the same value as Docker's Image ID for the same image
REGISTRY_CONFIG_DIGEST=$(docker manifest inspect $BASE_IMAGE 2>/dev/null | jq -r '.config.digest' || echo "")
# Get local image's ID (which is the config digest)
LOCAL_IMAGE_ID=$(docker image inspect $BASE_IMAGE --format='{{.Id}}' 2>/dev/null || echo "")
echo "Registry config digest: $REGISTRY_CONFIG_DIGEST"
echo "Local image ID: $LOCAL_IMAGE_ID"
if [ -z "$LOCAL_IMAGE_ID" ]; then
echo "📥 Base image not in local cache - pulling from registry"
docker pull $BASE_IMAGE
elif [ "$LOCAL_IMAGE_ID" != "$REGISTRY_CONFIG_DIGEST" ]; then
echo "🔄 Local cache has different image - pulling latest"
docker pull $BASE_IMAGE
else
echo "✅ Local cache matches registry - skipping pull"
fi
- name: Build and push app image
uses: docker/build-push-action@v6
with:
context: .
file: backend/Dockerfile
push: true
tags: |
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:latest
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:v${{ steps.version_check.outputs.version }}
# Registry cache for app layers
# ignore-error=true allows build to continue if cache doesn't exist yet
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,ignore-error=true
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,mode=max,ignore-error=true
provenance: false
# ============================================
# GPU Image Build (for audio worker with L4 GPU)
# Same pattern as base image: hash check → rebuild if needed
# GPU image lives in us-east4 (co-located with GPU instances)
# ============================================
- name: Calculate GPU base image hash
id: gpu-base-hash
run: |
# Hash Dockerfile.gpu-base + poetry.lock (resolved deps that affect GPU image)
# Using poetry.lock instead of pyproject.toml so that version-only bumps
# (which don't change dependencies) don't trigger expensive GPU base rebuilds
CURRENT_HASH=$(cat backend/Dockerfile.gpu-base poetry.lock | sha256sum | cut -d' ' -f1)
echo "hash=$CURRENT_HASH" >> $GITHUB_OUTPUT
echo "📋 GPU base content hash: $CURRENT_HASH"
- name: Check if GPU base image exists and is current
id: check-gpu-base
run: |
GPU_BASE_IMAGE="${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}:latest"
EXISTING_HASH=""
MANIFEST=$(docker manifest inspect $GPU_BASE_IMAGE 2>/dev/null || echo "")
CONFIG_DIGEST=$(echo "$MANIFEST" | jq -r '.config.digest // ""' 2>/dev/null || echo "")
if [ -n "$CONFIG_DIGEST" ] && [ "$CONFIG_DIGEST" != "null" ] && [ "$CONFIG_DIGEST" != "" ]; then
echo "📋 Found GPU manifest, config digest: $CONFIG_DIGEST"
TOKEN=$(gcloud auth print-access-token)
CONFIG_JSON=$(curl -sL -H "Authorization: Bearer $TOKEN" \
"https://${{ env.GPU_REGISTRY }}/v2/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}/blobs/$CONFIG_DIGEST" 2>/dev/null)
EXISTING_HASH=$(echo "$CONFIG_JSON" | jq -r '.config.Labels["base.content.hash"] // ""' 2>/dev/null || echo "")
echo "📋 Registry GPU base image hash: $EXISTING_HASH"
else
echo "⚠️ No existing GPU base image found in registry - will build"
fi
if [ "${{ steps.gpu-base-hash.outputs.hash }}" != "$EXISTING_HASH" ]; then
echo "🔄 GPU base image needs rebuild (Dockerfile.gpu-base or pyproject.toml changed)"
echo " Current: ${{ steps.gpu-base-hash.outputs.hash }}"
echo " Registry: $EXISTING_HASH"
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
else
echo "✅ GPU base image is up to date"
echo "needs_rebuild=false" >> $GITHUB_OUTPUT
fi
- name: Build and push GPU base image (if needed)
if: steps.check-gpu-base.outputs.needs_rebuild == 'true'
timeout-minutes: 45
uses: docker/build-push-action@v6
with:
context: .
file: backend/Dockerfile.gpu-base
push: true
tags: |
${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}:latest
${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}:${{ github.sha }}
cache-from: type=registry,ref=${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}:cache,ignore-error=true
cache-to: type=registry,ref=${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}:cache,mode=max,ignore-error=true
build-args: |
BUILD_DATE=${{ github.run_id }}
BUILD_ID=${{ github.sha }}
BASE_CONTENT_HASH=${{ steps.gpu-base-hash.outputs.hash }}
provenance: false
- name: Build and push GPU app image
timeout-minutes: 10
uses: docker/build-push-action@v6
with:
context: .
file: backend/Dockerfile
push: true
tags: |
${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:latest
${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:${{ github.sha }}
${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:v${{ steps.version_check.outputs.version }}
build-args: |
BASE_IMAGE=${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_BASE_IMAGE_NAME }}:latest
cache-from: type=registry,ref=${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:cache,ignore-error=true
cache-to: type=registry,ref=${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:cache,mode=max,ignore-error=true
provenance: false
# ============================================
# Deploy to Cloud Run
# ============================================
- name: Deploy to Cloud Run
run: |
VERSION="${{ steps.version_check.outputs.version }}"
IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}"
echo "🚀 Deploying to Cloud Run..."
echo "Image: $IMAGE"
echo "Version: $VERSION"
gcloud run deploy karaoke-backend \
--image $IMAGE \
--region us-central1 \
--platform managed \
--allow-unauthenticated \
--memory 2Gi \
--cpu 2 \
--cpu-boost \
--cpu-throttling \
--execution-environment gen2 \
--timeout 1800 \
--concurrency 80 \
--max-instances 20 \
--min-instances 1 \
--vpc-connector projects/nomadkaraoke/locations/us-central1/connectors/cloud-run-connector \
--vpc-egress private-ranges-only \
--set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,SENDGRID_API_KEY=sendgrid-api-key:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest"
- name: Update Cloud Run Jobs with new image
run: |
IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}"
GPU_IMAGE="${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:${{ github.sha }}"
echo "📦 Updating Cloud Run Jobs..."
# Video encoding job (CPU, us-central1)
gcloud run jobs update video-encoding-job \
--image "$IMAGE" \
--region us-central1 \
--project ${{ env.PROJECT_ID }} \
--quiet
# Lyrics transcription job (CPU, us-central1)
gcloud run jobs update lyrics-transcription-job \
--image "$IMAGE" \
--region us-central1 \
--project ${{ env.PROJECT_ID }} \
--quiet
# Audio separation job (GPU, us-east4) — uses GPU image
gcloud run jobs update audio-separation-job \
--image "$GPU_IMAGE" \
--region us-east4 \
--project ${{ env.PROJECT_ID }} \
--quiet
echo "✅ Cloud Run Jobs updated"
- name: Verify deployment and version
run: |
EXPECTED_VERSION="${{ steps.version_check.outputs.version }}"
SERVICE_URL=$(gcloud run services describe karaoke-backend \
--region us-central1 \
--format 'value(status.url)')
echo "Service URL: $SERVICE_URL"
echo "Expected version: $EXPECTED_VERSION"
# Wait for new revision to be ready
echo "Waiting for new revision to be ready..."
sleep 15
# Basic health check
curl -f "$SERVICE_URL/api/health" || exit 1
echo "✅ Service is healthy"
# Version verification with retries (Cloud Run may take time to route to new revision)
MAX_RETRIES=6
RETRY_DELAY=10
for i in $(seq 1 $MAX_RETRIES); do
echo "Checking deployed version (attempt $i/$MAX_RETRIES)..."
DEPLOYED_VERSION=$(curl -sf "$SERVICE_URL/api/health/detailed" | jq -r '.version // empty')
if [ "$DEPLOYED_VERSION" = "$EXPECTED_VERSION" ]; then
echo "✅ Version verified: $DEPLOYED_VERSION"
break
else
echo "⚠️ Version mismatch: deployed=$DEPLOYED_VERSION, expected=$EXPECTED_VERSION"
if [ $i -lt $MAX_RETRIES ]; then
echo " Retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
else
echo "❌ Version verification failed after $MAX_RETRIES attempts"
exit 1
fi
fi
done
- name: Summary
if: success()
run: |
echo "🎉 Deployment successful!"
echo "Version: ${{ steps.version_check.outputs.version }}"
echo "Commit: ${{ github.sha }}"
echo "Branch: ${{ github.ref_name }}"
echo "Run: ${{ github.run_number }}"
echo ""
echo "📊 Built on self-hosted GCP runner (200GB SSD, no disk space issues)"
echo " Runner: gcp-runner with 4 vCPU, 16GB RAM"
# ============================================
# PROPAGATE BASE IMAGE TO ALL RUNNERS
# When base image is rebuilt, pull it to all 3 runners
# so subsequent app builds don't wait 8 min for the pull
#
# NOTE: This is best-effort distribution. GitHub Actions doesn't
# support targeting specific runners by name, so we spawn 3 jobs
# and rely on the scheduler to distribute them across idle runners.
# In practice, with all runners idle after deploy-backend, distribution
# is good. Runners that miss this pull will get the image on their
# next app build (with the 8 min delay).
# ============================================
propagate-base-image:
name: "Propagate Base Image (${{ matrix.runner_index }})"
needs: deploy-backend
if: github.ref == 'refs/heads/main' && github.event_name == 'push' && needs.deploy-backend.outputs.base_rebuilt == 'true'
runs-on: [self-hosted, linux, gcp]
permissions:
contents: read
id-token: write
strategy:
matrix:
# 3 parallel jobs - one per self-hosted runner
runner_index: [1, 2, 3]
max-parallel: 3
fail-fast: false # Don't cancel other runners if one fails
env:
REGISTRY: us-central1-docker.pkg.dev
PROJECT_ID: nomadkaraoke
REPOSITORY: karaoke-repo
BASE_IMAGE_NAME: karaoke-backend-base
steps:
- name: Authenticate to Google Cloud (Workload Identity)
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
with:
version: "latest"
- name: Configure Docker for Artifact Registry
run: gcloud auth configure-docker ${{ env.REGISTRY }} --quiet
- name: Pull latest base image
run: |
BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest"
RUNNER_NAME=$(hostname)
echo "📥 Pulling latest base image on runner $RUNNER_NAME..."
docker pull $BASE_IMAGE
echo "✅ Base image cached on $RUNNER_NAME"
# NOTE: Pulumi infrastructure deploys are done locally, not in CI.
# CI runners run on the infrastructure Pulumi manages, creating a circular
# dependency. Always run `pulumi up` locally before merging infra PRs.
# See docs/LESSONS-LEARNED.md for details.