mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Separate whitelabel B2B tenant portals into their own Cloudflare Pages project (`karaoke-gen-tenant`) with edge-injected tenant config, fixing the magic link auth bug and eliminating flash of consumer content - Fix magic link emails to use tenant-specific URLs and branding (emails now link to `vocalstar.nomadkaraoke.com` instead of always `gen.nomadkaraoke.com`) - Add Cloudflare Pages Function that injects `window.__TENANT_CONFIG__` and CSS variables at the edge for instant branding - Add `NEXT_PUBLIC_TENANT_PORTAL=true` build flag for tree-shaking consumer content from tenant builds - Add admin tenant preview via `?preview_tenant=X` query parameter - Add tenant-scoped access tokens (tokens with `tenant_id` override hostname-based detection) - Dual build+deploy in CI (consumer → `karaoke-gen`, tenant → `karaoke-gen-tenant`) ## Test plan - [x] 17 backend tests pass (magic link params, tenant URLs, verify response, tenant-scoped tokens, require_auth override) - [x] 35 frontend tests pass (tenant store, edge-injected config, admin preview, feature flags) - [x] 181/182 E2E tests pass (1 pre-existing screenshot failure unrelated to changes) - [x] Both consumer and tenant portal builds succeed (`npm run build` with and without `NEXT_PUBLIC_TENANT_PORTAL=true`) - [ ] Manual: verify vocalstar.nomadkaraoke.com loads with tenant branding (already deployed) - [ ] Manual: verify magic link from tenant portal links to correct domain @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1464 lines
54 KiB
YAML
1464 lines
54 KiB
YAML
name: CI
|
||
|
||
on:
|
||
push:
|
||
pull_request:
|
||
branches: [main]
|
||
types: [opened, synchronize, reopened, labeled]
|
||
workflow_dispatch:
|
||
|
||
permissions:
|
||
contents: write
|
||
id-token: write
|
||
|
||
concurrency:
|
||
group: "ci-${{ github.ref }}"
|
||
cancel-in-progress: true
|
||
|
||
jobs:
|
||
# ============================================
|
||
# PATH FILTERING - Determine what changed
|
||
# Only needed for PRs (main branch skips tests)
|
||
# ============================================
|
||
|
||
changes:
|
||
name: "Detect Changes"
|
||
runs-on: ubuntu-latest
|
||
if: github.ref != 'refs/heads/main'
|
||
outputs:
|
||
frontend: ${{ steps.filter.outputs.frontend }}
|
||
backend: ${{ steps.filter.outputs.backend }}
|
||
package: ${{ steps.filter.outputs.package }}
|
||
infra: ${{ steps.filter.outputs.infra }}
|
||
deps: ${{ steps.filter.outputs.deps }}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- uses: dorny/paths-filter@v3
|
||
id: filter
|
||
with:
|
||
filters: |
|
||
frontend:
|
||
- 'frontend/**'
|
||
backend:
|
||
- 'backend/**'
|
||
package:
|
||
- 'karaoke_gen/**'
|
||
- 'lyrics_transcriber_temp/**'
|
||
- 'tests/**'
|
||
infra:
|
||
- 'infrastructure/**'
|
||
deps:
|
||
- 'poetry.lock'
|
||
|
||
- name: Summary
|
||
run: |
|
||
echo "### Change Detection Results" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Area | Changed |" >> $GITHUB_STEP_SUMMARY
|
||
echo "|------|---------|" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Frontend | ${{ steps.filter.outputs.frontend }} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Backend | ${{ steps.filter.outputs.backend }} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Package | ${{ steps.filter.outputs.package }} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Infrastructure | ${{ steps.filter.outputs.infra }} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Dependencies | ${{ steps.filter.outputs.deps }} |" >> $GITHUB_STEP_SUMMARY
|
||
|
||
# ============================================
|
||
# FRONTEND TESTS (PRs only - main skips tests)
|
||
# ============================================
|
||
|
||
frontend-lint:
|
||
name: "Frontend - Lint (Rules of Hooks)"
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js with caching
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
- name: Install dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
- name: Run ESLint
|
||
working-directory: frontend
|
||
run: npm run lint
|
||
|
||
frontend-unit-tests:
|
||
name: "Frontend - Unit Tests"
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js with caching
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
- name: Install dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
- name: Run unit tests with coverage
|
||
working-directory: frontend
|
||
run: npm run test:ci
|
||
|
||
- name: Upload coverage report
|
||
if: always()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: frontend-coverage
|
||
path: frontend/coverage/
|
||
retention-days: 7
|
||
|
||
frontend-e2e-smoke:
|
||
name: "Frontend - E2E Smoke (~2 min)"
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
# Quick smoke tests run on every PR with frontend changes
|
||
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js with caching
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
- name: Install dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
- name: Install Playwright browsers
|
||
working-directory: frontend
|
||
run: npx playwright install --with-deps chromium
|
||
|
||
- name: Run smoke tests only
|
||
working-directory: frontend
|
||
run: npx playwright test smoke.spec.ts --project=chromium --reporter=line
|
||
|
||
- name: Upload Playwright report
|
||
if: failure()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: playwright-smoke-report
|
||
path: frontend/playwright-report/
|
||
retention-days: 7
|
||
|
||
frontend-e2e-full:
|
||
name: "Frontend - E2E Full"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: changes
|
||
# Full E2E suite only runs when PR has 'e2e' label (on-demand)
|
||
if: >-
|
||
github.ref != 'refs/heads/main' &&
|
||
needs.changes.outputs.frontend == 'true' &&
|
||
contains(github.event.pull_request.labels.*.name, 'e2e')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js with caching
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
- name: Install dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
- name: Install Playwright browsers
|
||
working-directory: frontend
|
||
run: npx playwright install --with-deps chromium
|
||
|
||
- name: Run full E2E tests
|
||
working-directory: frontend
|
||
run: npx playwright test --reporter=list -x
|
||
|
||
- name: Upload Playwright report
|
||
if: failure()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: playwright-report
|
||
path: frontend/playwright-report/
|
||
retention-days: 7
|
||
|
||
# Happy Path Production Test - runs the comprehensive real-user test against prod
|
||
frontend-e2e-happy-path:
|
||
name: "Frontend - E2E Happy Path (Production)"
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
# Only runs when PR has 'e2e-happy-path' label
|
||
if: >-
|
||
github.ref != 'refs/heads/main' &&
|
||
contains(github.event.pull_request.labels.*.name, 'e2e-happy-path')
|
||
timeout-minutes: 90
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
- name: Install dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
- name: Install Playwright browsers
|
||
working-directory: frontend
|
||
run: npx playwright install --with-deps chromium
|
||
|
||
- name: Create test-results directory
|
||
working-directory: frontend
|
||
run: mkdir -p test-results
|
||
|
||
- name: Refresh E2E session token
|
||
id: refresh_token
|
||
run: |
|
||
RESPONSE=$(curl -sf -X POST \
|
||
"https://api.nomadkaraoke.com/api/admin/users/e2e-test-runner@nomadkaraoke.com/impersonate" \
|
||
-H "X-Admin-Token: ${{ secrets.E2E_ADMIN_TOKEN }}" \
|
||
-H "Content-Type: application/json")
|
||
TOKEN=$(echo "$RESPONSE" | python3 -c 'import json,sys; print(json.load(sys.stdin)["session_token"])')
|
||
echo "::add-mask::$TOKEN"
|
||
echo "token=$TOKEN" >> $GITHUB_OUTPUT
|
||
|
||
- name: Run Happy Path Production Test
|
||
working-directory: frontend
|
||
env:
|
||
TESTMAIL_API_KEY: ${{ secrets.TESTMAIL_API_KEY }}
|
||
TESTMAIL_NAMESPACE: ${{ secrets.TESTMAIL_NAMESPACE }}
|
||
E2E_TEST_TOKEN: ${{ steps.refresh_token.outputs.token }}
|
||
E2E_ADMIN_TOKEN: ${{ secrets.E2E_ADMIN_TOKEN }}
|
||
E2E_BYPASS_KEY: ${{ secrets.E2E_BYPASS_KEY }}
|
||
run: |
|
||
echo "Starting E2E Happy Path Test against Production"
|
||
echo "Time: $(date)"
|
||
npx playwright test happy-path-real-user.spec.ts \
|
||
--config=playwright.production.config.ts \
|
||
--project=chromium \
|
||
--reporter=list \
|
||
--timeout=3600000
|
||
|
||
- name: Upload test artifacts
|
||
if: always()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: e2e-happy-path-results
|
||
path: |
|
||
frontend/test-results/
|
||
frontend/playwright-report/
|
||
retention-days: 30
|
||
|
||
frontend-build:
|
||
name: "Frontend - Build"
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && needs.changes.outputs.frontend == 'true'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js with caching
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
- name: Install dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
- name: Build frontend
|
||
working-directory: frontend
|
||
env:
|
||
NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com
|
||
run: npm run build
|
||
|
||
# ============================================
|
||
# PACKAGE TESTS (PRs only - main skips tests)
|
||
# ============================================
|
||
|
||
package-unit-tests:
|
||
name: "Package - Unit Tests (70% coverage)"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
submodules: true
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Cache Poetry dependencies
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: .venv
|
||
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||
restore-keys: |
|
||
venv-package-${{ runner.os }}-py3.13-
|
||
|
||
- name: Install dependencies
|
||
run: poetry install --with dev
|
||
|
||
- name: Install system dependencies
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install -y ffmpeg
|
||
|
||
- name: Cache spaCy model
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: ~/.cache/spacy
|
||
key: spacy-en_core_web_sm-${{ runner.os }}
|
||
|
||
- name: Install spaCy language models
|
||
run: poetry run python -m spacy download en_core_web_sm
|
||
|
||
- name: Run unit tests
|
||
run: |
|
||
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=69
|
||
|
||
- name: Upload coverage reports
|
||
uses: codecov/codecov-action@v5
|
||
with:
|
||
token: ${{ secrets.CODECOV_TOKEN }}
|
||
files: ./coverage.xml
|
||
flags: unittests
|
||
name: codecov-umbrella
|
||
|
||
package-integration-tests:
|
||
name: "Package - Integration Tests"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
submodules: true
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Cache Poetry dependencies
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: .venv
|
||
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||
restore-keys: |
|
||
venv-package-${{ runner.os }}-py3.13-
|
||
|
||
- name: Install dependencies
|
||
run: poetry install --with dev
|
||
|
||
- name: Install system dependencies
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install -y ffmpeg
|
||
|
||
- name: Cache spaCy model
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: ~/.cache/spacy
|
||
key: spacy-en_core_web_sm-${{ runner.os }}
|
||
|
||
- name: Install spaCy language models
|
||
run: poetry run python -m spacy download en_core_web_sm
|
||
|
||
- name: Run integration tests
|
||
run: poetry run pytest tests/integration/ -v
|
||
|
||
package-build-test:
|
||
name: "Package - Build & Install (Test)"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: [changes, package-unit-tests, package-integration-tests]
|
||
if: >-
|
||
github.ref != 'refs/heads/main' && always() &&
|
||
(needs.package-unit-tests.result == 'success' || needs.package-unit-tests.result == 'skipped') &&
|
||
(needs.package-integration-tests.result == 'success' || needs.package-integration-tests.result == 'skipped') &&
|
||
(needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
submodules: true
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Build package
|
||
run: poetry build
|
||
|
||
- name: Test package installation
|
||
run: |
|
||
pip install dist/*.whl
|
||
python -m karaoke_gen.utils.gen_cli --help
|
||
|
||
# ============================================
|
||
# BACKEND TESTS (PRs only - main skips tests)
|
||
# ============================================
|
||
|
||
backend-unit-tests:
|
||
name: "Backend - Unit Tests"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
with:
|
||
submodules: false
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Install dependencies
|
||
run: poetry install
|
||
|
||
- name: Run all backend unit tests
|
||
env:
|
||
GOOGLE_CLOUD_PROJECT: test-project
|
||
GCS_BUCKET_NAME: test-bucket
|
||
FIRESTORE_COLLECTION: jobs
|
||
ENVIRONMENT: test
|
||
run: |
|
||
poetry run pytest \
|
||
backend/tests/ \
|
||
--ignore=backend/tests/emulator \
|
||
-v --tb=short
|
||
|
||
backend-emulator-tests:
|
||
name: "Backend - Emulator Integration Tests"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
with:
|
||
submodules: false
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Set up Java 21 (required by Firestore emulator)
|
||
uses: actions/setup-java@v4
|
||
with:
|
||
distribution: "temurin"
|
||
java-version: "21"
|
||
|
||
- name: Install dependencies
|
||
run: poetry install
|
||
|
||
- name: Start Docker for GCS emulator
|
||
run: docker pull fsouza/fake-gcs-server:latest
|
||
|
||
- name: Run emulator integration tests
|
||
run: |
|
||
chmod +x scripts/*.sh
|
||
./scripts/run-emulator-tests.sh
|
||
|
||
- name: Upload test results
|
||
if: always()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: backend-test-results
|
||
path: backend/htmlcov/
|
||
retention-days: 7
|
||
|
||
backend-lint:
|
||
name: "Backend - Code Quality"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
needs: changes
|
||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true')
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
with:
|
||
submodules: false
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Cache Poetry dependencies
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: .venv
|
||
key: venv-lint-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||
restore-keys: |
|
||
venv-lint-${{ runner.os }}-py3.13-
|
||
|
||
- name: Install dependencies
|
||
run: poetry install
|
||
|
||
- name: Run validation (syntax check only)
|
||
run: |
|
||
poetry run python -m py_compile backend/main.py backend/config.py
|
||
echo "✅ Basic syntax validation passed"
|
||
|
||
# ============================================
|
||
# CI GATE - Required status check for PRs
|
||
# Only runs on PRs, not on main
|
||
# ============================================
|
||
|
||
ci-gate:
|
||
name: "CI Gate"
|
||
runs-on: ubuntu-latest
|
||
# frontend-e2e-smoke IS required - fast smoke tests catch critical regressions
|
||
# frontend-lint IS required - catches React hooks violations that cause runtime crashes
|
||
# frontend-e2e-happy-path is required ONLY when 'e2e-happy-path' label is present
|
||
# frontend-e2e-full is optional (requires 'e2e' label) so not enforced here
|
||
needs: [frontend-lint, frontend-unit-tests, frontend-e2e-smoke, frontend-build, package-unit-tests, package-integration-tests, backend-unit-tests, backend-emulator-tests, backend-lint, frontend-e2e-happy-path]
|
||
if: github.ref != 'refs/heads/main' && always()
|
||
steps:
|
||
- name: Check results
|
||
run: |
|
||
# Check all always-required jobs
|
||
results=(
|
||
"${{ needs.frontend-lint.result }}"
|
||
"${{ needs.frontend-unit-tests.result }}"
|
||
"${{ needs.frontend-e2e-smoke.result }}"
|
||
"${{ needs.frontend-build.result }}"
|
||
"${{ needs.package-unit-tests.result }}"
|
||
"${{ needs.package-integration-tests.result }}"
|
||
"${{ needs.backend-unit-tests.result }}"
|
||
"${{ needs.backend-emulator-tests.result }}"
|
||
"${{ needs.backend-lint.result }}"
|
||
)
|
||
for result in "${results[@]}"; do
|
||
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
||
echo "❌ CI failed"
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
# E2E Happy Path: enforced when 'e2e-happy-path' label is on the PR
|
||
e2e_result="${{ needs.frontend-e2e-happy-path.result }}"
|
||
has_label="${{ contains(github.event.pull_request.labels.*.name, 'e2e-happy-path') }}"
|
||
if [[ "$has_label" == "true" ]]; then
|
||
if [[ "$e2e_result" != "success" ]]; then
|
||
echo "❌ E2E Happy Path is required (label present) but result is: $e2e_result"
|
||
exit 1
|
||
fi
|
||
echo "✅ E2E Happy Path passed (required by label)"
|
||
else
|
||
echo "ℹ️ E2E Happy Path skipped (no label)"
|
||
fi
|
||
|
||
echo "✅ CI passed"
|
||
|
||
# ============================================
|
||
# DEPLOYMENTS (main branch only)
|
||
# Tests already passed on PR, so we skip them here
|
||
# ============================================
|
||
|
||
package-build:
|
||
name: "Package - Build"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
submodules: true
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
with:
|
||
virtualenvs-create: true
|
||
virtualenvs-in-project: true
|
||
|
||
- name: Cache Poetry dependencies
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: .venv
|
||
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||
restore-keys: |
|
||
venv-package-${{ runner.os }}-py3.13-
|
||
|
||
- name: Build package
|
||
run: poetry build
|
||
|
||
- name: Upload build artifacts
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: package-dist
|
||
path: dist/
|
||
retention-days: 1
|
||
|
||
package-publish:
|
||
name: "Deploy - Publish to PyPI"
|
||
runs-on: ubuntu-latest
|
||
needs: [package-build]
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
submodules: true
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
|
||
- name: Download build artifacts
|
||
uses: actions/download-artifact@v4
|
||
with:
|
||
name: package-dist
|
||
path: dist/
|
||
|
||
- name: Extract version
|
||
id: extract_version
|
||
run: |
|
||
VERSION=$(poetry version --short)
|
||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||
echo "📋 Extracted version: $VERSION"
|
||
|
||
- name: Check if tag exists
|
||
id: check_tag
|
||
run: |
|
||
VERSION="${{ steps.extract_version.outputs.version }}"
|
||
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
|
||
echo "tag_exists=true" >> $GITHUB_OUTPUT
|
||
echo "🏷️ Tag v$VERSION already exists, skipping release"
|
||
else
|
||
echo "tag_exists=false" >> $GITHUB_OUTPUT
|
||
echo "✨ Tag v$VERSION does not exist, proceeding with release"
|
||
fi
|
||
|
||
- name: Publish to PyPI
|
||
if: steps.check_tag.outputs.tag_exists == 'false'
|
||
run: |
|
||
echo "📦 Publishing the following files to PyPI:"
|
||
ls -la dist/
|
||
echo "🚀 Publishing to PyPI..."
|
||
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
|
||
poetry publish
|
||
|
||
- name: Create Git Tag
|
||
if: steps.check_tag.outputs.tag_exists == 'false'
|
||
run: |
|
||
git config user.name github-actions
|
||
git config user.email github-actions@github.com
|
||
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
|
||
git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}"
|
||
|
||
- name: Create GitHub Release
|
||
if: steps.check_tag.outputs.tag_exists == 'false'
|
||
uses: softprops/action-gh-release@v1
|
||
with:
|
||
tag_name: "v${{ steps.extract_version.outputs.version }}"
|
||
name: "v${{ steps.extract_version.outputs.version }}"
|
||
body: |
|
||
## Changes in v${{ steps.extract_version.outputs.version }}
|
||
|
||
This release was automatically created from the main branch.
|
||
|
||
### Installation
|
||
```bash
|
||
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
|
||
```
|
||
|
||
### Assets
|
||
- Python package published to PyPI
|
||
- Frontend deployed to Cloudflare Pages at gen.nomadkaraoke.com
|
||
|
||
For detailed changes, see the commit history.
|
||
draft: false
|
||
prerelease: false
|
||
files: |
|
||
dist/*.whl
|
||
dist/*.tar.gz
|
||
|
||
deploy-frontend:
|
||
name: "Deploy - Frontend (Cloudflare Pages)"
|
||
runs-on: ubuntu-latest
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||
steps:
|
||
- name: Checkout
|
||
uses: actions/checkout@v4
|
||
|
||
- name: Setup Node.js with caching
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
cache: 'npm'
|
||
cache-dependency-path: frontend/package-lock.json
|
||
|
||
# ===== Build Unified Frontend (Next.js) =====
|
||
# NOTE: Lyrics review and instrumental review UIs were consolidated into
|
||
# the main Next.js frontend (PR #267, Jan 2025). The unified frontend
|
||
# serves all three UIs from a single codebase at:
|
||
# - / (main app)
|
||
# - /app/jobs/{id}/lyrics (lyrics review)
|
||
# - /app/jobs/{id}/instrumental (instrumental review)
|
||
- name: Install frontend dependencies
|
||
working-directory: frontend
|
||
run: npm ci --legacy-peer-deps
|
||
|
||
# ===== Consumer Build (gen.nomadkaraoke.com) =====
|
||
- name: Build frontend (consumer)
|
||
working-directory: frontend
|
||
env:
|
||
NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com
|
||
run: npm run build
|
||
|
||
- name: Deploy consumer to Cloudflare Pages
|
||
uses: cloudflare/wrangler-action@v3
|
||
with:
|
||
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||
command: pages deploy frontend/out --project-name=karaoke-gen --branch=main
|
||
|
||
# ===== Tenant Portal Build (vocalstar.nomadkaraoke.com, singa.nomadkaraoke.com, etc.) =====
|
||
# Separate build with NEXT_PUBLIC_TENANT_PORTAL=true which tree-shakes
|
||
# consumer marketing content and renders TenantLandingPage at root.
|
||
# The Cloudflare Pages Function (frontend/functions/) injects tenant config at the edge.
|
||
- name: Build frontend (tenant portal)
|
||
working-directory: frontend
|
||
env:
|
||
NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com
|
||
NEXT_PUBLIC_TENANT_PORTAL: "true"
|
||
run: npm run build
|
||
|
||
- name: Deploy tenant portal to Cloudflare Pages
|
||
uses: cloudflare/wrangler-action@v3
|
||
with:
|
||
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||
command: pages deploy frontend/out --project-name=karaoke-gen-tenant --branch=main
|
||
|
||
# NOTE: buy-site was consolidated into the main frontend (PR #122, Dec 2024)
|
||
# Landing page and payment flow are now at gen.nomadkaraoke.com (root)
|
||
# buy.nomadkaraoke.com subdomain has been deleted
|
||
|
||
deploy-backend:
|
||
name: "Deploy - Backend (Cloud Run)"
|
||
needs: [package-build]
|
||
runs-on: [self-hosted, linux, gcp]
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||
permissions:
|
||
contents: read
|
||
id-token: write
|
||
outputs:
|
||
base_rebuilt: ${{ steps.check-base.outputs.needs_rebuild }}
|
||
env:
|
||
REGISTRY: us-central1-docker.pkg.dev
|
||
PROJECT_ID: nomadkaraoke
|
||
REPOSITORY: karaoke-repo
|
||
BASE_IMAGE_NAME: karaoke-backend-base
|
||
APP_IMAGE_NAME: karaoke-backend
|
||
steps:
|
||
- name: Check disk space
|
||
run: |
|
||
echo "Checking disk space..."
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "Current disk usage: ${USAGE}%"
|
||
if [ "$USAGE" -gt 70 ]; then
|
||
echo "Disk usage high, running cleanup..."
|
||
docker system prune -af || true
|
||
docker builder prune -af || true
|
||
USAGE=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||
echo "After cleanup: ${USAGE}%"
|
||
fi
|
||
if [ "$USAGE" -gt 85 ]; then
|
||
echo "::error::Disk usage ${USAGE}% exceeds 85% even after cleanup. Runner needs maintenance."
|
||
exit 1
|
||
fi
|
||
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
with:
|
||
submodules: false
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install Poetry
|
||
uses: snok/install-poetry@v1
|
||
|
||
- name: Extract version
|
||
id: version_check
|
||
run: |
|
||
VERSION=$(poetry version --short)
|
||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||
echo "📋 Current version: $VERSION"
|
||
echo "🚀 Deploying on every push to main"
|
||
|
||
- name: Authenticate to Google Cloud (Workload Identity)
|
||
id: auth
|
||
uses: google-github-actions/auth@v2
|
||
with:
|
||
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
||
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
||
|
||
- name: Set up Cloud SDK
|
||
uses: google-github-actions/setup-gcloud@v2
|
||
with:
|
||
version: "latest"
|
||
|
||
# ============================================
|
||
# Deploy to GCE Encoding Worker (Immutable Deployment Pattern)
|
||
#
|
||
# This pattern ensures version consistency:
|
||
# 1. Upload wheel to FIXED path (no version sorting)
|
||
# 2. Upload startup.sh (logic can change without image rebuild)
|
||
# 3. Write version manifest for verification
|
||
# 4. Restart service (triggers bootstrap.sh -> startup.sh)
|
||
# 5. STRICTLY verify deployed version (fail CI on mismatch)
|
||
#
|
||
# See infrastructure/encoding-worker/README.md for details
|
||
# ============================================
|
||
|
||
- name: Download wheel artifact
|
||
uses: actions/download-artifact@v4
|
||
with:
|
||
name: package-dist
|
||
path: dist/
|
||
|
||
- name: Deploy to GCE Encoding Worker
|
||
run: |
|
||
VERSION="${{ steps.version_check.outputs.version }}"
|
||
BUCKET="gs://karaoke-gen-storage-${{ env.PROJECT_ID }}"
|
||
|
||
echo "🚀 Deploying encoding worker v${VERSION}..."
|
||
|
||
# 1. Find the wheel file
|
||
WHEEL_FILE=$(ls dist/*.whl | head -1)
|
||
if [ -z "$WHEEL_FILE" ]; then
|
||
echo "::error::No wheel file found in dist/"
|
||
exit 1
|
||
fi
|
||
echo "📦 Found wheel: $WHEEL_FILE"
|
||
|
||
# 2. Upload wheel to FIXED path (atomic via temp file)
|
||
# This eliminates version sorting - always use karaoke_gen-current.whl
|
||
echo "📤 Uploading wheel to fixed path..."
|
||
gsutil cp "$WHEEL_FILE" "${BUCKET}/wheels/karaoke_gen-current.whl.tmp"
|
||
gsutil mv "${BUCKET}/wheels/karaoke_gen-current.whl.tmp" \
|
||
"${BUCKET}/wheels/karaoke_gen-current.whl"
|
||
|
||
# Also keep versioned copy for rollback/debugging
|
||
gsutil cp "$WHEEL_FILE" "${BUCKET}/wheels/"
|
||
|
||
# 3. Upload startup.sh (allows logic changes without image rebuild)
|
||
echo "📤 Uploading startup.sh..."
|
||
gsutil cp infrastructure/encoding-worker/startup.sh \
|
||
"${BUCKET}/encoding-worker/startup.sh"
|
||
|
||
# 4. Write version manifest for verification
|
||
echo "📝 Writing version manifest..."
|
||
echo "${VERSION}" | gsutil cp - "${BUCKET}/encoding-worker/version.txt"
|
||
|
||
echo "✅ GCS deployment complete"
|
||
|
||
- name: Restart GCE Encoding Worker
|
||
run: |
|
||
VERSION="${{ steps.version_check.outputs.version }}"
|
||
|
||
echo "🔄 Restarting encoding worker..."
|
||
|
||
# Use IAP tunneling - works directly with service account credentials
|
||
gcloud compute ssh encoding-worker \
|
||
--zone=us-central1-c \
|
||
--project=nomadkaraoke \
|
||
--tunnel-through-iap \
|
||
--command="sudo systemctl restart encoding-worker"
|
||
|
||
echo "⏳ Waiting for service to start (up to 60s)..."
|
||
sleep 20
|
||
|
||
- name: Verify Encoding Worker Version
|
||
run: |
|
||
VERSION="${{ steps.version_check.outputs.version }}"
|
||
|
||
echo "🔍 Verifying encoding worker version..."
|
||
|
||
# Retry loop - service may take time to fully start
|
||
MAX_RETRIES=6
|
||
RETRY_DELAY=10
|
||
|
||
for i in $(seq 1 $MAX_RETRIES); do
|
||
echo "Attempt $i/$MAX_RETRIES..."
|
||
|
||
NEW_VERSION=$(gcloud compute ssh encoding-worker \
|
||
--zone=us-central1-c \
|
||
--project=nomadkaraoke \
|
||
--tunnel-through-iap \
|
||
--command="curl -s http://localhost:8080/health 2>/dev/null | grep -o '\"wheel_version\":\"[^\"]*\"' | cut -d'\"' -f4" 2>/dev/null || echo "")
|
||
|
||
if [ "$NEW_VERSION" = "$VERSION" ]; then
|
||
echo "✅ Encoding worker version verified: v$NEW_VERSION"
|
||
exit 0
|
||
fi
|
||
|
||
if [ -n "$NEW_VERSION" ]; then
|
||
echo "⚠️ Version: $NEW_VERSION (expected: $VERSION)"
|
||
else
|
||
echo "⚠️ Health endpoint not responding yet"
|
||
fi
|
||
|
||
if [ $i -lt $MAX_RETRIES ]; then
|
||
echo " Retrying in ${RETRY_DELAY}s..."
|
||
sleep $RETRY_DELAY
|
||
fi
|
||
done
|
||
|
||
# STRICT VERIFICATION: Fail CI on version mismatch
|
||
echo ""
|
||
echo "=============================================="
|
||
echo "❌ ENCODING WORKER DEPLOYMENT FAILED"
|
||
echo "=============================================="
|
||
echo "Expected version: $VERSION"
|
||
echo "Deployed version: $NEW_VERSION"
|
||
echo ""
|
||
echo "Possible causes:"
|
||
echo " - GCS upload failed (check 'Deploy to GCE' step)"
|
||
echo " - startup.sh download failed (check VM logs)"
|
||
echo " - Wheel installation failed (check VM logs)"
|
||
echo ""
|
||
echo "To debug, SSH to VM and check logs:"
|
||
echo " gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke --tunnel-through-iap"
|
||
echo " sudo journalctl -u encoding-worker -n 100"
|
||
echo " cat /var/log/encoding-worker-startup.log"
|
||
echo "=============================================="
|
||
|
||
exit 1
|
||
|
||
- name: Configure Docker for Artifact Registry
|
||
run: gcloud auth configure-docker ${{ env.REGISTRY }} --quiet
|
||
|
||
# ============================================
|
||
# Docker Build with BuildKit + Registry Cache
|
||
# This replaces expensive Cloud Build with free GitHub Actions
|
||
# Registry cache persists between builds for fast rebuilds
|
||
# ============================================
|
||
|
||
- name: Set up Docker Buildx
|
||
uses: docker/setup-buildx-action@v3
|
||
|
||
- name: Calculate base image hash
|
||
id: base-hash
|
||
run: |
|
||
# Hash both poetry.lock AND Dockerfile.base - rebuild if either changes
|
||
CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
|
||
echo "hash=$CURRENT_HASH" >> $GITHUB_OUTPUT
|
||
echo "📋 Base content hash: $CURRENT_HASH"
|
||
|
||
- name: Check if base image exists and is current
|
||
id: check-base
|
||
run: |
|
||
BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest"
|
||
|
||
# Fetch hash label directly from registry (10KB config blob, not full image)
|
||
# This avoids issues with stale local Docker cache on GitHub runners
|
||
EXISTING_HASH=""
|
||
|
||
# Get the manifest (small JSON, ~1KB) - includes config digest
|
||
MANIFEST=$(docker manifest inspect $BASE_IMAGE 2>/dev/null || echo "")
|
||
CONFIG_DIGEST=$(echo "$MANIFEST" | jq -r '.config.digest // ""' 2>/dev/null || echo "")
|
||
|
||
if [ -n "$CONFIG_DIGEST" ] && [ "$CONFIG_DIGEST" != "null" ] && [ "$CONFIG_DIGEST" != "" ]; then
|
||
echo "📋 Found manifest, config digest: $CONFIG_DIGEST"
|
||
|
||
# Fetch just the config blob (~10KB) to get labels - much faster than pulling full image
|
||
TOKEN=$(gcloud auth print-access-token)
|
||
CONFIG_JSON=$(curl -sL -H "Authorization: Bearer $TOKEN" \
|
||
"https://${{ env.REGISTRY }}/v2/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}/blobs/$CONFIG_DIGEST" 2>/dev/null)
|
||
|
||
EXISTING_HASH=$(echo "$CONFIG_JSON" | jq -r '.config.Labels["base.content.hash"] // .config.Labels["poetry.lock.hash"] // ""' 2>/dev/null || echo "")
|
||
echo "📋 Registry base image hash: $EXISTING_HASH"
|
||
else
|
||
echo "⚠️ No existing base image found in registry - will build"
|
||
fi
|
||
|
||
if [ "${{ steps.base-hash.outputs.hash }}" != "$EXISTING_HASH" ]; then
|
||
echo "🔄 Base image needs rebuild (poetry.lock or Dockerfile.base changed)"
|
||
echo " Current: ${{ steps.base-hash.outputs.hash }}"
|
||
echo " Registry: $EXISTING_HASH"
|
||
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "✅ Base image is up to date"
|
||
echo "needs_rebuild=false" >> $GITHUB_OUTPUT
|
||
fi
|
||
|
||
- name: Build and push base image (if needed)
|
||
if: steps.check-base.outputs.needs_rebuild == 'true'
|
||
uses: docker/build-push-action@v6
|
||
with:
|
||
context: .
|
||
file: backend/Dockerfile.base
|
||
push: true
|
||
tags: |
|
||
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest
|
||
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:${{ github.sha }}
|
||
# Registry cache for ALL layers (mode=max) - persists between builds
|
||
# ignore-error=true allows build to continue if cache doesn't exist yet
|
||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:cache,ignore-error=true
|
||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:cache,mode=max,ignore-error=true
|
||
build-args: |
|
||
BUILD_DATE=${{ github.run_id }}
|
||
BUILD_ID=${{ github.sha }}
|
||
BASE_CONTENT_HASH=${{ steps.base-hash.outputs.hash }}
|
||
# Provenance disabled for smaller image and faster push
|
||
provenance: false
|
||
|
||
- name: Ensure local base image is current (if not rebuilt)
|
||
if: steps.check-base.outputs.needs_rebuild == 'false'
|
||
run: |
|
||
BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest"
|
||
|
||
# Get the config digest from registry manifest (identifies the image content)
|
||
# This is the same value as Docker's Image ID for the same image
|
||
REGISTRY_CONFIG_DIGEST=$(docker manifest inspect $BASE_IMAGE 2>/dev/null | jq -r '.config.digest' || echo "")
|
||
|
||
# Get local image's ID (which is the config digest)
|
||
LOCAL_IMAGE_ID=$(docker image inspect $BASE_IMAGE --format='{{.Id}}' 2>/dev/null || echo "")
|
||
|
||
echo "Registry config digest: $REGISTRY_CONFIG_DIGEST"
|
||
echo "Local image ID: $LOCAL_IMAGE_ID"
|
||
|
||
if [ -z "$LOCAL_IMAGE_ID" ]; then
|
||
echo "📥 Base image not in local cache - pulling from registry"
|
||
docker pull $BASE_IMAGE
|
||
elif [ "$LOCAL_IMAGE_ID" != "$REGISTRY_CONFIG_DIGEST" ]; then
|
||
echo "🔄 Local cache has different image - pulling latest"
|
||
docker pull $BASE_IMAGE
|
||
else
|
||
echo "✅ Local cache matches registry - skipping pull"
|
||
fi
|
||
|
||
- name: Build and push app image
|
||
uses: docker/build-push-action@v6
|
||
with:
|
||
context: .
|
||
file: backend/Dockerfile
|
||
push: true
|
||
tags: |
|
||
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:latest
|
||
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}
|
||
${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:v${{ steps.version_check.outputs.version }}
|
||
# Registry cache for app layers
|
||
# ignore-error=true allows build to continue if cache doesn't exist yet
|
||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,ignore-error=true
|
||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,mode=max,ignore-error=true
|
||
provenance: false
|
||
|
||
# ============================================
|
||
# Deploy to Cloud Run
|
||
# ============================================
|
||
|
||
- name: Deploy to Cloud Run
|
||
run: |
|
||
VERSION="${{ steps.version_check.outputs.version }}"
|
||
IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}"
|
||
|
||
echo "🚀 Deploying to Cloud Run..."
|
||
echo "Image: $IMAGE"
|
||
echo "Version: $VERSION"
|
||
|
||
gcloud run deploy karaoke-backend \
|
||
--image $IMAGE \
|
||
--region us-central1 \
|
||
--platform managed \
|
||
--allow-unauthenticated \
|
||
--memory 16Gi \
|
||
--cpu 8 \
|
||
--cpu-boost \
|
||
--no-cpu-throttling \
|
||
--execution-environment gen2 \
|
||
--timeout 1800 \
|
||
--concurrency 1 \
|
||
--max-instances 50 \
|
||
--min-instances 4 \
|
||
--vpc-connector projects/nomadkaraoke/locations/us-central1/connectors/cloud-run-connector \
|
||
--vpc-egress private-ranges-only \
|
||
--set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true" \
|
||
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,SENDGRID_API_KEY=sendgrid-api-key:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest"
|
||
|
||
- name: Verify deployment and version
|
||
run: |
|
||
EXPECTED_VERSION="${{ steps.version_check.outputs.version }}"
|
||
SERVICE_URL=$(gcloud run services describe karaoke-backend \
|
||
--region us-central1 \
|
||
--format 'value(status.url)')
|
||
|
||
echo "Service URL: $SERVICE_URL"
|
||
echo "Expected version: $EXPECTED_VERSION"
|
||
|
||
# Wait for new revision to be ready
|
||
echo "Waiting for new revision to be ready..."
|
||
sleep 15
|
||
|
||
# Basic health check
|
||
curl -f "$SERVICE_URL/api/health" || exit 1
|
||
echo "✅ Service is healthy"
|
||
|
||
# Version verification with retries (Cloud Run may take time to route to new revision)
|
||
MAX_RETRIES=6
|
||
RETRY_DELAY=10
|
||
for i in $(seq 1 $MAX_RETRIES); do
|
||
echo "Checking deployed version (attempt $i/$MAX_RETRIES)..."
|
||
DEPLOYED_VERSION=$(curl -sf "$SERVICE_URL/api/health/detailed" | jq -r '.version // empty')
|
||
|
||
if [ "$DEPLOYED_VERSION" = "$EXPECTED_VERSION" ]; then
|
||
echo "✅ Version verified: $DEPLOYED_VERSION"
|
||
break
|
||
else
|
||
echo "⚠️ Version mismatch: deployed=$DEPLOYED_VERSION, expected=$EXPECTED_VERSION"
|
||
if [ $i -lt $MAX_RETRIES ]; then
|
||
echo " Retrying in ${RETRY_DELAY}s..."
|
||
sleep $RETRY_DELAY
|
||
else
|
||
echo "❌ Version verification failed after $MAX_RETRIES attempts"
|
||
exit 1
|
||
fi
|
||
fi
|
||
done
|
||
|
||
- name: Summary
|
||
if: success()
|
||
run: |
|
||
echo "🎉 Deployment successful!"
|
||
echo "Version: ${{ steps.version_check.outputs.version }}"
|
||
echo "Commit: ${{ github.sha }}"
|
||
echo "Branch: ${{ github.ref_name }}"
|
||
echo "Run: ${{ github.run_number }}"
|
||
echo ""
|
||
echo "📊 Built on self-hosted GCP runner (200GB SSD, no disk space issues)"
|
||
echo " Runner: gcp-runner with 4 vCPU, 16GB RAM"
|
||
|
||
# ============================================
|
||
# PROPAGATE BASE IMAGE TO ALL RUNNERS
|
||
# When base image is rebuilt, pull it to all 20 runners
|
||
# so subsequent app builds don't wait 8 min for the pull
|
||
#
|
||
# NOTE: This is best-effort distribution. GitHub Actions doesn't
|
||
# support targeting specific runners by name, so we spawn 20 jobs
|
||
# and rely on the scheduler to distribute them across idle runners.
|
||
# In practice, with all runners idle after deploy-backend, distribution
|
||
# is good. Runners that miss this pull will get the image on their
|
||
# next app build (with the 8 min delay).
|
||
# ============================================
|
||
|
||
propagate-base-image:
|
||
name: "Propagate Base Image (${{ matrix.runner_index }})"
|
||
needs: deploy-backend
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push' && needs.deploy-backend.outputs.base_rebuilt == 'true'
|
||
runs-on: [self-hosted, linux, gcp]
|
||
permissions:
|
||
contents: read
|
||
id-token: write
|
||
strategy:
|
||
matrix:
|
||
# 20 parallel jobs - GitHub distributes across available runners
|
||
runner_index: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
|
||
max-parallel: 20
|
||
fail-fast: false # Don't cancel other runners if one fails
|
||
env:
|
||
REGISTRY: us-central1-docker.pkg.dev
|
||
PROJECT_ID: nomadkaraoke
|
||
REPOSITORY: karaoke-repo
|
||
BASE_IMAGE_NAME: karaoke-backend-base
|
||
steps:
|
||
- name: Authenticate to Google Cloud (Workload Identity)
|
||
uses: google-github-actions/auth@v2
|
||
with:
|
||
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
||
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
||
|
||
- name: Set up Cloud SDK
|
||
uses: google-github-actions/setup-gcloud@v2
|
||
with:
|
||
version: "latest"
|
||
|
||
- name: Configure Docker for Artifact Registry
|
||
run: gcloud auth configure-docker ${{ env.REGISTRY }} --quiet
|
||
|
||
- name: Pull latest base image
|
||
run: |
|
||
BASE_IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.BASE_IMAGE_NAME }}:latest"
|
||
RUNNER_NAME=$(hostname)
|
||
echo "📥 Pulling latest base image on runner $RUNNER_NAME..."
|
||
docker pull $BASE_IMAGE
|
||
echo "✅ Base image cached on $RUNNER_NAME"
|
||
|
||
deploy-infrastructure:
|
||
name: "Deploy - Infrastructure (Pulumi)"
|
||
runs-on: [self-hosted, linux, gcp]
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||
permissions:
|
||
contents: read
|
||
id-token: write
|
||
steps:
|
||
- name: Checkout
|
||
uses: actions/checkout@v4
|
||
|
||
- name: Authenticate to Google Cloud (Workload Identity)
|
||
uses: google-github-actions/auth@v2
|
||
with:
|
||
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
||
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
||
|
||
- name: Set up Cloud SDK
|
||
uses: google-github-actions/setup-gcloud@v2
|
||
with:
|
||
version: "latest"
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: '3.13'
|
||
|
||
- name: Install Pulumi
|
||
run: |
|
||
curl -fsSL https://get.pulumi.com | sh
|
||
echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
|
||
|
||
- name: Install Python dependencies
|
||
working-directory: infrastructure
|
||
run: |
|
||
pip install pulumi pulumi-gcp pulumi-docker
|
||
|
||
- name: Configure Pulumi
|
||
env:
|
||
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
|
||
run: |
|
||
pulumi login
|
||
|
||
- name: Refresh Infrastructure State
|
||
working-directory: infrastructure
|
||
env:
|
||
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
|
||
run: |
|
||
echo "🔄 Refreshing Pulumi state to sync with actual cloud resources..."
|
||
# Refresh detects resources deleted outside of Pulumi (e.g., via gcloud CLI)
|
||
# This ensures Pulumi recreates missing resources
|
||
pulumi refresh --yes --skip-preview --stack nomadkaraoke/karaoke-gen-infrastructure/prod || true
|
||
|
||
- name: Deploy Infrastructure
|
||
working-directory: infrastructure
|
||
env:
|
||
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
|
||
run: |
|
||
echo "🚀 Deploying infrastructure changes..."
|
||
# Skip preview to avoid needing broad read permissions on all GCP resources
|
||
# Pulumi state is authoritative since infrastructure is only managed via Pulumi
|
||
pulumi up --yes --skip-preview --stack nomadkaraoke/karaoke-gen-infrastructure/prod
|
||
|
||
- name: Summary
|
||
if: success()
|
||
run: |
|
||
echo "🎉 Infrastructure deployment successful!"
|
||
echo "Commit: ${{ github.sha }}"
|
||
echo "Branch: ${{ github.ref_name }}"
|