mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Merged test-and-publish.yml and test.yml into a single ci.yml workflow. PR checks now show clear job names: - Package - Unit Tests (70% coverage) - Package - Integration Tests - Package - Build & Install - Backend - Unit Tests - Backend - Emulator Integration Tests - Backend - Code Quality Deploy jobs (main only): - Deploy - Publish to PyPI - Deploy - Frontend (GitHub Pages) - Deploy - Backend (Cloud Run) Benefits: - Single workflow run per PR instead of two - Clear visibility of what's being tested - Logical grouping of package vs backend tests - All deployments still gated behind successful tests
540 lines
17 KiB
YAML
540 lines
17 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: write
|
|
pages: write
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: "ci-${{ github.ref }}"
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# ============================================
|
|
# PACKAGE TESTS (karaoke-gen Python CLI)
|
|
# ============================================
|
|
|
|
package-unit-tests:
|
|
name: "Package - Unit Tests (70% coverage)"
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
|
|
- name: Install dependencies
|
|
run: poetry install --with dev
|
|
|
|
- name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ffmpeg
|
|
|
|
- name: Install spaCy language models
|
|
run: poetry run python -m spacy download en_core_web_sm
|
|
|
|
- name: Run unit tests
|
|
run: |
|
|
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70
|
|
|
|
- name: Upload coverage reports
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
files: ./coverage.xml
|
|
flags: unittests
|
|
name: codecov-umbrella
|
|
|
|
package-integration-tests:
|
|
name: "Package - Integration Tests"
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
|
|
- name: Install dependencies
|
|
run: poetry install --with dev
|
|
|
|
- name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ffmpeg
|
|
|
|
- name: Install spaCy language models
|
|
run: poetry run python -m spacy download en_core_web_sm
|
|
|
|
- name: Run integration tests
|
|
run: poetry run pytest tests/integration/ -v
|
|
|
|
package-build:
|
|
name: "Package - Build & Install"
|
|
runs-on: ubuntu-latest
|
|
needs: [package-unit-tests, package-integration-tests]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
|
|
- name: Build package
|
|
run: poetry build
|
|
|
|
- name: Free disk space
|
|
uses: jlumbroso/free-disk-space@main
|
|
with:
|
|
tool-cache: false
|
|
android: true
|
|
haskell: true
|
|
dotnet: false
|
|
large-packages: false
|
|
docker-images: false
|
|
swap-storage: false
|
|
|
|
- name: Test package installation
|
|
run: |
|
|
pip install dist/*.whl
|
|
python -m karaoke_gen.utils.gen_cli --help
|
|
|
|
- name: Upload build artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: package-dist
|
|
path: dist/
|
|
retention-days: 1
|
|
|
|
# ============================================
|
|
# BACKEND TESTS (Cloud Run API service)
|
|
# ============================================
|
|
|
|
backend-unit-tests:
|
|
name: "Backend - Unit Tests"
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: false
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
|
|
- name: Install dependencies
|
|
run: poetry install
|
|
|
|
- name: Run unit tests
|
|
env:
|
|
GOOGLE_CLOUD_PROJECT: test-project
|
|
GCS_BUCKET_NAME: test-bucket
|
|
FIRESTORE_COLLECTION: jobs
|
|
ENVIRONMENT: test
|
|
run: |
|
|
poetry run pytest \
|
|
backend/tests/test_models.py \
|
|
backend/tests/test_job_manager.py \
|
|
backend/tests/test_file_upload.py \
|
|
backend/tests/test_services.py \
|
|
-v --tb=short
|
|
|
|
backend-emulator-tests:
|
|
name: "Backend - Emulator Integration Tests"
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
df -h
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: false
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
|
|
- name: Set up Java 21 (required by Firestore emulator)
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: "temurin"
|
|
java-version: "21"
|
|
|
|
- name: Set up Cloud SDK
|
|
uses: google-github-actions/setup-gcloud@v2
|
|
with:
|
|
version: "latest"
|
|
|
|
- name: Install Firestore emulator
|
|
run: gcloud components install beta cloud-firestore-emulator --quiet
|
|
|
|
- name: Install dependencies
|
|
run: poetry install
|
|
|
|
- name: Start Docker for GCS emulator
|
|
run: docker pull fsouza/fake-gcs-server:latest
|
|
|
|
- name: Run emulator integration tests
|
|
run: |
|
|
chmod +x scripts/*.sh
|
|
./scripts/run-emulator-tests.sh
|
|
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: backend-test-results
|
|
path: backend/htmlcov/
|
|
retention-days: 7
|
|
|
|
backend-lint:
|
|
name: "Backend - Code Quality"
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: false
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
|
|
- name: Install dependencies
|
|
run: poetry install
|
|
|
|
- name: Run validation (syntax check only)
|
|
run: |
|
|
poetry run python -m py_compile backend/main.py backend/config.py
|
|
echo "✅ Basic syntax validation passed"
|
|
|
|
# ============================================
|
|
# DEPLOYMENTS (main branch only)
|
|
# ============================================
|
|
|
|
package-publish:
|
|
name: "Deploy - Publish to PyPI"
|
|
runs-on: ubuntu-latest
|
|
needs: [package-build]
|
|
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Python 3.13
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.13"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1
|
|
|
|
- name: Download build artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: package-dist
|
|
path: dist/
|
|
|
|
- name: Extract version
|
|
id: extract_version
|
|
run: |
|
|
VERSION=$(poetry version --short)
|
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
echo "📋 Extracted version: $VERSION"
|
|
|
|
- name: Check if tag exists
|
|
id: check_tag
|
|
run: |
|
|
VERSION="${{ steps.extract_version.outputs.version }}"
|
|
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
|
|
echo "tag_exists=true" >> $GITHUB_OUTPUT
|
|
echo "🏷️ Tag v$VERSION already exists, skipping release"
|
|
else
|
|
echo "tag_exists=false" >> $GITHUB_OUTPUT
|
|
echo "✨ Tag v$VERSION does not exist, proceeding with release"
|
|
fi
|
|
|
|
- name: Publish to PyPI
|
|
if: steps.check_tag.outputs.tag_exists == 'false'
|
|
run: |
|
|
echo "📦 Publishing the following files to PyPI:"
|
|
ls -la dist/
|
|
echo "🚀 Publishing to PyPI..."
|
|
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
|
|
poetry publish
|
|
|
|
- name: Create Git Tag
|
|
if: steps.check_tag.outputs.tag_exists == 'false'
|
|
run: |
|
|
git config user.name github-actions
|
|
git config user.email github-actions@github.com
|
|
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
|
|
git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}"
|
|
|
|
- name: Create GitHub Release
|
|
if: steps.check_tag.outputs.tag_exists == 'false'
|
|
uses: softprops/action-gh-release@v1
|
|
with:
|
|
tag_name: "v${{ steps.extract_version.outputs.version }}"
|
|
name: "v${{ steps.extract_version.outputs.version }}"
|
|
body: |
|
|
## Changes in v${{ steps.extract_version.outputs.version }}
|
|
|
|
This release was automatically created from the main branch.
|
|
|
|
### Installation
|
|
```bash
|
|
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
|
|
```
|
|
|
|
### Assets
|
|
- Python package published to PyPI
|
|
- Frontend deployed to GitHub Pages at gen.nomadkaraoke.com
|
|
|
|
For detailed changes, see the commit history.
|
|
draft: false
|
|
prerelease: false
|
|
files: |
|
|
dist/*.whl
|
|
dist/*.tar.gz
|
|
|
|
deploy-frontend:
|
|
name: "Deploy - Frontend (GitHub Pages)"
|
|
runs-on: ubuntu-latest
|
|
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
|
environment:
|
|
name: github-pages
|
|
url: ${{ steps.deployment.outputs.page_url }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Pages
|
|
uses: actions/configure-pages@v4
|
|
|
|
- name: Prepare frontend files
|
|
run: |
|
|
echo "📁 Preparing frontend files for deployment..."
|
|
mkdir -p _site
|
|
cp -r frontend/* _site/
|
|
echo "✅ Frontend files copied to _site/"
|
|
ls -la _site/
|
|
|
|
- name: Upload artifact
|
|
uses: actions/upload-pages-artifact@v3
|
|
with:
|
|
path: "_site"
|
|
|
|
- name: Deploy to GitHub Pages
|
|
id: deployment
|
|
uses: actions/deploy-pages@v4
|
|
|
|
deploy-backend:
|
|
name: "Deploy - Backend (Cloud Run)"
|
|
runs-on: ubuntu-latest
|
|
needs: [backend-unit-tests, backend-emulator-tests, backend-lint]
|
|
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: false
|
|
|
|
- name: Authenticate to Google Cloud (Workload Identity)
|
|
id: auth
|
|
uses: google-github-actions/auth@v2
|
|
with:
|
|
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
|
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
|
|
|
- name: Set up Cloud SDK
|
|
uses: google-github-actions/setup-gcloud@v2
|
|
with:
|
|
version: "latest"
|
|
|
|
- name: Check if base image needs rebuild
|
|
id: check-base
|
|
run: |
|
|
echo "🔍 Checking if base image needs rebuild..."
|
|
|
|
CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1)
|
|
echo "Current poetry.lock hash: $CURRENT_HASH"
|
|
|
|
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
|
|
|
|
BASE_IMAGE="us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest"
|
|
EXISTING_HASH=""
|
|
|
|
if docker pull $BASE_IMAGE 2>/dev/null; then
|
|
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
|
|
echo "Existing base image hash: $EXISTING_HASH"
|
|
else
|
|
echo "⚠️ No existing base image found"
|
|
fi
|
|
|
|
if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then
|
|
echo "🔄 Base image needs rebuild (poetry.lock changed)"
|
|
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "✅ Base image is up to date"
|
|
echo "needs_rebuild=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Rebuild base image (if needed)
|
|
if: steps.check-base.outputs.needs_rebuild == 'true'
|
|
run: |
|
|
echo "🏗️ Rebuilding base image with updated dependencies..."
|
|
|
|
BASE_BUILD_ID=$(gcloud builds submit \
|
|
--config=cloudbuild-base.yaml \
|
|
--project=nomadkaraoke \
|
|
--format='value(id)' \
|
|
--async \
|
|
.)
|
|
|
|
echo "Base Build ID: $BASE_BUILD_ID"
|
|
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BASE_BUILD_ID?project=nomadkaraoke"
|
|
|
|
echo "Waiting for base image build to complete..."
|
|
while true; do
|
|
BUILD_STATUS=$(gcloud builds describe $BASE_BUILD_ID --project=nomadkaraoke --format='value(status)')
|
|
echo "Base build status: $BUILD_STATUS"
|
|
|
|
if [ "$BUILD_STATUS" = "SUCCESS" ]; then
|
|
echo "✅ Base image build completed successfully"
|
|
break
|
|
elif [ "$BUILD_STATUS" = "FAILURE" ] || [ "$BUILD_STATUS" = "CANCELLED" ] || [ "$BUILD_STATUS" = "TIMEOUT" ]; then
|
|
echo "❌ Base image build failed with status: $BUILD_STATUS"
|
|
echo ""
|
|
echo "==================== BUILD LOGS ===================="
|
|
gcloud logging read "resource.type=build AND resource.labels.build_id=$BASE_BUILD_ID" \
|
|
--project=nomadkaraoke \
|
|
--format="value(textPayload)" \
|
|
--limit=200 \
|
|
--order=asc || echo "Failed to retrieve logs from Cloud Logging"
|
|
echo "===================================================="
|
|
gcloud builds log $BASE_BUILD_ID --project=nomadkaraoke 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
sleep 30
|
|
done
|
|
|
|
- name: Trigger Cloud Build
|
|
run: |
|
|
echo "🚀 Triggering Cloud Build for deployment..."
|
|
echo "Commit: ${{ github.sha }}"
|
|
|
|
BUILD_ID=$(gcloud builds submit \
|
|
--config=cloudbuild.yaml \
|
|
--project=nomadkaraoke \
|
|
--format='value(id)' \
|
|
--async \
|
|
.)
|
|
|
|
echo "Build ID: $BUILD_ID"
|
|
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke"
|
|
|
|
echo "Waiting for build to complete..."
|
|
while true; do
|
|
BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)')
|
|
echo "Build status: $BUILD_STATUS"
|
|
|
|
if [ "$BUILD_STATUS" = "SUCCESS" ]; then
|
|
echo "✅ Build completed successfully"
|
|
break
|
|
elif [ "$BUILD_STATUS" = "FAILURE" ] || [ "$BUILD_STATUS" = "CANCELLED" ] || [ "$BUILD_STATUS" = "TIMEOUT" ]; then
|
|
echo "❌ Build failed with status: $BUILD_STATUS"
|
|
echo ""
|
|
echo "==================== BUILD LOGS ===================="
|
|
gcloud logging read "resource.type=build AND resource.labels.build_id=$BUILD_ID" \
|
|
--project=nomadkaraoke \
|
|
--format="value(textPayload)" \
|
|
--limit=200 \
|
|
--order=asc || echo "Failed to retrieve logs from Cloud Logging"
|
|
echo "===================================================="
|
|
echo ""
|
|
gcloud builds log $BUILD_ID --project=nomadkaraoke 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
sleep 30
|
|
done
|
|
|
|
- name: Verify deployment
|
|
run: |
|
|
SERVICE_URL=$(gcloud run services describe karaoke-backend \
|
|
--region us-central1 \
|
|
--format 'value(status.url)')
|
|
|
|
echo "Service URL: $SERVICE_URL"
|
|
sleep 10
|
|
curl -f "$SERVICE_URL/api/health" || exit 1
|
|
echo "✅ Deployment verified - service is healthy"
|
|
|
|
- name: Summary
|
|
if: success()
|
|
run: |
|
|
echo "🎉 Deployment successful!"
|
|
echo "Commit: ${{ github.sha }}"
|
|
echo "Branch: ${{ github.ref_name }}"
|
|
echo "Run: ${{ github.run_number }}"
|