mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-21 05:18:08 +09:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7b5a150e81 | ||
|
|
0b57b8f610 | ||
|
|
050ca6a441 | ||
|
|
2725352d89 | ||
|
|
762cdf03c9 | ||
|
|
7df3a21525 |
No files matched your search
@@ -235,7 +235,7 @@ jobs:
|
||||
|
||||
package-build-test:
|
||||
name: "Package - Build & Install (Test)"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, linux, gcp]
|
||||
needs: [changes, package-unit-tests, package-integration-tests]
|
||||
if: >-
|
||||
github.ref != 'refs/heads/main' && always() &&
|
||||
@@ -258,28 +258,9 @@ jobs:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Cache Poetry dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: .venv
|
||||
key: venv-package-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||||
restore-keys: |
|
||||
venv-package-${{ runner.os }}-py3.13-
|
||||
|
||||
- name: Build package
|
||||
run: poetry build
|
||||
|
||||
- name: Free disk space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
haskell: true
|
||||
dotnet: false
|
||||
large-packages: false
|
||||
docker-images: false
|
||||
swap-storage: false
|
||||
|
||||
- name: Test package installation
|
||||
run: |
|
||||
pip install dist/*.whl
|
||||
@@ -291,17 +272,10 @@ jobs:
|
||||
|
||||
backend-unit-tests:
|
||||
name: "Backend - Unit Tests"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, linux, gcp]
|
||||
needs: changes
|
||||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||||
steps:
|
||||
- name: Free disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /usr/local/share/boost
|
||||
df -h
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -318,14 +292,6 @@ jobs:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Cache Poetry dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: .venv
|
||||
key: venv-backend-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||||
restore-keys: |
|
||||
venv-backend-${{ runner.os }}-py3.13-
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install
|
||||
|
||||
@@ -343,21 +309,10 @@ jobs:
|
||||
|
||||
backend-emulator-tests:
|
||||
name: "Backend - Emulator Integration Tests"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, linux, gcp]
|
||||
needs: changes
|
||||
if: github.ref != 'refs/heads/main' && (needs.changes.outputs.backend == 'true' || needs.changes.outputs.package == 'true' || needs.changes.outputs.deps == 'true')
|
||||
steps:
|
||||
- name: Free disk space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
haskell: true
|
||||
dotnet: true
|
||||
large-packages: false
|
||||
docker-images: false
|
||||
swap-storage: false
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -374,28 +329,12 @@ jobs:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Cache Poetry dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: .venv
|
||||
key: venv-backend-${{ runner.os }}-py3.13-${{ hashFiles('poetry.lock') }}
|
||||
restore-keys: |
|
||||
venv-backend-${{ runner.os }}-py3.13-
|
||||
|
||||
- name: Set up Java 21 (required by Firestore emulator)
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: "temurin"
|
||||
java-version: "21"
|
||||
|
||||
- name: Set up Cloud SDK
|
||||
uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
version: "latest"
|
||||
|
||||
- name: Install Firestore emulator
|
||||
run: gcloud components install beta cloud-firestore-emulator --quiet
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install
|
||||
|
||||
@@ -687,7 +626,7 @@ jobs:
|
||||
|
||||
deploy-backend:
|
||||
name: "Deploy - Backend (Cloud Run)"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, linux, gcp]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -701,7 +640,6 @@ jobs:
|
||||
steps:
|
||||
# ============================================
|
||||
# Check if deployment is needed FIRST
|
||||
# This avoids wasting 3+ mins on disk cleanup when skipping
|
||||
# ============================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -709,7 +647,7 @@ jobs:
|
||||
submodules: false
|
||||
fetch-depth: 0 # Need full history for tag check
|
||||
|
||||
- name: Set up Python (for version extraction)
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
@@ -733,21 +671,7 @@ jobs:
|
||||
echo "✨ Tag v$VERSION does not exist - proceeding with deployment"
|
||||
fi
|
||||
|
||||
# ============================================
|
||||
# Free disk space only if deploying
|
||||
# This prevents "No space left on device" errors during Docker build
|
||||
# ============================================
|
||||
- name: Free disk space (aggressive)
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
haskell: true
|
||||
dotnet: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
# No disk cleanup needed - self-hosted runner has 200GB SSD
|
||||
|
||||
- name: Authenticate to Google Cloud (Workload Identity)
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
@@ -931,9 +855,8 @@ jobs:
|
||||
echo "Branch: ${{ github.ref_name }}"
|
||||
echo "Run: ${{ github.run_number }}"
|
||||
echo ""
|
||||
echo "📊 Cost savings: This build used FREE GitHub Actions instead of Cloud Build"
|
||||
echo " Previously: ~\$0.064/min with N1_HIGHCPU_32"
|
||||
echo " Now: \$0 (public repo) with registry-cached builds"
|
||||
echo "📊 Built on self-hosted GCP runner (200GB SSD, no disk space issues)"
|
||||
echo " Runner: gcp-runner with 4 vCPU, 16GB RAM"
|
||||
|
||||
- name: Skipped deployment summary
|
||||
if: steps.version_check.outputs.needs_deploy == 'false'
|
||||
|
||||
@@ -76,7 +76,7 @@ Run Whisper directly on your local machine using whisper-timestamped. Works on C
|
||||
|
||||
```bash
|
||||
# Install with local Whisper support
|
||||
pip install karaoke-gen[local-whisper]
|
||||
pip install "karaoke-gen[local-whisper]"
|
||||
|
||||
# Optional: Configure model size (tiny, base, small, medium, large)
|
||||
export WHISPER_MODEL_SIZE="medium"
|
||||
@@ -98,7 +98,7 @@ export WHISPER_DEVICE="cpu"
|
||||
```bash
|
||||
# Pre-install CPU-only PyTorch first
|
||||
pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu
|
||||
pip install karaoke-gen[local-whisper]
|
||||
pip install "karaoke-gen[local-whisper]"
|
||||
```
|
||||
|
||||
Local Whisper runs automatically as a fallback when no cloud transcription services are configured.
|
||||
@@ -633,7 +633,7 @@ Whisper models are downloaded on first use (~1-3GB depending on size). If downlo
|
||||
#### whisper-timestamped Not Found
|
||||
If you get "whisper-timestamped is not installed":
|
||||
```bash
|
||||
pip install karaoke-gen[local-whisper]
|
||||
pip install "karaoke-gen[local-whisper]"
|
||||
# Or install directly:
|
||||
pip install whisper-timestamped
|
||||
```
|
||||
|
||||
@@ -126,6 +126,19 @@ Migrated from Cloud Build to GitHub Actions for:
|
||||
- Better visibility
|
||||
- Faster feedback
|
||||
|
||||
### Self-Hosted Runner on GCP
|
||||
|
||||
GitHub-hosted runners have ~14GB disk, which caused failures during Docker builds. Solution: self-hosted runner on GCP with 200GB SSD.
|
||||
|
||||
**Key learnings:**
|
||||
|
||||
1. **GPG in startup scripts**: Always use `gpg --batch` for non-interactive execution
|
||||
2. **setup-python on self-hosted**: Needs Python in tool cache at `$RUNNER/_work/_tool/Python/<version>/x64/` with `x64.complete` marker
|
||||
3. **Debian vs Ubuntu**: PPAs don't work on Debian; use upstream repos (Temurin, NodeSource)
|
||||
4. **pyenv for Python**: More reliable than source builds or distro packages for specific versions
|
||||
|
||||
See `docs/archive/2025-12-28-self-hosted-github-runner.md` for full details.
|
||||
|
||||
## Performance Observations
|
||||
|
||||
| Operation | Duration |
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
| Multi-format encoding | Working |
|
||||
| Token-based auth | Working |
|
||||
| Payment flow | Not started |
|
||||
| CI/CD self-hosted runner | Working (GCP) |
|
||||
|
||||
## Known Issues
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
# Self-Hosted GitHub Actions Runner on GCP
|
||||
|
||||
**Date**: 2025-12-28
|
||||
**Status**: Deployed and operational
|
||||
|
||||
## Problem
|
||||
|
||||
GitHub-hosted runners have ~14GB disk space. CI jobs were failing with "No space left on device" during:
|
||||
- Docker builds (multi-stage, large dependencies)
|
||||
- Emulator tests (Docker + Java + Firestore/GCS emulators)
|
||||
- Package builds with poetry
|
||||
|
||||
Workarounds like `jlumbroso/free-disk-space` action helped but weren't reliable.
|
||||
|
||||
## Solution
|
||||
|
||||
Deployed a self-hosted runner on GCP with 200GB SSD via Pulumi IaC.
|
||||
|
||||
### Infrastructure Added
|
||||
|
||||
| Resource | Purpose |
|
||||
|----------|---------|
|
||||
| `github-runner` VM | e2-standard-4, 200GB SSD, Debian 12 |
|
||||
| `github-runner` service account | Minimal permissions for runner |
|
||||
| `github-runner-pat` secret | PAT for runner registration |
|
||||
| IAM bindings | Artifact Registry reader, Secret Manager accessor, logging |
|
||||
|
||||
### Jobs Moved to Self-Hosted
|
||||
|
||||
- `package-build-test`
|
||||
- `backend-unit-tests`
|
||||
- `backend-emulator-tests`
|
||||
- `deploy-backend`
|
||||
|
||||
These use `runs-on: [self-hosted, linux, gcp]` with labels.
|
||||
|
||||
## Issues Encountered
|
||||
|
||||
### 1. GPG tty errors in startup script
|
||||
|
||||
**Problem**: `gpg --dearmor` failed with "cannot open /dev/tty" in non-interactive startup.
|
||||
|
||||
**Fix**: Add `--batch` flag to all GPG commands.
|
||||
|
||||
### 2. setup-python can't find Python 3.13 on Debian 12
|
||||
|
||||
**Problem**: `actions/setup-python@v5` couldn't find pre-built Python 3.13 for Debian 12 on self-hosted runners.
|
||||
|
||||
**Fix**:
|
||||
1. Install Python 3.13 via pyenv on the runner
|
||||
2. Copy to tool cache: `_work/_tool/Python/3.13.0/x64/`
|
||||
3. Create marker file: `x64.complete`
|
||||
|
||||
### 3. Java 21 not in Debian repos
|
||||
|
||||
**Problem**: `openjdk-21-jdk` not available in Debian 12 stable.
|
||||
|
||||
**Fix**: Use Temurin (Adoptium) repository directly.
|
||||
|
||||
## Key Files
|
||||
|
||||
- `infrastructure/__main__.py` - Runner VM and IAM setup
|
||||
- `infrastructure/README.md` - Setup instructions
|
||||
- `.github/workflows/ci.yml` - Jobs using self-hosted runner
|
||||
|
||||
## Lessons for Future
|
||||
|
||||
1. **Self-hosted runners need explicit tool setup** - Unlike GitHub-hosted, tools aren't pre-installed in expected locations
|
||||
2. **setup-python tool cache location**: `$RUNNER_DIR/_work/_tool/Python/<version>/x64/`
|
||||
3. **GPG in scripts**: Always use `--batch` for non-interactive execution
|
||||
4. **Debian vs Ubuntu**: Many GitHub Actions assume Ubuntu; PPAs don't work on Debian
|
||||
|
||||
## Cost
|
||||
|
||||
~$50/month for e2-standard-4 running 24/7 (covered by GCP free credits).
|
||||
|
||||
## PRs
|
||||
|
||||
- #95: Initial self-hosted runner setup
|
||||
- #97: Make startup script reproducible (pyenv, tool cache, GPG fixes)
|
||||
@@ -254,10 +254,76 @@ The function uses environment variables:
|
||||
|
||||
Known gaps are configured in `main.py` and should match your local `validate_and_sync.py`.
|
||||
|
||||
## Self-Hosted GitHub Actions Runner
|
||||
|
||||
A dedicated GCP VM runs self-hosted GitHub Actions runners to solve disk space issues
|
||||
on GitHub-hosted runners (14GB vs 200GB).
|
||||
|
||||
### Specs
|
||||
|
||||
- **Instance**: e2-standard-4 (4 vCPU, 16GB RAM)
|
||||
- **Disk**: 200GB SSD
|
||||
- **Pre-installed**: Docker, Python 3.13, Node.js 20, Java 21, Poetry, FFmpeg, gcloud CLI
|
||||
- **Labels**: `self-hosted`, `linux`, `x64`, `gcp`, `large-disk`
|
||||
|
||||
### Setup
|
||||
|
||||
1. Create a GitHub Personal Access Token (PAT) with `repo` scope:
|
||||
- Go to https://github.com/settings/tokens
|
||||
- Generate new token (classic)
|
||||
- Select `repo` scope (full control of private repositories)
|
||||
- Copy the token
|
||||
|
||||
2. Add the PAT to Secret Manager:
|
||||
```bash
|
||||
echo -n "ghp_your_token_here" | \
|
||||
gcloud secrets versions add github-runner-pat --data-file=-
|
||||
```
|
||||
|
||||
3. Run `pulumi up` to create the runner VM
|
||||
|
||||
4. Verify the runner registered:
|
||||
- Go to https://github.com/nomadkaraoke/karaoke-gen/settings/actions/runners
|
||||
- You should see `gcp-runner-github-runner` with status "Idle"
|
||||
|
||||
### Usage in Workflows
|
||||
|
||||
Jobs that need more disk space use the self-hosted runner:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
build:
|
||||
runs-on: [self-hosted, linux, gcp]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# ... rest of job
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
**Runner not appearing in GitHub:**
|
||||
```bash
|
||||
# Check startup logs
|
||||
gcloud compute ssh github-runner --zone=us-central1-a -- \
|
||||
"sudo cat /var/log/github-runner-startup.log"
|
||||
```
|
||||
|
||||
**Re-register runner:**
|
||||
```bash
|
||||
# Restart the VM to re-run startup script
|
||||
gcloud compute instances reset github-runner --zone=us-central1-a
|
||||
```
|
||||
|
||||
**Check runner service status:**
|
||||
```bash
|
||||
gcloud compute ssh github-runner --zone=us-central1-a -- \
|
||||
"sudo systemctl status actions.runner.*"
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Run `pulumi up` to create infrastructure
|
||||
2. Add secret values
|
||||
2. Add secret values (including GitHub runner PAT)
|
||||
3. Build and deploy Cloud Run service
|
||||
4. Deploy GDrive validator function
|
||||
5. Update documentation to reference Pulumi workflow
|
||||
|
||||
@@ -1182,3 +1182,241 @@ pulumi.export("gdrive_validator_function_name", gdrive_validator_function.name)
|
||||
pulumi.export("gdrive_validator_scheduler_name", gdrive_validator_scheduler.name)
|
||||
pulumi.export("gdrive_validator_service_account", gdrive_validator_sa.email)
|
||||
|
||||
# ==================== Self-Hosted GitHub Actions Runner ====================
|
||||
# A dedicated VM running GitHub Actions runner for CI/CD jobs that need more
|
||||
# disk space than GitHub-hosted runners provide (14GB vs 200GB here).
|
||||
# This solves "No space left on device" errors during Docker builds.
|
||||
|
||||
# Service account for GitHub runner VM
|
||||
github_runner_sa = serviceaccount.Account(
|
||||
"github-runner-sa",
|
||||
account_id="github-runner",
|
||||
display_name="GitHub Actions Runner",
|
||||
description="Service account for self-hosted GitHub Actions runner VM",
|
||||
)
|
||||
|
||||
# Grant runner service account Artifact Registry reader (for pulling base images)
|
||||
github_runner_artifact_reader = gcp.projects.IAMMember(
|
||||
"github-runner-artifact-reader",
|
||||
project=project_id,
|
||||
role="roles/artifactregistry.reader",
|
||||
member=github_runner_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant runner service account Secret Manager access (for PAT)
|
||||
github_runner_secrets_iam = gcp.projects.IAMMember(
|
||||
"github-runner-secrets-access",
|
||||
project=project_id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=github_runner_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant runner service account logging permissions
|
||||
github_runner_logging_iam = gcp.projects.IAMMember(
|
||||
"github-runner-logging-access",
|
||||
project=project_id,
|
||||
role="roles/logging.logWriter",
|
||||
member=github_runner_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# GitHub PAT secret for runner registration
|
||||
# The PAT needs 'repo' scope to register a runner
|
||||
github_runner_pat_secret = secretmanager.Secret(
|
||||
"github-runner-pat",
|
||||
secret_id="github-runner-pat",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Startup script for the GitHub runner VM
|
||||
GITHUB_RUNNER_STARTUP_SCRIPT = """#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Log to a file for debugging
|
||||
exec > >(tee /var/log/github-runner-startup.log) 2>&1
|
||||
echo "Starting GitHub Actions runner setup at $(date)"
|
||||
|
||||
# System updates and core dependencies
|
||||
apt-get update
|
||||
apt-get install -y \\
|
||||
curl \\
|
||||
git \\
|
||||
jq \\
|
||||
unzip \\
|
||||
build-essential \\
|
||||
libssl-dev \\
|
||||
libffi-dev \\
|
||||
software-properties-common \\
|
||||
apt-transport-https \\
|
||||
ca-certificates \\
|
||||
gnupg \\
|
||||
lsb-release
|
||||
|
||||
# ==================== Docker ====================
|
||||
echo "Installing Docker..."
|
||||
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
|
||||
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
|
||||
apt-get update
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
|
||||
# Configure Docker for the runner user
|
||||
systemctl enable docker
|
||||
systemctl start docker
|
||||
|
||||
# ==================== Python 3.13 ====================
|
||||
echo "Installing Python 3.13..."
|
||||
add-apt-repository -y ppa:deadsnakes/ppa || true
|
||||
apt-get update
|
||||
apt-get install -y python3.13 python3.13-venv python3.13-dev python3-pip || {
|
||||
# Fallback: build from source if PPA not available
|
||||
echo "PPA not available, installing Python 3.13 from source..."
|
||||
apt-get install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libreadline-dev libffi-dev libsqlite3-dev wget libbz2-dev
|
||||
cd /tmp
|
||||
wget https://www.python.org/ftp/python/3.13.0/Python-3.13.0.tgz
|
||||
tar -xf Python-3.13.0.tgz
|
||||
cd Python-3.13.0
|
||||
./configure --enable-optimizations
|
||||
make -j $(nproc)
|
||||
make altinstall
|
||||
ln -sf /usr/local/bin/python3.13 /usr/bin/python3.13
|
||||
}
|
||||
|
||||
# ==================== Node.js 20 ====================
|
||||
echo "Installing Node.js 20..."
|
||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||||
apt-get install -y nodejs
|
||||
|
||||
# ==================== Java 21 (for Firestore emulator) ====================
|
||||
echo "Installing Java 21..."
|
||||
apt-get install -y openjdk-21-jdk || {
|
||||
# Fallback: use Temurin
|
||||
curl -fsSL https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor -o /usr/share/keyrings/adoptium.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/adoptium.list
|
||||
apt-get update
|
||||
apt-get install -y temurin-21-jdk
|
||||
}
|
||||
|
||||
# ==================== FFmpeg ====================
|
||||
echo "Installing FFmpeg..."
|
||||
apt-get install -y ffmpeg
|
||||
|
||||
# ==================== Poetry ====================
|
||||
echo "Installing Poetry..."
|
||||
curl -sSL https://install.python-poetry.org | python3 -
|
||||
ln -sf /root/.local/bin/poetry /usr/local/bin/poetry
|
||||
|
||||
# ==================== Google Cloud SDK ====================
|
||||
echo "Installing Google Cloud SDK..."
|
||||
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
apt-get update
|
||||
apt-get install -y google-cloud-cli google-cloud-cli-firestore-emulator
|
||||
|
||||
# ==================== Create runner user ====================
|
||||
echo "Creating runner user..."
|
||||
useradd -m -s /bin/bash runner || true
|
||||
usermod -aG docker runner
|
||||
|
||||
# ==================== GitHub Actions Runner ====================
|
||||
echo "Setting up GitHub Actions Runner..."
|
||||
RUNNER_VERSION="2.321.0"
|
||||
RUNNER_DIR="/home/runner/actions-runner"
|
||||
|
||||
mkdir -p $RUNNER_DIR
|
||||
cd $RUNNER_DIR
|
||||
|
||||
# Download runner
|
||||
curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz -L \\
|
||||
https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
|
||||
tar xzf actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
|
||||
rm actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
|
||||
|
||||
chown -R runner:runner $RUNNER_DIR
|
||||
|
||||
# Get GitHub PAT from Secret Manager
|
||||
echo "Fetching GitHub PAT from Secret Manager..."
|
||||
GITHUB_PAT=$(gcloud secrets versions access latest --secret=github-runner-pat)
|
||||
|
||||
if [ -z "$GITHUB_PAT" ]; then
|
||||
echo "ERROR: GitHub PAT not found in Secret Manager"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get runner registration token using PAT
|
||||
echo "Getting runner registration token..."
|
||||
REGISTRATION_TOKEN=$(curl -s -X POST \\
|
||||
-H "Authorization: token $GITHUB_PAT" \\
|
||||
-H "Accept: application/vnd.github.v3+json" \\
|
||||
https://api.github.com/repos/nomadkaraoke/karaoke-gen/actions/runners/registration-token | jq -r .token)
|
||||
|
||||
if [ -z "$REGISTRATION_TOKEN" ] || [ "$REGISTRATION_TOKEN" = "null" ]; then
|
||||
echo "ERROR: Failed to get registration token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Configure runner (non-interactive)
|
||||
echo "Configuring runner..."
|
||||
cd $RUNNER_DIR
|
||||
sudo -u runner ./config.sh \\
|
||||
--url https://github.com/nomadkaraoke/karaoke-gen \\
|
||||
--token "$REGISTRATION_TOKEN" \\
|
||||
--name "gcp-runner-$(hostname)" \\
|
||||
--labels "self-hosted,linux,x64,gcp,large-disk" \\
|
||||
--work "_work" \\
|
||||
--unattended \\
|
||||
--replace
|
||||
|
||||
# Install and start runner as a service
|
||||
echo "Installing runner service..."
|
||||
./svc.sh install runner
|
||||
./svc.sh start
|
||||
|
||||
echo "GitHub Actions runner setup complete at $(date)"
|
||||
echo "Runner registered with labels: self-hosted,linux,x64,gcp,large-disk"
|
||||
|
||||
# ==================== Docker cleanup cron ====================
|
||||
# Prevent disk from filling up with old Docker images
|
||||
echo "Setting up Docker cleanup cron..."
|
||||
cat > /etc/cron.daily/docker-cleanup << 'CRON'
|
||||
#!/bin/bash
|
||||
# Clean up Docker resources older than 7 days
|
||||
docker system prune -af --filter "until=168h"
|
||||
CRON
|
||||
chmod +x /etc/cron.daily/docker-cleanup
|
||||
|
||||
echo "Setup complete!"
|
||||
"""
|
||||
|
||||
# GitHub runner VM instance
|
||||
github_runner_vm = gcp.compute.Instance(
|
||||
"github-runner",
|
||||
name="github-runner",
|
||||
machine_type="e2-standard-4", # 4 vCPU, 16GB RAM - good for Docker builds
|
||||
zone="us-central1-a",
|
||||
boot_disk=gcp.compute.InstanceBootDiskArgs(
|
||||
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=200, # GB - plenty for Docker builds and caches
|
||||
type="pd-ssd", # SSD for faster I/O
|
||||
),
|
||||
),
|
||||
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs()], # Ephemeral IP is fine
|
||||
)],
|
||||
service_account=gcp.compute.InstanceServiceAccountArgs(
|
||||
email=github_runner_sa.email,
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
metadata_startup_script=GITHUB_RUNNER_STARTUP_SCRIPT,
|
||||
tags=["github-runner"],
|
||||
allow_stopping_for_update=True,
|
||||
# Ensure the secret exists before creating the VM
|
||||
opts=pulumi.ResourceOptions(depends_on=[github_runner_pat_secret]),
|
||||
)
|
||||
|
||||
# Export GitHub runner resources
|
||||
pulumi.export("github_runner_vm_name", github_runner_vm.name)
|
||||
pulumi.export("github_runner_service_account", github_runner_sa.email)
|
||||
|
||||
Reference in new issue
Block a user