mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-21 05:18:08 +09:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93c6254ab6 |
No files matched your search
@@ -37,7 +37,7 @@ from config import (
|
||||
DiskSizes,
|
||||
)
|
||||
from modules import database, storage as storage_module, artifact_registry, secrets
|
||||
from modules import cloud_tasks, cloud_run, monitoring, networking
|
||||
from modules import cloud_tasks, cloud_run, monitoring, networking, runner_manager
|
||||
from modules.iam import backend_sa, github_actions_sa, claude_automation_sa, worker_sas
|
||||
from compute import encoding_worker_vm, github_runners
|
||||
|
||||
@@ -249,8 +249,18 @@ encoding_worker_instance = encoding_worker_vm.create_encoding_worker_vm(
|
||||
encoding_worker_firewall = encoding_worker_vm.create_encoding_worker_firewall()
|
||||
|
||||
# GitHub Runners (CI/CD self-hosted runners)
|
||||
# Create Cloud NAT for outbound internet access (no external IPs needed)
|
||||
github_runners_router, github_runners_nat = github_runners.create_cloud_nat()
|
||||
|
||||
# Create runner VMs (Spot instances for 60-91% cost savings)
|
||||
github_runner_vms = github_runners.create_github_runners(
|
||||
github_runner_sa, all_secrets["github-runner-pat"]
|
||||
github_runner_sa, all_secrets["github-runner-pat"], github_runners_nat
|
||||
)
|
||||
|
||||
# Create runner manager (auto-start/stop VMs based on CI activity)
|
||||
runner_manager_resources = runner_manager.create_runner_manager_resources(
|
||||
all_secrets["github-webhook-secret"],
|
||||
all_secrets["github-runner-pat"],
|
||||
)
|
||||
|
||||
|
||||
@@ -334,3 +344,8 @@ pulumi.export("encoding_worker_vm_name", encoding_worker_instance.name)
|
||||
# GitHub runners
|
||||
pulumi.export("github_runner_vm_names", [vm.name for vm in github_runner_vms])
|
||||
pulumi.export("github_runner_service_account", github_runner_sa.email)
|
||||
pulumi.export("github_runners_nat", github_runners_nat.name)
|
||||
|
||||
# GitHub runner manager (auto-start/stop)
|
||||
pulumi.export("runner_manager_function_url", runner_manager_resources["function"].url)
|
||||
pulumi.export("runner_manager_scheduler_job", runner_manager_resources["scheduler_job"].name)
|
||||
@@ -11,11 +11,12 @@ from .encoding_worker_vm import (
|
||||
create_encoding_worker_ip,
|
||||
create_encoding_worker_firewall,
|
||||
)
|
||||
from .github_runners import create_github_runners
|
||||
from .github_runners import create_github_runners, create_cloud_nat
|
||||
|
||||
__all__ = [
|
||||
"create_encoding_worker_vm",
|
||||
"create_encoding_worker_ip",
|
||||
"create_encoding_worker_firewall",
|
||||
"create_github_runners",
|
||||
"create_cloud_nat",
|
||||
]
|
||||
@@ -1,30 +1,82 @@
|
||||
"""
|
||||
GitHub Actions self-hosted runner VMs.
|
||||
GitHub Actions Self-Hosted Runners on GCP.
|
||||
|
||||
Manages multiple VM instances that serve as self-hosted runners
|
||||
for GitHub Actions CI/CD workflows.
|
||||
Creates Spot/Preemptible VMs that register as self-hosted runners
|
||||
at the organization level. Uses Spot instances for 60-91% cost savings.
|
||||
|
||||
Resources created:
|
||||
- Cloud Router and Cloud NAT (for outbound internet without external IPs)
|
||||
- N Compute Engine VMs (Spot instances)
|
||||
- Each VM runs the GitHub Actions runner agent
|
||||
|
||||
The VMs are managed by the runner_manager Cloud Function which:
|
||||
- Starts VMs when CI jobs are queued (via GitHub webhook)
|
||||
- Stops VMs when idle for too long (via Cloud Scheduler)
|
||||
"""
|
||||
|
||||
import pulumi
|
||||
from pulumi_gcp import compute, serviceaccount, secretmanager
|
||||
from pulumi_gcp import compute, secretmanager, serviceaccount
|
||||
|
||||
from config import ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
|
||||
from config import REGION, ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
|
||||
from .startup_scripts import read_script
|
||||
|
||||
|
||||
def create_cloud_nat() -> tuple[compute.Router, compute.RouterNat]:
|
||||
"""
|
||||
Create Cloud Router and Cloud NAT for outbound internet access.
|
||||
|
||||
This allows VMs without external IPs to access the internet,
|
||||
which saves costs and improves security.
|
||||
|
||||
Returns:
|
||||
tuple: (router, nat) resources
|
||||
"""
|
||||
# Create Cloud Router
|
||||
router = compute.Router(
|
||||
"github-runners-router",
|
||||
name="github-runners-router",
|
||||
region=REGION,
|
||||
network="default",
|
||||
description="Router for GitHub Actions runner NAT",
|
||||
)
|
||||
|
||||
# Create Cloud NAT
|
||||
nat = compute.RouterNat(
|
||||
"github-runners-nat",
|
||||
name="github-runners-nat",
|
||||
router=router.name,
|
||||
region=REGION,
|
||||
nat_ip_allocate_option="AUTO_ONLY",
|
||||
source_subnetwork_ip_ranges_to_nat="ALL_SUBNETWORKS_ALL_IP_RANGES",
|
||||
log_config=compute.RouterNatLogConfigArgs(
|
||||
enable=True,
|
||||
filter="ERRORS_ONLY",
|
||||
),
|
||||
opts=pulumi.ResourceOptions(depends_on=[router]),
|
||||
)
|
||||
|
||||
return router, nat
|
||||
|
||||
|
||||
def create_github_runners(
|
||||
service_account: serviceaccount.Account,
|
||||
runner_pat_secret: secretmanager.Secret,
|
||||
nat: compute.RouterNat,
|
||||
) -> list[compute.Instance]:
|
||||
"""
|
||||
Create GitHub Actions self-hosted runner VM instances.
|
||||
|
||||
Creates NUM_GITHUB_RUNNERS VMs, each configured as a GitHub Actions
|
||||
Creates NUM_GITHUB_RUNNERS Spot VMs, each configured as a GitHub Actions
|
||||
self-hosted runner for the nomadkaraoke organization (available to all repos).
|
||||
|
||||
Uses Spot instances (preemptible) for 60-91% cost savings.
|
||||
VMs start in TERMINATED state and are started by the runner_manager
|
||||
Cloud Function when CI jobs are queued.
|
||||
|
||||
Args:
|
||||
service_account: The GitHub runner service account.
|
||||
runner_pat_secret: The Secret Manager secret containing the GitHub PAT.
|
||||
nat: Cloud NAT for outbound internet access (VMs have no external IPs).
|
||||
|
||||
Returns:
|
||||
list[compute.Instance]: List of runner VM instances.
|
||||
@@ -33,33 +85,91 @@ def create_github_runners(
|
||||
|
||||
runner_vms = []
|
||||
for i in range(1, NUM_GITHUB_RUNNERS + 1):
|
||||
instance_name = f"github-runner-{i}"
|
||||
|
||||
vm = compute.Instance(
|
||||
f"github-runner-{i}",
|
||||
name=f"github-runner-{i}",
|
||||
instance_name,
|
||||
name=instance_name,
|
||||
machine_type=MachineTypes.GITHUB_RUNNER,
|
||||
zone=ZONE,
|
||||
# Boot disk configuration
|
||||
boot_disk=compute.InstanceBootDiskArgs(
|
||||
initialize_params=compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=DiskSizes.GITHUB_RUNNER,
|
||||
type="pd-ssd",
|
||||
type="pd-ssd", # SSD for faster Docker builds
|
||||
),
|
||||
auto_delete=True,
|
||||
),
|
||||
network_interfaces=[compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[compute.InstanceNetworkInterfaceAccessConfigArgs()],
|
||||
)],
|
||||
# Network configuration - no external IP (uses Cloud NAT)
|
||||
# This saves ~$3/month per VM and improves security
|
||||
network_interfaces=[
|
||||
compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
# No access_configs = no external IP
|
||||
# Outbound internet via Cloud NAT
|
||||
)
|
||||
],
|
||||
# Service account for GCP API access
|
||||
service_account=compute.InstanceServiceAccountArgs(
|
||||
email=service_account.email,
|
||||
scopes=["cloud-platform"],
|
||||
scopes=["https://www.googleapis.com/auth/cloud-platform"],
|
||||
),
|
||||
# Spot/Preemptible instance configuration (60-91% cheaper)
|
||||
# Preemptible VMs may be terminated at any time but cost much less.
|
||||
# The startup script re-registers the runner on boot.
|
||||
scheduling=compute.InstanceSchedulingArgs(
|
||||
preemptible=True,
|
||||
automatic_restart=False, # Required for preemptible
|
||||
on_host_maintenance="TERMINATE", # Required for preemptible
|
||||
),
|
||||
# Metadata for startup script
|
||||
metadata={
|
||||
"github-runner-pat-secret": "github-runner-pat",
|
||||
"github-org": "nomadkaraoke",
|
||||
"runner-labels": "self-hosted,linux,x64,gcp,large-disk",
|
||||
},
|
||||
metadata_startup_script=startup_script,
|
||||
# Labels for organization and cost tracking
|
||||
labels={
|
||||
"purpose": "github-runner",
|
||||
"managed-by": "pulumi",
|
||||
},
|
||||
tags=["github-runner"],
|
||||
# Allow stopping for updates
|
||||
allow_stopping_for_update=True,
|
||||
# Deletion protection disabled (runners are ephemeral)
|
||||
deletion_protection=False,
|
||||
opts=pulumi.ResourceOptions(
|
||||
depends_on=[runner_pat_secret],
|
||||
depends_on=[runner_pat_secret, nat],
|
||||
# Runners can be replaced without affecting other infrastructure
|
||||
delete_before_replace=True,
|
||||
),
|
||||
)
|
||||
|
||||
runner_vms.append(vm)
|
||||
|
||||
return runner_vms
|
||||
|
||||
|
||||
def create_instance_group_for_restart(
|
||||
instances: list[compute.Instance],
|
||||
) -> compute.InstanceGroup:
|
||||
"""
|
||||
Create an unmanaged instance group for the runner VMs.
|
||||
|
||||
This enables bulk operations like restarting all runners.
|
||||
|
||||
Args:
|
||||
instances: List of runner VM instances.
|
||||
|
||||
Returns:
|
||||
compute.InstanceGroup: The instance group.
|
||||
"""
|
||||
return compute.InstanceGroup(
|
||||
"github-runners-group",
|
||||
name="github-runners",
|
||||
description="Self-hosted GitHub Actions runners",
|
||||
zone=ZONE,
|
||||
instances=[instance.self_link for instance in instances],
|
||||
)
|
||||
@@ -87,11 +87,29 @@ class QueueConfigs:
|
||||
|
||||
|
||||
# Number of self-hosted GitHub Action runners
|
||||
NUM_GITHUB_RUNNERS = 20
|
||||
# Reduced from 20 to 3 - runners auto-scale with start/stop management
|
||||
NUM_GITHUB_RUNNERS = 3
|
||||
|
||||
# Google Drive folder ID for validator (public share folder)
|
||||
GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
|
||||
|
||||
|
||||
class RunnerManagerConfig:
|
||||
"""Configuration for GitHub runner auto-start/stop management."""
|
||||
|
||||
FUNCTION_NAME = "github-runner-manager"
|
||||
FUNCTION_MEMORY = "256M"
|
||||
FUNCTION_TIMEOUT = 300 # 5 minutes
|
||||
IDLE_TIMEOUT_HOURS = 1 # Shorter than Aquarius (3h) since we have more CI activity
|
||||
IDLE_CHECK_SCHEDULE = "*/15 * * * *" # Every 15 minutes
|
||||
|
||||
|
||||
class SecretNames:
|
||||
"""Secret Manager secret names."""
|
||||
|
||||
GITHUB_RUNNER_PAT = "github-runner-pat"
|
||||
GITHUB_WEBHOOK_SECRET = "github-webhook-secret"
|
||||
|
||||
# GitHub repository for runner registration
|
||||
GITHUB_REPO_OWNER = "nomadkaraoke"
|
||||
GITHUB_REPO_NAME = "karaoke-gen"
|
||||
@@ -0,0 +1,417 @@
|
||||
"""
|
||||
GitHub Actions Runner Manager Cloud Function.
|
||||
|
||||
Handles two types of triggers:
|
||||
1. GitHub Webhook (workflow_job.queued) - Starts runner VMs when CI jobs are queued
|
||||
2. Cloud Scheduler (every 15 min) - Stops idle runner VMs after timeout
|
||||
|
||||
Environment variables:
|
||||
- GCP_PROJECT: GCP project ID
|
||||
- GCP_ZONE: Zone where runner VMs are located
|
||||
- WEBHOOK_SECRET_NAME: Secret Manager secret name for webhook verification
|
||||
- RUNNER_PAT_SECRET_NAME: Secret Manager secret name for GitHub PAT
|
||||
- IDLE_TIMEOUT_HOURS: Hours of inactivity before stopping runners (default: 1)
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import functions_framework
|
||||
from flask import Request
|
||||
from google.cloud import compute_v1, secretmanager
|
||||
|
||||
# Configuration from environment
|
||||
PROJECT_ID = os.environ.get("GCP_PROJECT", "nomadkaraoke")
|
||||
ZONE = os.environ.get("GCP_ZONE", "us-central1-a")
|
||||
WEBHOOK_SECRET_NAME = os.environ.get("WEBHOOK_SECRET_NAME", "github-webhook-secret")
|
||||
RUNNER_PAT_SECRET_NAME = os.environ.get("RUNNER_PAT_SECRET_NAME", "github-runner-pat")
|
||||
IDLE_TIMEOUT_HOURS = int(os.environ.get("IDLE_TIMEOUT_HOURS", "1"))
|
||||
RUNNER_PREFIX = "github-runner-"
|
||||
NUM_RUNNERS = 3
|
||||
|
||||
# Lazy-loaded clients and secrets
|
||||
_compute_client = None
|
||||
_secret_client = None
|
||||
_webhook_secret = None
|
||||
_github_pat = None
|
||||
|
||||
|
||||
def get_compute_client() -> compute_v1.InstancesClient:
|
||||
"""Get or create the Compute Engine client."""
|
||||
global _compute_client
|
||||
if _compute_client is None:
|
||||
_compute_client = compute_v1.InstancesClient()
|
||||
return _compute_client
|
||||
|
||||
|
||||
def get_secret_client() -> secretmanager.SecretManagerServiceClient:
|
||||
"""Get or create the Secret Manager client."""
|
||||
global _secret_client
|
||||
if _secret_client is None:
|
||||
_secret_client = secretmanager.SecretManagerServiceClient()
|
||||
return _secret_client
|
||||
|
||||
|
||||
def get_webhook_secret() -> str:
|
||||
"""Get the webhook secret from Secret Manager (cached)."""
|
||||
global _webhook_secret
|
||||
if _webhook_secret is None:
|
||||
client = get_secret_client()
|
||||
name = f"projects/{PROJECT_ID}/secrets/{WEBHOOK_SECRET_NAME}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
_webhook_secret = response.payload.data.decode("utf-8").strip()
|
||||
return _webhook_secret
|
||||
|
||||
|
||||
def get_github_pat() -> str:
|
||||
"""Get the GitHub PAT from Secret Manager (cached)."""
|
||||
global _github_pat
|
||||
if _github_pat is None:
|
||||
client = get_secret_client()
|
||||
name = f"projects/{PROJECT_ID}/secrets/{RUNNER_PAT_SECRET_NAME}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
_github_pat = response.payload.data.decode("utf-8").strip()
|
||||
return _github_pat
|
||||
|
||||
|
||||
def verify_webhook_signature(request: Request) -> bool:
|
||||
"""
|
||||
Verify the GitHub webhook signature.
|
||||
|
||||
GitHub sends a signature in the X-Hub-Signature-256 header.
|
||||
We compute the expected signature and compare.
|
||||
"""
|
||||
signature_header = request.headers.get("X-Hub-Signature-256")
|
||||
if not signature_header:
|
||||
print("Missing X-Hub-Signature-256 header")
|
||||
return False
|
||||
|
||||
# Get the raw body for signature verification
|
||||
body = request.get_data()
|
||||
secret = get_webhook_secret()
|
||||
|
||||
# Compute expected signature
|
||||
expected_signature = (
|
||||
"sha256="
|
||||
+ hmac.new(
|
||||
secret.encode("utf-8"),
|
||||
body,
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
)
|
||||
|
||||
# Constant-time comparison to prevent timing attacks
|
||||
return hmac.compare_digest(expected_signature, signature_header)
|
||||
|
||||
|
||||
def get_runner_instances() -> list[compute_v1.Instance]:
|
||||
"""Get all GitHub runner instances."""
|
||||
client = get_compute_client()
|
||||
instances = []
|
||||
|
||||
for i in range(1, NUM_RUNNERS + 1):
|
||||
instance_name = f"{RUNNER_PREFIX}{i}"
|
||||
try:
|
||||
instance = client.get(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
)
|
||||
instances.append(instance)
|
||||
except Exception as e:
|
||||
print(f"Error getting instance {instance_name}: {e}")
|
||||
|
||||
return instances
|
||||
|
||||
|
||||
def start_runners() -> dict:
|
||||
"""
|
||||
Start any TERMINATED runner VMs.
|
||||
|
||||
Returns a dict with lists of started, already_running, and failed instances.
|
||||
"""
|
||||
client = get_compute_client()
|
||||
result = {"started": [], "already_running": [], "failed": []}
|
||||
|
||||
for i in range(1, NUM_RUNNERS + 1):
|
||||
instance_name = f"{RUNNER_PREFIX}{i}"
|
||||
try:
|
||||
instance = client.get(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
)
|
||||
|
||||
if instance.status == "TERMINATED":
|
||||
print(f"Starting {instance_name}...")
|
||||
operation = client.start(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
)
|
||||
# Wait for the operation to complete (timeout after 5 min)
|
||||
try:
|
||||
operation.result(timeout=300)
|
||||
print(f"{instance_name} started successfully")
|
||||
result["started"].append(instance_name)
|
||||
except Exception as op_error:
|
||||
print(f"Start operation failed for {instance_name}: {op_error}")
|
||||
result["failed"].append(instance_name)
|
||||
elif instance.status == "RUNNING":
|
||||
result["already_running"].append(instance_name)
|
||||
else:
|
||||
print(f"{instance_name} is in state {instance.status}")
|
||||
result["already_running"].append(instance_name)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error with {instance_name}: {e}")
|
||||
result["failed"].append(instance_name)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def update_instance_metadata(instance_name: str, key: str, value: str) -> None:
|
||||
"""Update a single metadata key on an instance."""
|
||||
client = get_compute_client()
|
||||
|
||||
# Get current instance to get fingerprint
|
||||
instance = client.get(project=PROJECT_ID, zone=ZONE, instance=instance_name)
|
||||
current_metadata = instance.metadata
|
||||
|
||||
# Build new items list, replacing or adding the key
|
||||
new_items = []
|
||||
found = False
|
||||
for item in current_metadata.items:
|
||||
if item.key == key:
|
||||
new_items.append(compute_v1.Items(key=key, value=value))
|
||||
found = True
|
||||
else:
|
||||
new_items.append(item)
|
||||
|
||||
if not found:
|
||||
new_items.append(compute_v1.Items(key=key, value=value))
|
||||
|
||||
# Update metadata
|
||||
metadata = compute_v1.Metadata(
|
||||
fingerprint=current_metadata.fingerprint,
|
||||
items=new_items,
|
||||
)
|
||||
|
||||
client.set_metadata(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance_name,
|
||||
metadata_resource=metadata,
|
||||
)
|
||||
|
||||
|
||||
def get_instance_metadata(instance: compute_v1.Instance, key: str) -> str | None:
|
||||
"""Get a metadata value from an instance."""
|
||||
if not instance.metadata or not instance.metadata.items:
|
||||
return None
|
||||
|
||||
for item in instance.metadata.items:
|
||||
if item.key == key:
|
||||
return item.value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def check_github_for_pending_jobs() -> bool:
|
||||
"""
|
||||
Check GitHub API for any pending/queued jobs for our runners.
|
||||
|
||||
Returns True if there are jobs waiting for our self-hosted runners.
|
||||
"""
|
||||
import urllib.request
|
||||
|
||||
pat = get_github_pat()
|
||||
org = "nomadkaraoke"
|
||||
|
||||
# Check for queued workflow runs in karaoke-gen repo
|
||||
url = f"https://api.github.com/repos/{org}/karaoke-gen/actions/runs?status=queued"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {pat}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
if data.get("total_count", 0) > 0:
|
||||
print(f"Found {data['total_count']} queued runs in karaoke-gen")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error checking GitHub API: {e}")
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def check_and_stop_idle_runners() -> dict:
|
||||
"""
|
||||
Check for idle runners and stop them if idle for IDLE_TIMEOUT_HOURS.
|
||||
|
||||
A runner is considered idle if:
|
||||
1. It's RUNNING
|
||||
2. No jobs are pending/queued on GitHub
|
||||
3. last-activity metadata is older than IDLE_TIMEOUT_HOURS
|
||||
|
||||
Returns a dict with stopped and kept instances.
|
||||
"""
|
||||
result = {"stopped": [], "kept": [], "errors": []}
|
||||
|
||||
# First check if there are any pending jobs
|
||||
if check_github_for_pending_jobs():
|
||||
print("Jobs are pending - keeping all runners active")
|
||||
# Update last-activity on all running instances
|
||||
for instance in get_runner_instances():
|
||||
if instance.status == "RUNNING":
|
||||
result["kept"].append(instance.name)
|
||||
# Refresh activity timestamp since there are pending jobs
|
||||
try:
|
||||
update_instance_metadata(
|
||||
instance.name,
|
||||
"last-activity",
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error updating metadata for {instance.name}: {e}")
|
||||
return result
|
||||
|
||||
# No pending jobs - check each instance for idle timeout
|
||||
client = get_compute_client()
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for instance in get_runner_instances():
|
||||
if instance.status != "RUNNING":
|
||||
continue
|
||||
|
||||
# Get last-activity timestamp
|
||||
last_activity_str = get_instance_metadata(instance, "last-activity")
|
||||
|
||||
if not last_activity_str:
|
||||
# No activity recorded - set it now and don't stop
|
||||
print(f"{instance.name}: No last-activity metadata, setting now")
|
||||
try:
|
||||
update_instance_metadata(
|
||||
instance.name,
|
||||
"last-activity",
|
||||
now.isoformat(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error setting metadata for {instance.name}: {e}")
|
||||
result["kept"].append(instance.name)
|
||||
continue
|
||||
|
||||
try:
|
||||
last_activity = datetime.fromisoformat(last_activity_str.replace("Z", "+00:00"))
|
||||
idle_hours = (now - last_activity).total_seconds() / 3600
|
||||
|
||||
if idle_hours >= IDLE_TIMEOUT_HOURS:
|
||||
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, stopping...")
|
||||
client.stop(
|
||||
project=PROJECT_ID,
|
||||
zone=ZONE,
|
||||
instance=instance.name,
|
||||
)
|
||||
result["stopped"].append(instance.name)
|
||||
else:
|
||||
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, keeping...")
|
||||
result["kept"].append(instance.name)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing {instance.name}: {e}")
|
||||
result["errors"].append(instance.name)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@functions_framework.http
|
||||
def handle_request(request: Request):
|
||||
"""
|
||||
Main entry point for the Cloud Function.
|
||||
|
||||
Handles:
|
||||
1. Cloud Scheduler triggers (action=check_idle parameter)
|
||||
2. GitHub webhook events (workflow_job.queued)
|
||||
"""
|
||||
# Check if this is a scheduler trigger (idle check)
|
||||
action = request.args.get("action")
|
||||
if action == "check_idle":
|
||||
print("Scheduler trigger: checking for idle runners")
|
||||
result = check_and_stop_idle_runners()
|
||||
return json.dumps(result), 200, {"Content-Type": "application/json"}
|
||||
|
||||
# Otherwise, this should be a GitHub webhook
|
||||
# Verify the webhook signature
|
||||
if not verify_webhook_signature(request):
|
||||
print("Webhook signature verification failed")
|
||||
return "Unauthorized", 401
|
||||
|
||||
# Get the event type
|
||||
event = request.headers.get("X-GitHub-Event")
|
||||
if not event:
|
||||
return "Missing X-GitHub-Event header", 400
|
||||
|
||||
# We only care about workflow_job events
|
||||
if event != "workflow_job":
|
||||
return "OK - event ignored", 200
|
||||
|
||||
# Parse the payload
|
||||
try:
|
||||
payload = request.get_json()
|
||||
except Exception as e:
|
||||
print(f"Error parsing JSON: {e}")
|
||||
return "Invalid JSON", 400
|
||||
|
||||
action = payload.get("action")
|
||||
job = payload.get("workflow_job", {})
|
||||
labels = job.get("labels", [])
|
||||
|
||||
print(f"Received workflow_job.{action} event")
|
||||
print(f"Job labels: {labels}")
|
||||
|
||||
# Check if this job requires our self-hosted runners
|
||||
if "self-hosted" not in labels:
|
||||
print("Job doesn't require self-hosted runners, ignoring")
|
||||
return "OK - not for self-hosted runners", 200
|
||||
|
||||
# Handle different actions
|
||||
if action == "queued":
|
||||
# Job is queued - ensure runners are started
|
||||
print("Job queued - starting runners if needed")
|
||||
result = start_runners()
|
||||
return json.dumps(result), 200, {"Content-Type": "application/json"}
|
||||
|
||||
elif action == "in_progress":
|
||||
# Job started - update activity timestamp on all running instances
|
||||
print("Job in progress - updating activity timestamps")
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
for instance in get_runner_instances():
|
||||
if instance.status == "RUNNING":
|
||||
try:
|
||||
update_instance_metadata(instance.name, "last-activity", now)
|
||||
except Exception as e:
|
||||
print(f"Error updating {instance.name}: {e}")
|
||||
return "OK", 200
|
||||
|
||||
elif action == "completed":
|
||||
# Job completed - update activity timestamp
|
||||
print("Job completed - updating activity timestamps")
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
for instance in get_runner_instances():
|
||||
if instance.status == "RUNNING":
|
||||
try:
|
||||
update_instance_metadata(instance.name, "last-activity", now)
|
||||
except Exception as e:
|
||||
print(f"Error updating {instance.name}: {e}")
|
||||
return "OK", 200
|
||||
|
||||
return "OK", 200
|
||||
@@ -0,0 +1,4 @@
|
||||
functions-framework==3.*
|
||||
google-cloud-compute==1.*
|
||||
google-cloud-secret-manager==2.*
|
||||
flask>=2.0.0
|
||||
@@ -13,6 +13,7 @@ from . import cloud_run
|
||||
from . import monitoring
|
||||
from . import iam
|
||||
from . import networking
|
||||
from . import runner_manager
|
||||
|
||||
__all__ = [
|
||||
"database",
|
||||
@@ -24,4 +25,5 @@ __all__ = [
|
||||
"monitoring",
|
||||
"iam",
|
||||
"networking",
|
||||
"runner_manager",
|
||||
]
|
||||
@@ -0,0 +1,297 @@
|
||||
"""
|
||||
GitHub Actions Runner Manager - Auto-start and auto-stop self-hosted runners.
|
||||
|
||||
Creates infrastructure to automatically manage runner lifecycle:
|
||||
1. Cloud Function (Gen2) - Handles GitHub webhooks and idle checks
|
||||
2. Cloud Scheduler - Triggers idle checks every 15 minutes
|
||||
3. IAM bindings - Permissions to start/stop Compute Engine instances
|
||||
|
||||
Architecture:
|
||||
- GitHub sends workflow_job webhooks to the Cloud Function
|
||||
- When a job is queued, the function starts any TERMINATED runner VMs
|
||||
- Cloud Scheduler triggers idle checks every 15 minutes
|
||||
- If no jobs are pending and runners have been idle, they're stopped
|
||||
|
||||
Manual Setup Required:
|
||||
1. Create a webhook secret:
|
||||
python -c "import secrets; print(secrets.token_hex(32))" | \
|
||||
gcloud secrets versions add github-webhook-secret --data-file=-
|
||||
|
||||
2. Configure GitHub org webhook at:
|
||||
https://github.com/organizations/nomadkaraoke/settings/hooks
|
||||
- Payload URL: <Cloud Function URL from pulumi output>
|
||||
- Content type: application/json
|
||||
- Secret: <value from step 1>
|
||||
- Events: Workflow jobs
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pulumi
|
||||
import pulumi_gcp as gcp
|
||||
from config import (
|
||||
PROJECT_ID,
|
||||
REGION,
|
||||
ZONE,
|
||||
RunnerManagerConfig,
|
||||
SecretNames,
|
||||
)
|
||||
|
||||
|
||||
def create_runner_manager_service_account() -> gcp.serviceaccount.Account:
|
||||
"""Create service account for the runner manager Cloud Function."""
|
||||
return gcp.serviceaccount.Account(
|
||||
"runner-manager-sa",
|
||||
account_id="runner-manager",
|
||||
display_name="GitHub Runner Manager",
|
||||
description="Service account for the GitHub Actions runner manager Cloud Function",
|
||||
)
|
||||
|
||||
|
||||
def grant_runner_manager_permissions(
|
||||
service_account: gcp.serviceaccount.Account,
|
||||
webhook_secret: gcp.secretmanager.Secret,
|
||||
pat_secret: gcp.secretmanager.Secret,
|
||||
) -> dict:
|
||||
"""Grant permissions to the runner manager service account."""
|
||||
bindings = {}
|
||||
|
||||
# Grant Compute Instance Admin - needed to start/stop VMs
|
||||
bindings["compute_admin"] = gcp.projects.IAMMember(
|
||||
"runner-manager-compute-admin",
|
||||
project=PROJECT_ID,
|
||||
role="roles/compute.instanceAdmin.v1",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Secret Manager accessor for webhook secret
|
||||
bindings["webhook_secret_accessor"] = gcp.secretmanager.SecretIamMember(
|
||||
"runner-manager-webhook-secret-access",
|
||||
secret_id=webhook_secret.id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Secret Manager accessor for GitHub PAT (for checking pending jobs)
|
||||
bindings["pat_secret_accessor"] = gcp.secretmanager.SecretIamMember(
|
||||
"runner-manager-pat-secret-access",
|
||||
secret_id=pat_secret.id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant logging writer
|
||||
bindings["logging_writer"] = gcp.projects.IAMMember(
|
||||
"runner-manager-logging-writer",
|
||||
project=PROJECT_ID,
|
||||
role="roles/logging.logWriter",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
return bindings
|
||||
|
||||
|
||||
def create_function_storage_bucket() -> gcp.storage.Bucket:
|
||||
"""Create a GCS bucket for Cloud Function source code."""
|
||||
return gcp.storage.Bucket(
|
||||
"runner-manager-source-bucket",
|
||||
name=f"{PROJECT_ID}-runner-manager-source",
|
||||
location=REGION,
|
||||
uniform_bucket_level_access=True,
|
||||
lifecycle_rules=[
|
||||
gcp.storage.BucketLifecycleRuleArgs(
|
||||
action=gcp.storage.BucketLifecycleRuleActionArgs(type="Delete"),
|
||||
condition=gcp.storage.BucketLifecycleRuleConditionArgs(age=7),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.BucketObject:
|
||||
"""Create a GCS object with the Cloud Function source code."""
|
||||
source_dir = Path(__file__).parent.parent / "functions" / "runner_manager"
|
||||
|
||||
return gcp.storage.BucketObject(
|
||||
"runner-manager-source",
|
||||
bucket=bucket.name,
|
||||
name="runner-manager-source.zip",
|
||||
source=pulumi.FileArchive(str(source_dir)),
|
||||
)
|
||||
|
||||
|
||||
def create_cloud_function(
|
||||
service_account: gcp.serviceaccount.Account,
|
||||
bucket: gcp.storage.Bucket,
|
||||
source_archive: gcp.storage.BucketObject,
|
||||
permissions: dict,
|
||||
) -> gcp.cloudfunctionsv2.Function:
|
||||
"""Create the Cloud Function (Gen2) for runner management."""
|
||||
return gcp.cloudfunctionsv2.Function(
|
||||
"runner-manager-function",
|
||||
name=RunnerManagerConfig.FUNCTION_NAME,
|
||||
location=REGION,
|
||||
description="Manages GitHub Actions runner VM lifecycle (auto-start/stop)",
|
||||
build_config=gcp.cloudfunctionsv2.FunctionBuildConfigArgs(
|
||||
runtime="python312",
|
||||
entry_point="handle_request",
|
||||
source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceArgs(
|
||||
storage_source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceStorageSourceArgs(
|
||||
bucket=bucket.name,
|
||||
object=source_archive.name,
|
||||
),
|
||||
),
|
||||
),
|
||||
service_config=gcp.cloudfunctionsv2.FunctionServiceConfigArgs(
|
||||
available_memory=RunnerManagerConfig.FUNCTION_MEMORY,
|
||||
timeout_seconds=RunnerManagerConfig.FUNCTION_TIMEOUT,
|
||||
service_account_email=service_account.email,
|
||||
environment_variables={
|
||||
"GCP_PROJECT": PROJECT_ID,
|
||||
"GCP_ZONE": ZONE,
|
||||
"WEBHOOK_SECRET_NAME": SecretNames.GITHUB_WEBHOOK_SECRET,
|
||||
"RUNNER_PAT_SECRET_NAME": SecretNames.GITHUB_RUNNER_PAT,
|
||||
"IDLE_TIMEOUT_HOURS": str(RunnerManagerConfig.IDLE_TIMEOUT_HOURS),
|
||||
},
|
||||
min_instance_count=0, # Scale to zero when not in use
|
||||
max_instance_count=5, # Allow some concurrency for multiple webhooks
|
||||
),
|
||||
opts=pulumi.ResourceOptions(
|
||||
depends_on=list(permissions.values()),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def allow_unauthenticated_invocation(
|
||||
function: gcp.cloudfunctionsv2.Function,
|
||||
) -> gcp.cloudfunctionsv2.FunctionIamMember:
|
||||
"""
|
||||
Allow unauthenticated invocations of the Cloud Function.
|
||||
|
||||
Required for GitHub webhooks since GitHub doesn't support GCP auth.
|
||||
Security is provided by webhook signature verification.
|
||||
"""
|
||||
return gcp.cloudfunctionsv2.FunctionIamMember(
|
||||
"runner-manager-public-invoker",
|
||||
project=PROJECT_ID,
|
||||
location=REGION,
|
||||
cloud_function=function.name,
|
||||
role="roles/cloudfunctions.invoker",
|
||||
member="allUsers",
|
||||
)
|
||||
|
||||
|
||||
def create_scheduler_service_account() -> gcp.serviceaccount.Account:
|
||||
"""Create service account for Cloud Scheduler to invoke the function."""
|
||||
return gcp.serviceaccount.Account(
|
||||
"runner-manager-scheduler-sa",
|
||||
account_id="runner-manager-scheduler",
|
||||
display_name="Runner Manager Scheduler",
|
||||
description="Service account for Cloud Scheduler to trigger runner idle checks",
|
||||
)
|
||||
|
||||
|
||||
def grant_scheduler_invoker_permission(
|
||||
scheduler_sa: gcp.serviceaccount.Account,
|
||||
function: gcp.cloudfunctionsv2.Function,
|
||||
) -> gcp.cloudfunctionsv2.FunctionIamMember:
|
||||
"""Grant the scheduler service account permission to invoke the function."""
|
||||
return gcp.cloudfunctionsv2.FunctionIamMember(
|
||||
"runner-manager-scheduler-invoker",
|
||||
project=PROJECT_ID,
|
||||
location=REGION,
|
||||
cloud_function=function.name,
|
||||
role="roles/cloudfunctions.invoker",
|
||||
member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
|
||||
def create_idle_check_scheduler(
|
||||
scheduler_sa: gcp.serviceaccount.Account,
|
||||
function: gcp.cloudfunctionsv2.Function,
|
||||
) -> gcp.cloudscheduler.Job:
|
||||
"""Create Cloud Scheduler job for idle runner checks."""
|
||||
return gcp.cloudscheduler.Job(
|
||||
"runner-manager-idle-check",
|
||||
name="runner-manager-idle-check",
|
||||
description="Triggers idle check for GitHub Actions runners every 15 minutes",
|
||||
schedule=RunnerManagerConfig.IDLE_CHECK_SCHEDULE,
|
||||
time_zone="UTC",
|
||||
region=REGION,
|
||||
http_target=gcp.cloudscheduler.JobHttpTargetArgs(
|
||||
uri=function.url.apply(lambda url: f"{url}?action=check_idle"),
|
||||
http_method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body=base64.b64encode(json.dumps({"trigger": "scheduler"}).encode()).decode(),
|
||||
oidc_token=gcp.cloudscheduler.JobHttpTargetOidcTokenArgs(
|
||||
service_account_email=scheduler_sa.email,
|
||||
audience=function.url,
|
||||
),
|
||||
),
|
||||
retry_config=gcp.cloudscheduler.JobRetryConfigArgs(
|
||||
retry_count=2,
|
||||
max_retry_duration="60s",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_runner_manager_resources(
|
||||
webhook_secret: gcp.secretmanager.Secret,
|
||||
pat_secret: gcp.secretmanager.Secret,
|
||||
) -> dict:
|
||||
"""
|
||||
Create all resources for the GitHub Actions runner manager.
|
||||
|
||||
Args:
|
||||
webhook_secret: The webhook secret for signature verification.
|
||||
pat_secret: The GitHub PAT secret for API calls.
|
||||
|
||||
Returns:
|
||||
dict: Dictionary of all created resources.
|
||||
"""
|
||||
# Create service accounts
|
||||
function_sa = create_runner_manager_service_account()
|
||||
scheduler_sa = create_scheduler_service_account()
|
||||
|
||||
# Grant permissions to function service account
|
||||
permissions = grant_runner_manager_permissions(
|
||||
function_sa,
|
||||
webhook_secret,
|
||||
pat_secret,
|
||||
)
|
||||
|
||||
# Create source storage bucket and archive
|
||||
bucket = create_function_storage_bucket()
|
||||
source_archive = create_function_source_archive(bucket)
|
||||
|
||||
# Create the Cloud Function
|
||||
function = create_cloud_function(
|
||||
function_sa,
|
||||
bucket,
|
||||
source_archive,
|
||||
permissions,
|
||||
)
|
||||
|
||||
# Allow unauthenticated invocation (GitHub webhooks)
|
||||
public_access = allow_unauthenticated_invocation(function)
|
||||
|
||||
# Grant scheduler permission to invoke function
|
||||
scheduler_invoker = grant_scheduler_invoker_permission(scheduler_sa, function)
|
||||
|
||||
# Create the idle check scheduler job
|
||||
scheduler_job = create_idle_check_scheduler(scheduler_sa, function)
|
||||
|
||||
return {
|
||||
"function_service_account": function_sa,
|
||||
"scheduler_service_account": scheduler_sa,
|
||||
"permissions": permissions,
|
||||
"source_bucket": bucket,
|
||||
"source_archive": source_archive,
|
||||
"function": function,
|
||||
"public_access": public_access,
|
||||
"scheduler_invoker": scheduler_invoker,
|
||||
"scheduler_job": scheduler_job,
|
||||
}
|
||||
@@ -47,6 +47,7 @@ def create_secrets() -> dict[str, secretmanager.Secret]:
|
||||
|
||||
# GitHub
|
||||
"github-runner-pat",
|
||||
"github-webhook-secret", # For runner manager webhook verification
|
||||
|
||||
# E2E Testing
|
||||
"e2e-bypass-key",
|
||||
|
||||
Reference in new issue
Block a user