Compare commits

...
Author SHA1 Message Date
Andrew BeveridgeandClaude Opus 4.5 93c6254ab6 feat(infra): Add auto-scaling GitHub runners with Spot VMs
Implement cost optimization for GitHub Actions self-hosted runners:

- Reduce runners from 20 to 3 (auto-scaling handles demand)
- Switch to Spot/Preemptible instances (60-91% cheaper)
- Add Cloud NAT for outbound internet (no external IPs needed)
- Add runner_manager Cloud Function for auto-start/stop:
  - Starts VMs when workflow_job.queued webhook received
  - Stops idle VMs after 1 hour of no activity
  - Cloud Scheduler triggers idle check every 15 minutes

Estimated savings: ~$1,300-1,400/month

Manual setup required after deploy:
1. Create github-webhook-secret in Secret Manager
2. Configure GitHub org webhook with function URL

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-05 09:20:37 -08:00
9 changed files with 884 additions and 19 deletions

No files matched your search

+17 -2
View File
@@ -37,7 +37,7 @@ from config import (
DiskSizes,
)
from modules import database, storage as storage_module, artifact_registry, secrets
from modules import cloud_tasks, cloud_run, monitoring, networking
from modules import cloud_tasks, cloud_run, monitoring, networking, runner_manager
from modules.iam import backend_sa, github_actions_sa, claude_automation_sa, worker_sas
from compute import encoding_worker_vm, github_runners
@@ -249,8 +249,18 @@ encoding_worker_instance = encoding_worker_vm.create_encoding_worker_vm(
encoding_worker_firewall = encoding_worker_vm.create_encoding_worker_firewall()
# GitHub Runners (CI/CD self-hosted runners)
# Create Cloud NAT for outbound internet access (no external IPs needed)
github_runners_router, github_runners_nat = github_runners.create_cloud_nat()
# Create runner VMs (Spot instances for 60-91% cost savings)
github_runner_vms = github_runners.create_github_runners(
github_runner_sa, all_secrets["github-runner-pat"]
github_runner_sa, all_secrets["github-runner-pat"], github_runners_nat
)
# Create runner manager (auto-start/stop VMs based on CI activity)
runner_manager_resources = runner_manager.create_runner_manager_resources(
all_secrets["github-webhook-secret"],
all_secrets["github-runner-pat"],
)
@@ -334,3 +344,8 @@ pulumi.export("encoding_worker_vm_name", encoding_worker_instance.name)
# GitHub runners
pulumi.export("github_runner_vm_names", [vm.name for vm in github_runner_vms])
pulumi.export("github_runner_service_account", github_runner_sa.email)
pulumi.export("github_runners_nat", github_runners_nat.name)
# GitHub runner manager (auto-start/stop)
pulumi.export("runner_manager_function_url", runner_manager_resources["function"].url)
pulumi.export("runner_manager_scheduler_job", runner_manager_resources["scheduler_job"].name)
+2 -1
View File
@@ -11,11 +11,12 @@ from .encoding_worker_vm import (
create_encoding_worker_ip,
create_encoding_worker_firewall,
)
from .github_runners import create_github_runners
from .github_runners import create_github_runners, create_cloud_nat
__all__ = [
"create_encoding_worker_vm",
"create_encoding_worker_ip",
"create_encoding_worker_firewall",
"create_github_runners",
"create_cloud_nat",
]
+125 -15
View File
@@ -1,30 +1,82 @@
"""
GitHub Actions self-hosted runner VMs.
GitHub Actions Self-Hosted Runners on GCP.
Manages multiple VM instances that serve as self-hosted runners
for GitHub Actions CI/CD workflows.
Creates Spot/Preemptible VMs that register as self-hosted runners
at the organization level. Uses Spot instances for 60-91% cost savings.
Resources created:
- Cloud Router and Cloud NAT (for outbound internet without external IPs)
- N Compute Engine VMs (Spot instances)
- Each VM runs the GitHub Actions runner agent
The VMs are managed by the runner_manager Cloud Function which:
- Starts VMs when CI jobs are queued (via GitHub webhook)
- Stops VMs when idle for too long (via Cloud Scheduler)
"""
import pulumi
from pulumi_gcp import compute, serviceaccount, secretmanager
from pulumi_gcp import compute, secretmanager, serviceaccount
from config import ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
from config import REGION, ZONE, MachineTypes, DiskSizes, NUM_GITHUB_RUNNERS
from .startup_scripts import read_script
def create_cloud_nat() -> tuple[compute.Router, compute.RouterNat]:
"""
Create Cloud Router and Cloud NAT for outbound internet access.
This allows VMs without external IPs to access the internet,
which saves costs and improves security.
Returns:
tuple: (router, nat) resources
"""
# Create Cloud Router
router = compute.Router(
"github-runners-router",
name="github-runners-router",
region=REGION,
network="default",
description="Router for GitHub Actions runner NAT",
)
# Create Cloud NAT
nat = compute.RouterNat(
"github-runners-nat",
name="github-runners-nat",
router=router.name,
region=REGION,
nat_ip_allocate_option="AUTO_ONLY",
source_subnetwork_ip_ranges_to_nat="ALL_SUBNETWORKS_ALL_IP_RANGES",
log_config=compute.RouterNatLogConfigArgs(
enable=True,
filter="ERRORS_ONLY",
),
opts=pulumi.ResourceOptions(depends_on=[router]),
)
return router, nat
def create_github_runners(
service_account: serviceaccount.Account,
runner_pat_secret: secretmanager.Secret,
nat: compute.RouterNat,
) -> list[compute.Instance]:
"""
Create GitHub Actions self-hosted runner VM instances.
Creates NUM_GITHUB_RUNNERS VMs, each configured as a GitHub Actions
Creates NUM_GITHUB_RUNNERS Spot VMs, each configured as a GitHub Actions
self-hosted runner for the nomadkaraoke organization (available to all repos).
Uses Spot instances (preemptible) for 60-91% cost savings.
VMs start in TERMINATED state and are started by the runner_manager
Cloud Function when CI jobs are queued.
Args:
service_account: The GitHub runner service account.
runner_pat_secret: The Secret Manager secret containing the GitHub PAT.
nat: Cloud NAT for outbound internet access (VMs have no external IPs).
Returns:
list[compute.Instance]: List of runner VM instances.
@@ -33,33 +85,91 @@ def create_github_runners(
runner_vms = []
for i in range(1, NUM_GITHUB_RUNNERS + 1):
instance_name = f"github-runner-{i}"
vm = compute.Instance(
f"github-runner-{i}",
name=f"github-runner-{i}",
instance_name,
name=instance_name,
machine_type=MachineTypes.GITHUB_RUNNER,
zone=ZONE,
# Boot disk configuration
boot_disk=compute.InstanceBootDiskArgs(
initialize_params=compute.InstanceBootDiskInitializeParamsArgs(
image="debian-cloud/debian-12",
size=DiskSizes.GITHUB_RUNNER,
type="pd-ssd",
type="pd-ssd", # SSD for faster Docker builds
),
auto_delete=True,
),
network_interfaces=[compute.InstanceNetworkInterfaceArgs(
network="default",
access_configs=[compute.InstanceNetworkInterfaceAccessConfigArgs()],
)],
# Network configuration - no external IP (uses Cloud NAT)
# This saves ~$3/month per VM and improves security
network_interfaces=[
compute.InstanceNetworkInterfaceArgs(
network="default",
# No access_configs = no external IP
# Outbound internet via Cloud NAT
)
],
# Service account for GCP API access
service_account=compute.InstanceServiceAccountArgs(
email=service_account.email,
scopes=["cloud-platform"],
scopes=["https://www.googleapis.com/auth/cloud-platform"],
),
# Spot/Preemptible instance configuration (60-91% cheaper)
# Preemptible VMs may be terminated at any time but cost much less.
# The startup script re-registers the runner on boot.
scheduling=compute.InstanceSchedulingArgs(
preemptible=True,
automatic_restart=False, # Required for preemptible
on_host_maintenance="TERMINATE", # Required for preemptible
),
# Metadata for startup script
metadata={
"github-runner-pat-secret": "github-runner-pat",
"github-org": "nomadkaraoke",
"runner-labels": "self-hosted,linux,x64,gcp,large-disk",
},
metadata_startup_script=startup_script,
# Labels for organization and cost tracking
labels={
"purpose": "github-runner",
"managed-by": "pulumi",
},
tags=["github-runner"],
# Allow stopping for updates
allow_stopping_for_update=True,
# Deletion protection disabled (runners are ephemeral)
deletion_protection=False,
opts=pulumi.ResourceOptions(
depends_on=[runner_pat_secret],
depends_on=[runner_pat_secret, nat],
# Runners can be replaced without affecting other infrastructure
delete_before_replace=True,
),
)
runner_vms.append(vm)
return runner_vms
def create_instance_group_for_restart(
instances: list[compute.Instance],
) -> compute.InstanceGroup:
"""
Create an unmanaged instance group for the runner VMs.
This enables bulk operations like restarting all runners.
Args:
instances: List of runner VM instances.
Returns:
compute.InstanceGroup: The instance group.
"""
return compute.InstanceGroup(
"github-runners-group",
name="github-runners",
description="Self-hosted GitHub Actions runners",
zone=ZONE,
instances=[instance.self_link for instance in instances],
)
+19 -1
View File
@@ -87,11 +87,29 @@ class QueueConfigs:
# Number of self-hosted GitHub Action runners
NUM_GITHUB_RUNNERS = 20
# Reduced from 20 to 3 - runners auto-scale with start/stop management
NUM_GITHUB_RUNNERS = 3
# Google Drive folder ID for validator (public share folder)
GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
class RunnerManagerConfig:
"""Configuration for GitHub runner auto-start/stop management."""
FUNCTION_NAME = "github-runner-manager"
FUNCTION_MEMORY = "256M"
FUNCTION_TIMEOUT = 300 # 5 minutes
IDLE_TIMEOUT_HOURS = 1 # Shorter than Aquarius (3h) since we have more CI activity
IDLE_CHECK_SCHEDULE = "*/15 * * * *" # Every 15 minutes
class SecretNames:
"""Secret Manager secret names."""
GITHUB_RUNNER_PAT = "github-runner-pat"
GITHUB_WEBHOOK_SECRET = "github-webhook-secret"
# GitHub repository for runner registration
GITHUB_REPO_OWNER = "nomadkaraoke"
GITHUB_REPO_NAME = "karaoke-gen"
@@ -0,0 +1,417 @@
"""
GitHub Actions Runner Manager Cloud Function.
Handles two types of triggers:
1. GitHub Webhook (workflow_job.queued) - Starts runner VMs when CI jobs are queued
2. Cloud Scheduler (every 15 min) - Stops idle runner VMs after timeout
Environment variables:
- GCP_PROJECT: GCP project ID
- GCP_ZONE: Zone where runner VMs are located
- WEBHOOK_SECRET_NAME: Secret Manager secret name for webhook verification
- RUNNER_PAT_SECRET_NAME: Secret Manager secret name for GitHub PAT
- IDLE_TIMEOUT_HOURS: Hours of inactivity before stopping runners (default: 1)
"""
import hashlib
import hmac
import json
import os
from datetime import datetime, timezone
import functions_framework
from flask import Request
from google.cloud import compute_v1, secretmanager
# Configuration from environment
PROJECT_ID = os.environ.get("GCP_PROJECT", "nomadkaraoke")
ZONE = os.environ.get("GCP_ZONE", "us-central1-a")
WEBHOOK_SECRET_NAME = os.environ.get("WEBHOOK_SECRET_NAME", "github-webhook-secret")
RUNNER_PAT_SECRET_NAME = os.environ.get("RUNNER_PAT_SECRET_NAME", "github-runner-pat")
IDLE_TIMEOUT_HOURS = int(os.environ.get("IDLE_TIMEOUT_HOURS", "1"))
RUNNER_PREFIX = "github-runner-"
NUM_RUNNERS = 3
# Lazy-loaded clients and secrets
_compute_client = None
_secret_client = None
_webhook_secret = None
_github_pat = None
def get_compute_client() -> compute_v1.InstancesClient:
"""Get or create the Compute Engine client."""
global _compute_client
if _compute_client is None:
_compute_client = compute_v1.InstancesClient()
return _compute_client
def get_secret_client() -> secretmanager.SecretManagerServiceClient:
"""Get or create the Secret Manager client."""
global _secret_client
if _secret_client is None:
_secret_client = secretmanager.SecretManagerServiceClient()
return _secret_client
def get_webhook_secret() -> str:
"""Get the webhook secret from Secret Manager (cached)."""
global _webhook_secret
if _webhook_secret is None:
client = get_secret_client()
name = f"projects/{PROJECT_ID}/secrets/{WEBHOOK_SECRET_NAME}/versions/latest"
response = client.access_secret_version(request={"name": name})
_webhook_secret = response.payload.data.decode("utf-8").strip()
return _webhook_secret
def get_github_pat() -> str:
"""Get the GitHub PAT from Secret Manager (cached)."""
global _github_pat
if _github_pat is None:
client = get_secret_client()
name = f"projects/{PROJECT_ID}/secrets/{RUNNER_PAT_SECRET_NAME}/versions/latest"
response = client.access_secret_version(request={"name": name})
_github_pat = response.payload.data.decode("utf-8").strip()
return _github_pat
def verify_webhook_signature(request: Request) -> bool:
"""
Verify the GitHub webhook signature.
GitHub sends a signature in the X-Hub-Signature-256 header.
We compute the expected signature and compare.
"""
signature_header = request.headers.get("X-Hub-Signature-256")
if not signature_header:
print("Missing X-Hub-Signature-256 header")
return False
# Get the raw body for signature verification
body = request.get_data()
secret = get_webhook_secret()
# Compute expected signature
expected_signature = (
"sha256="
+ hmac.new(
secret.encode("utf-8"),
body,
hashlib.sha256,
).hexdigest()
)
# Constant-time comparison to prevent timing attacks
return hmac.compare_digest(expected_signature, signature_header)
def get_runner_instances() -> list[compute_v1.Instance]:
"""Get all GitHub runner instances."""
client = get_compute_client()
instances = []
for i in range(1, NUM_RUNNERS + 1):
instance_name = f"{RUNNER_PREFIX}{i}"
try:
instance = client.get(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
)
instances.append(instance)
except Exception as e:
print(f"Error getting instance {instance_name}: {e}")
return instances
def start_runners() -> dict:
"""
Start any TERMINATED runner VMs.
Returns a dict with lists of started, already_running, and failed instances.
"""
client = get_compute_client()
result = {"started": [], "already_running": [], "failed": []}
for i in range(1, NUM_RUNNERS + 1):
instance_name = f"{RUNNER_PREFIX}{i}"
try:
instance = client.get(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
)
if instance.status == "TERMINATED":
print(f"Starting {instance_name}...")
operation = client.start(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
)
# Wait for the operation to complete (timeout after 5 min)
try:
operation.result(timeout=300)
print(f"{instance_name} started successfully")
result["started"].append(instance_name)
except Exception as op_error:
print(f"Start operation failed for {instance_name}: {op_error}")
result["failed"].append(instance_name)
elif instance.status == "RUNNING":
result["already_running"].append(instance_name)
else:
print(f"{instance_name} is in state {instance.status}")
result["already_running"].append(instance_name)
except Exception as e:
print(f"Error with {instance_name}: {e}")
result["failed"].append(instance_name)
return result
def update_instance_metadata(instance_name: str, key: str, value: str) -> None:
"""Update a single metadata key on an instance."""
client = get_compute_client()
# Get current instance to get fingerprint
instance = client.get(project=PROJECT_ID, zone=ZONE, instance=instance_name)
current_metadata = instance.metadata
# Build new items list, replacing or adding the key
new_items = []
found = False
for item in current_metadata.items:
if item.key == key:
new_items.append(compute_v1.Items(key=key, value=value))
found = True
else:
new_items.append(item)
if not found:
new_items.append(compute_v1.Items(key=key, value=value))
# Update metadata
metadata = compute_v1.Metadata(
fingerprint=current_metadata.fingerprint,
items=new_items,
)
client.set_metadata(
project=PROJECT_ID,
zone=ZONE,
instance=instance_name,
metadata_resource=metadata,
)
def get_instance_metadata(instance: compute_v1.Instance, key: str) -> str | None:
"""Get a metadata value from an instance."""
if not instance.metadata or not instance.metadata.items:
return None
for item in instance.metadata.items:
if item.key == key:
return item.value
return None
def check_github_for_pending_jobs() -> bool:
"""
Check GitHub API for any pending/queued jobs for our runners.
Returns True if there are jobs waiting for our self-hosted runners.
"""
import urllib.request
pat = get_github_pat()
org = "nomadkaraoke"
# Check for queued workflow runs in karaoke-gen repo
url = f"https://api.github.com/repos/{org}/karaoke-gen/actions/runs?status=queued"
req = urllib.request.Request(
url,
headers={
"Authorization": f"Bearer {pat}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
},
)
try:
with urllib.request.urlopen(req, timeout=10) as response:
data = json.loads(response.read().decode("utf-8"))
if data.get("total_count", 0) > 0:
print(f"Found {data['total_count']} queued runs in karaoke-gen")
return True
except Exception as e:
print(f"Error checking GitHub API: {e}")
return False
def check_and_stop_idle_runners() -> dict:
"""
Check for idle runners and stop them if idle for IDLE_TIMEOUT_HOURS.
A runner is considered idle if:
1. It's RUNNING
2. No jobs are pending/queued on GitHub
3. last-activity metadata is older than IDLE_TIMEOUT_HOURS
Returns a dict with stopped and kept instances.
"""
result = {"stopped": [], "kept": [], "errors": []}
# First check if there are any pending jobs
if check_github_for_pending_jobs():
print("Jobs are pending - keeping all runners active")
# Update last-activity on all running instances
for instance in get_runner_instances():
if instance.status == "RUNNING":
result["kept"].append(instance.name)
# Refresh activity timestamp since there are pending jobs
try:
update_instance_metadata(
instance.name,
"last-activity",
datetime.now(timezone.utc).isoformat(),
)
except Exception as e:
print(f"Error updating metadata for {instance.name}: {e}")
return result
# No pending jobs - check each instance for idle timeout
client = get_compute_client()
now = datetime.now(timezone.utc)
for instance in get_runner_instances():
if instance.status != "RUNNING":
continue
# Get last-activity timestamp
last_activity_str = get_instance_metadata(instance, "last-activity")
if not last_activity_str:
# No activity recorded - set it now and don't stop
print(f"{instance.name}: No last-activity metadata, setting now")
try:
update_instance_metadata(
instance.name,
"last-activity",
now.isoformat(),
)
except Exception as e:
print(f"Error setting metadata for {instance.name}: {e}")
result["kept"].append(instance.name)
continue
try:
last_activity = datetime.fromisoformat(last_activity_str.replace("Z", "+00:00"))
idle_hours = (now - last_activity).total_seconds() / 3600
if idle_hours >= IDLE_TIMEOUT_HOURS:
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, stopping...")
client.stop(
project=PROJECT_ID,
zone=ZONE,
instance=instance.name,
)
result["stopped"].append(instance.name)
else:
print(f"{instance.name}: Idle for {idle_hours:.1f} hours, keeping...")
result["kept"].append(instance.name)
except Exception as e:
print(f"Error processing {instance.name}: {e}")
result["errors"].append(instance.name)
return result
@functions_framework.http
def handle_request(request: Request):
"""
Main entry point for the Cloud Function.
Handles:
1. Cloud Scheduler triggers (action=check_idle parameter)
2. GitHub webhook events (workflow_job.queued)
"""
# Check if this is a scheduler trigger (idle check)
action = request.args.get("action")
if action == "check_idle":
print("Scheduler trigger: checking for idle runners")
result = check_and_stop_idle_runners()
return json.dumps(result), 200, {"Content-Type": "application/json"}
# Otherwise, this should be a GitHub webhook
# Verify the webhook signature
if not verify_webhook_signature(request):
print("Webhook signature verification failed")
return "Unauthorized", 401
# Get the event type
event = request.headers.get("X-GitHub-Event")
if not event:
return "Missing X-GitHub-Event header", 400
# We only care about workflow_job events
if event != "workflow_job":
return "OK - event ignored", 200
# Parse the payload
try:
payload = request.get_json()
except Exception as e:
print(f"Error parsing JSON: {e}")
return "Invalid JSON", 400
action = payload.get("action")
job = payload.get("workflow_job", {})
labels = job.get("labels", [])
print(f"Received workflow_job.{action} event")
print(f"Job labels: {labels}")
# Check if this job requires our self-hosted runners
if "self-hosted" not in labels:
print("Job doesn't require self-hosted runners, ignoring")
return "OK - not for self-hosted runners", 200
# Handle different actions
if action == "queued":
# Job is queued - ensure runners are started
print("Job queued - starting runners if needed")
result = start_runners()
return json.dumps(result), 200, {"Content-Type": "application/json"}
elif action == "in_progress":
# Job started - update activity timestamp on all running instances
print("Job in progress - updating activity timestamps")
now = datetime.now(timezone.utc).isoformat()
for instance in get_runner_instances():
if instance.status == "RUNNING":
try:
update_instance_metadata(instance.name, "last-activity", now)
except Exception as e:
print(f"Error updating {instance.name}: {e}")
return "OK", 200
elif action == "completed":
# Job completed - update activity timestamp
print("Job completed - updating activity timestamps")
now = datetime.now(timezone.utc).isoformat()
for instance in get_runner_instances():
if instance.status == "RUNNING":
try:
update_instance_metadata(instance.name, "last-activity", now)
except Exception as e:
print(f"Error updating {instance.name}: {e}")
return "OK", 200
return "OK", 200
@@ -0,0 +1,4 @@
functions-framework==3.*
google-cloud-compute==1.*
google-cloud-secret-manager==2.*
flask>=2.0.0
+2
View File
@@ -13,6 +13,7 @@ from . import cloud_run
from . import monitoring
from . import iam
from . import networking
from . import runner_manager
__all__ = [
"database",
@@ -24,4 +25,5 @@ __all__ = [
"monitoring",
"iam",
"networking",
"runner_manager",
]
+297
View File
@@ -0,0 +1,297 @@
"""
GitHub Actions Runner Manager - Auto-start and auto-stop self-hosted runners.
Creates infrastructure to automatically manage runner lifecycle:
1. Cloud Function (Gen2) - Handles GitHub webhooks and idle checks
2. Cloud Scheduler - Triggers idle checks every 15 minutes
3. IAM bindings - Permissions to start/stop Compute Engine instances
Architecture:
- GitHub sends workflow_job webhooks to the Cloud Function
- When a job is queued, the function starts any TERMINATED runner VMs
- Cloud Scheduler triggers idle checks every 15 minutes
- If no jobs are pending and runners have been idle, they're stopped
Manual Setup Required:
1. Create a webhook secret:
python -c "import secrets; print(secrets.token_hex(32))" | \
gcloud secrets versions add github-webhook-secret --data-file=-
2. Configure GitHub org webhook at:
https://github.com/organizations/nomadkaraoke/settings/hooks
- Payload URL: <Cloud Function URL from pulumi output>
- Content type: application/json
- Secret: <value from step 1>
- Events: Workflow jobs
"""
import base64
import json
from pathlib import Path
import pulumi
import pulumi_gcp as gcp
from config import (
PROJECT_ID,
REGION,
ZONE,
RunnerManagerConfig,
SecretNames,
)
def create_runner_manager_service_account() -> gcp.serviceaccount.Account:
"""Create service account for the runner manager Cloud Function."""
return gcp.serviceaccount.Account(
"runner-manager-sa",
account_id="runner-manager",
display_name="GitHub Runner Manager",
description="Service account for the GitHub Actions runner manager Cloud Function",
)
def grant_runner_manager_permissions(
service_account: gcp.serviceaccount.Account,
webhook_secret: gcp.secretmanager.Secret,
pat_secret: gcp.secretmanager.Secret,
) -> dict:
"""Grant permissions to the runner manager service account."""
bindings = {}
# Grant Compute Instance Admin - needed to start/stop VMs
bindings["compute_admin"] = gcp.projects.IAMMember(
"runner-manager-compute-admin",
project=PROJECT_ID,
role="roles/compute.instanceAdmin.v1",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Secret Manager accessor for webhook secret
bindings["webhook_secret_accessor"] = gcp.secretmanager.SecretIamMember(
"runner-manager-webhook-secret-access",
secret_id=webhook_secret.id,
role="roles/secretmanager.secretAccessor",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Secret Manager accessor for GitHub PAT (for checking pending jobs)
bindings["pat_secret_accessor"] = gcp.secretmanager.SecretIamMember(
"runner-manager-pat-secret-access",
secret_id=pat_secret.id,
role="roles/secretmanager.secretAccessor",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant logging writer
bindings["logging_writer"] = gcp.projects.IAMMember(
"runner-manager-logging-writer",
project=PROJECT_ID,
role="roles/logging.logWriter",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
return bindings
def create_function_storage_bucket() -> gcp.storage.Bucket:
"""Create a GCS bucket for Cloud Function source code."""
return gcp.storage.Bucket(
"runner-manager-source-bucket",
name=f"{PROJECT_ID}-runner-manager-source",
location=REGION,
uniform_bucket_level_access=True,
lifecycle_rules=[
gcp.storage.BucketLifecycleRuleArgs(
action=gcp.storage.BucketLifecycleRuleActionArgs(type="Delete"),
condition=gcp.storage.BucketLifecycleRuleConditionArgs(age=7),
)
],
)
def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.BucketObject:
"""Create a GCS object with the Cloud Function source code."""
source_dir = Path(__file__).parent.parent / "functions" / "runner_manager"
return gcp.storage.BucketObject(
"runner-manager-source",
bucket=bucket.name,
name="runner-manager-source.zip",
source=pulumi.FileArchive(str(source_dir)),
)
def create_cloud_function(
service_account: gcp.serviceaccount.Account,
bucket: gcp.storage.Bucket,
source_archive: gcp.storage.BucketObject,
permissions: dict,
) -> gcp.cloudfunctionsv2.Function:
"""Create the Cloud Function (Gen2) for runner management."""
return gcp.cloudfunctionsv2.Function(
"runner-manager-function",
name=RunnerManagerConfig.FUNCTION_NAME,
location=REGION,
description="Manages GitHub Actions runner VM lifecycle (auto-start/stop)",
build_config=gcp.cloudfunctionsv2.FunctionBuildConfigArgs(
runtime="python312",
entry_point="handle_request",
source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceArgs(
storage_source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceStorageSourceArgs(
bucket=bucket.name,
object=source_archive.name,
),
),
),
service_config=gcp.cloudfunctionsv2.FunctionServiceConfigArgs(
available_memory=RunnerManagerConfig.FUNCTION_MEMORY,
timeout_seconds=RunnerManagerConfig.FUNCTION_TIMEOUT,
service_account_email=service_account.email,
environment_variables={
"GCP_PROJECT": PROJECT_ID,
"GCP_ZONE": ZONE,
"WEBHOOK_SECRET_NAME": SecretNames.GITHUB_WEBHOOK_SECRET,
"RUNNER_PAT_SECRET_NAME": SecretNames.GITHUB_RUNNER_PAT,
"IDLE_TIMEOUT_HOURS": str(RunnerManagerConfig.IDLE_TIMEOUT_HOURS),
},
min_instance_count=0, # Scale to zero when not in use
max_instance_count=5, # Allow some concurrency for multiple webhooks
),
opts=pulumi.ResourceOptions(
depends_on=list(permissions.values()),
),
)
def allow_unauthenticated_invocation(
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudfunctionsv2.FunctionIamMember:
"""
Allow unauthenticated invocations of the Cloud Function.
Required for GitHub webhooks since GitHub doesn't support GCP auth.
Security is provided by webhook signature verification.
"""
return gcp.cloudfunctionsv2.FunctionIamMember(
"runner-manager-public-invoker",
project=PROJECT_ID,
location=REGION,
cloud_function=function.name,
role="roles/cloudfunctions.invoker",
member="allUsers",
)
def create_scheduler_service_account() -> gcp.serviceaccount.Account:
"""Create service account for Cloud Scheduler to invoke the function."""
return gcp.serviceaccount.Account(
"runner-manager-scheduler-sa",
account_id="runner-manager-scheduler",
display_name="Runner Manager Scheduler",
description="Service account for Cloud Scheduler to trigger runner idle checks",
)
def grant_scheduler_invoker_permission(
scheduler_sa: gcp.serviceaccount.Account,
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudfunctionsv2.FunctionIamMember:
"""Grant the scheduler service account permission to invoke the function."""
return gcp.cloudfunctionsv2.FunctionIamMember(
"runner-manager-scheduler-invoker",
project=PROJECT_ID,
location=REGION,
cloud_function=function.name,
role="roles/cloudfunctions.invoker",
member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"),
)
def create_idle_check_scheduler(
scheduler_sa: gcp.serviceaccount.Account,
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudscheduler.Job:
"""Create Cloud Scheduler job for idle runner checks."""
return gcp.cloudscheduler.Job(
"runner-manager-idle-check",
name="runner-manager-idle-check",
description="Triggers idle check for GitHub Actions runners every 15 minutes",
schedule=RunnerManagerConfig.IDLE_CHECK_SCHEDULE,
time_zone="UTC",
region=REGION,
http_target=gcp.cloudscheduler.JobHttpTargetArgs(
uri=function.url.apply(lambda url: f"{url}?action=check_idle"),
http_method="POST",
headers={
"Content-Type": "application/json",
},
body=base64.b64encode(json.dumps({"trigger": "scheduler"}).encode()).decode(),
oidc_token=gcp.cloudscheduler.JobHttpTargetOidcTokenArgs(
service_account_email=scheduler_sa.email,
audience=function.url,
),
),
retry_config=gcp.cloudscheduler.JobRetryConfigArgs(
retry_count=2,
max_retry_duration="60s",
),
)
def create_runner_manager_resources(
webhook_secret: gcp.secretmanager.Secret,
pat_secret: gcp.secretmanager.Secret,
) -> dict:
"""
Create all resources for the GitHub Actions runner manager.
Args:
webhook_secret: The webhook secret for signature verification.
pat_secret: The GitHub PAT secret for API calls.
Returns:
dict: Dictionary of all created resources.
"""
# Create service accounts
function_sa = create_runner_manager_service_account()
scheduler_sa = create_scheduler_service_account()
# Grant permissions to function service account
permissions = grant_runner_manager_permissions(
function_sa,
webhook_secret,
pat_secret,
)
# Create source storage bucket and archive
bucket = create_function_storage_bucket()
source_archive = create_function_source_archive(bucket)
# Create the Cloud Function
function = create_cloud_function(
function_sa,
bucket,
source_archive,
permissions,
)
# Allow unauthenticated invocation (GitHub webhooks)
public_access = allow_unauthenticated_invocation(function)
# Grant scheduler permission to invoke function
scheduler_invoker = grant_scheduler_invoker_permission(scheduler_sa, function)
# Create the idle check scheduler job
scheduler_job = create_idle_check_scheduler(scheduler_sa, function)
return {
"function_service_account": function_sa,
"scheduler_service_account": scheduler_sa,
"permissions": permissions,
"source_bucket": bucket,
"source_archive": source_archive,
"function": function,
"public_access": public_access,
"scheduler_invoker": scheduler_invoker,
"scheduler_job": scheduler_job,
}
+1
View File
@@ -47,6 +47,7 @@ def create_secrets() -> dict[str, secretmanager.Secret]:
# GitHub
"github-runner-pat",
"github-webhook-secret", # For runner manager webhook verification
# E2E Testing
"e2e-bypass-key",