Files
karaoke-gen/infrastructure/modules/runner_manager.py
T
Andrew Beveridge ☄️andClaude Opus 5.5 0449d0c107 fix(infra): build Cloud Function source archives without __pycache__ (v0.297.4) (#1272)
## Summary
A full `pulumi preview` from main showed runner-manager drift ("replace
source, update function"), last applied from the
`karaoke-gen-runner-scan-retry` worktree. I checked it:

- #1262 (that worktree's branch) is fully on main. The squash merge is
byte-identical to the branch tip. The live scheduler job (`retryCount
2`, `maxRetryDuration 60s`, `attemptDeadline 300s`) also matches the
code exactly.
- The drift comes from the **deployed zip containing
`__pycache__/*.pyc`** (including a pytest pyc) from the worktree's test
run. `pulumi.FileArchive` hashes every file, so every clean checkout saw
a phantom diff. Separately, the 7-day lifecycle rule had already deleted
the bucket object.

## Changes
- New `infrastructure/modules/source_archive.clean_source_archive()`
builds an `AssetArchive` of source files only. It skips `__pycache__`,
`*.pyc`, `.pytest_cache`, other tool caches and `.DS_Store`.
- Used for runner-manager, divebar-lookup and divebar-mirror (the three
`FileArchive` users).
- Unit tests, added to the "Infra - Cloud Function tests" CI job.
- LESSONS-LEARNED entry.

## Applied (before merge, per repo convention)
- I ran a targeted preview with `__pycache__` present in the worktree.
Runner-manager showed source replace + function update (the one-time
correction). Divebar-lookup and divebar-mirror showed **no change**.
- Targeted `pulumi up` ran as claude-admin (Andrew's grant): 2 changes,
289 unchanged.
- The new revision is `github-runner-manager-00031-sam`, and the
deployed zip now holds only the 4 source files.
- The re-preview shows 291 unchanged.

## Testing
- `pytest infrastructure/test_source_archive.py`: 4 passed
- `pytest infrastructure/functions/runner_manager`: 75 passed
- Local CodeRabbit review: no findings

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_012RYLVSfwoV9WyncjqMPHrq

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 15:16:34 -05:00

344 lines
13 KiB
Python

"""
GitHub Actions Runner Manager - Ephemeral runner dispatcher.
Creates the infrastructure that turns a `workflow_job.queued` GitHub webhook
into a single-use GCE VM via JIT registration, plus a scheduled orphan-cleanup
pass:
1. Cloud Function (Gen2) — webhook entry point + scheduler entry point
2. Cloud Scheduler — invokes the cleanup + stalled-job re-dispatch pass every 5 minutes
3. IAM bindings — permissions to create/delete Compute Engine instances and
read the webhook secret + GitHub PAT from Secret Manager
Manual setup required (one-time):
1. Create the webhook secret:
python -c "import secrets; print(secrets.token_hex(32))" | \
gcloud secrets versions add github-webhook-secret --data-file=-
2. Configure the GitHub org webhook at:
https://github.com/organizations/nomadkaraoke/settings/hooks
- Payload URL: <Cloud Function URL from pulumi output>
- Content type: application/json
- Secret: <value from step 1>
- Events: Workflow jobs
"""
import base64
import json
from pathlib import Path
import pulumi
import pulumi_gcp as gcp
from modules.source_archive import clean_source_archive
from config import (
PROJECT_ID,
REGION,
ZONE,
RunnerManagerConfig,
SecretNames,
)
# Optional fallback zone for ephemeral GPU runners when the primary zone is
# exhausted. Default us-east4-c — uses ephemeral external IPs (no NAT there).
_pulumi_config = pulumi.Config()
RUNNER_FALLBACK_ZONE = _pulumi_config.get("runnerFallbackZone", "us-east4-c")
def create_runner_manager_service_account() -> gcp.serviceaccount.Account:
"""Create service account for the runner manager Cloud Function."""
return gcp.serviceaccount.Account(
"runner-manager-sa",
account_id="runner-manager",
display_name="GitHub Runner Manager",
description="Service account for the GitHub Actions runner manager Cloud Function",
)
def grant_runner_manager_permissions(
service_account: gcp.serviceaccount.Account,
webhook_secret: gcp.secretmanager.Secret,
pat_secret: gcp.secretmanager.Secret,
runner_service_account: gcp.serviceaccount.Account | None = None,
) -> dict:
"""Grant permissions to the runner manager service account."""
bindings = {}
# Grant Compute Instance Admin - needed to start/stop VMs
bindings["compute_admin"] = gcp.projects.IAMMember(
"runner-manager-compute-admin",
project=PROJECT_ID,
role="roles/compute.instanceAdmin.v1",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Service Account User on the runner SA — required to set metadata
# on VMs that run as the runner service account. Without this, metadata
# writes fail with SERVICE_ACCOUNT_ACCESS_DENIED.
if runner_service_account is not None:
bindings["runner_sa_user"] = gcp.serviceaccount.IAMMember(
"runner-manager-runner-sa-user",
service_account_id=runner_service_account.name,
role="roles/iam.serviceAccountUser",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Secret Manager accessor for webhook secret
bindings["webhook_secret_accessor"] = gcp.secretmanager.SecretIamMember(
"runner-manager-webhook-secret-access",
secret_id=webhook_secret.id,
role="roles/secretmanager.secretAccessor",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Secret Manager accessor for GitHub PAT (for checking pending jobs)
bindings["pat_secret_accessor"] = gcp.secretmanager.SecretIamMember(
"runner-manager-pat-secret-access",
secret_id=pat_secret.id,
role="roles/secretmanager.secretAccessor",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant logging writer
bindings["logging_writer"] = gcp.projects.IAMMember(
"runner-manager-logging-writer",
project=PROJECT_ID,
role="roles/logging.logWriter",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
return bindings
def create_function_storage_bucket() -> gcp.storage.Bucket:
"""Create a GCS bucket for Cloud Function source code."""
return gcp.storage.Bucket(
"runner-manager-source-bucket",
name=f"{PROJECT_ID}-runner-manager-source",
location=REGION,
uniform_bucket_level_access=True,
lifecycle_rules=[
gcp.storage.BucketLifecycleRuleArgs(
action=gcp.storage.BucketLifecycleRuleActionArgs(type="Delete"),
condition=gcp.storage.BucketLifecycleRuleConditionArgs(age=7),
)
],
)
def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.BucketObject:
"""Create a GCS object with the Cloud Function source code."""
source_dir = Path(__file__).parent.parent / "functions" / "runner_manager"
return gcp.storage.BucketObject(
"runner-manager-source",
bucket=bucket.name,
name="runner-manager-source.zip",
source=clean_source_archive(source_dir),
)
def create_cloud_function(
service_account: gcp.serviceaccount.Account,
bucket: gcp.storage.Bucket,
source_archive: gcp.storage.BucketObject,
permissions: dict,
runner_service_account: gcp.serviceaccount.Account | None = None,
) -> gcp.cloudfunctionsv2.Function:
"""Create the Cloud Function (Gen2) for runner management."""
env_vars = {
"GCP_PROJECT": PROJECT_ID,
"GCP_ZONE": ZONE,
"GCP_FALLBACK_ZONE": RUNNER_FALLBACK_ZONE,
"WEBHOOK_SECRET_NAME": SecretNames.GITHUB_WEBHOOK_SECRET,
"RUNNER_PAT_SECRET_NAME": SecretNames.GITHUB_RUNNER_PAT,
"GITHUB_ORG": "nomadkaraoke",
}
# Service account used by ephemeral VMs at runtime (gcloud auth + secret reads).
if runner_service_account is not None:
env_vars["RUNNER_SERVICE_ACCOUNT"] = runner_service_account.email
return gcp.cloudfunctionsv2.Function(
"runner-manager-function",
name=RunnerManagerConfig.FUNCTION_NAME,
location=REGION,
description="Manages GitHub Actions runner VM lifecycle (auto-start/stop)",
build_config=gcp.cloudfunctionsv2.FunctionBuildConfigArgs(
runtime="python312",
entry_point="handle_request",
source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceArgs(
storage_source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceStorageSourceArgs(
bucket=bucket.name,
object=source_archive.name,
# Pin to the current generation so Pulumi treats the source
# bundle being replaced (new code commit) as a diff on the
# Function itself, forcing a redeploy. Without this, the
# bucket-object replaces in place and the live function
# keeps serving the previous staged copy.
generation=source_archive.generation,
),
),
),
service_config=gcp.cloudfunctionsv2.FunctionServiceConfigArgs(
available_memory=RunnerManagerConfig.FUNCTION_MEMORY,
timeout_seconds=RunnerManagerConfig.FUNCTION_TIMEOUT,
service_account_email=service_account.email,
environment_variables=env_vars,
min_instance_count=0, # Scale to zero when not in use
max_instance_count=RunnerManagerConfig.FUNCTION_MAX_INSTANCES,
),
opts=pulumi.ResourceOptions(
depends_on=list(permissions.values()),
),
)
def allow_unauthenticated_invocation(
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudfunctionsv2.FunctionIamMember:
"""
Allow unauthenticated invocations of the Cloud Function.
Required for GitHub webhooks since GitHub doesn't support GCP auth.
Security is provided by webhook signature verification.
"""
return gcp.cloudfunctionsv2.FunctionIamMember(
"runner-manager-public-invoker",
project=PROJECT_ID,
location=REGION,
cloud_function=function.name,
role="roles/cloudfunctions.invoker",
member="allUsers",
)
def create_scheduler_service_account() -> gcp.serviceaccount.Account:
"""Create service account for Cloud Scheduler to invoke the function."""
return gcp.serviceaccount.Account(
"runner-manager-scheduler-sa",
account_id="runner-manager-scheduler",
display_name="Runner Manager Scheduler",
description="Service account for Cloud Scheduler to trigger runner idle checks",
)
def grant_scheduler_invoker_permission(
scheduler_sa: gcp.serviceaccount.Account,
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudfunctionsv2.FunctionIamMember:
"""Grant the scheduler service account permission to invoke the function."""
return gcp.cloudfunctionsv2.FunctionIamMember(
"runner-manager-scheduler-invoker",
project=PROJECT_ID,
location=REGION,
cloud_function=function.name,
role="roles/cloudfunctions.invoker",
member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"),
)
def create_idle_check_scheduler(
scheduler_sa: gcp.serviceaccount.Account,
function: gcp.cloudfunctionsv2.Function,
) -> gcp.cloudscheduler.Job:
"""Create the Cloud Scheduler job that drives the orphan-cleanup pass.
The resource name + ``?action=check_idle`` query parameter are preserved
from the original legacy-mode behaviour so Pulumi doesn't recreate the job
(and so the Cloud Function keeps routing the request). The function now
interprets every scheduler tick as an orphan-cleanup invocation.
"""
return gcp.cloudscheduler.Job(
"runner-manager-idle-check",
name="runner-manager-idle-check",
description="Triggers orphan-cleanup + stalled-job re-dispatch for ephemeral GHA runners every 5 minutes",
schedule=RunnerManagerConfig.IDLE_CHECK_SCHEDULE,
time_zone="UTC",
region=REGION,
http_target=gcp.cloudscheduler.JobHttpTargetArgs(
uri=function.url.apply(lambda url: f"{url}?action=check_idle"),
http_method="POST",
headers={
"Content-Type": "application/json",
},
body=base64.b64encode(json.dumps({"trigger": "scheduler"}).encode()).decode(),
oidc_token=gcp.cloudscheduler.JobHttpTargetOidcTokenArgs(
service_account_email=scheduler_sa.email,
audience=function.url,
),
),
# The pass now scans every org repo and may launch VMs (each insert
# confirm blocks up to 90s). Match the function timeout so a slow tick
# isn't treated as failed and retried while still running — a
# concurrent retry could re-dispatch the same stalled jobs twice.
attempt_deadline=f"{RunnerManagerConfig.FUNCTION_TIMEOUT}s",
retry_config=gcp.cloudscheduler.JobRetryConfigArgs(
retry_count=2,
max_retry_duration="60s",
),
)
def create_runner_manager_resources(
webhook_secret: gcp.secretmanager.Secret,
pat_secret: gcp.secretmanager.Secret,
runner_service_account: gcp.serviceaccount.Account | None = None,
) -> dict:
"""
Create all resources for the GitHub Actions runner manager.
Args:
webhook_secret: The webhook secret for signature verification.
pat_secret: The GitHub PAT secret for API calls.
runner_service_account: The SA that runner VMs run as (for IAM binding).
Returns:
dict: Dictionary of all created resources.
"""
# Create service accounts
function_sa = create_runner_manager_service_account()
scheduler_sa = create_scheduler_service_account()
# Grant permissions to function service account
permissions = grant_runner_manager_permissions(
function_sa,
webhook_secret,
pat_secret,
runner_service_account,
)
# Create source storage bucket and archive
bucket = create_function_storage_bucket()
source_archive = create_function_source_archive(bucket)
# Create the Cloud Function
function = create_cloud_function(
function_sa,
bucket,
source_archive,
permissions,
runner_service_account=runner_service_account,
)
# Allow unauthenticated invocation (GitHub webhooks)
public_access = allow_unauthenticated_invocation(function)
# Grant scheduler permission to invoke function
scheduler_invoker = grant_scheduler_invoker_permission(scheduler_sa, function)
# Create the idle check scheduler job
scheduler_job = create_idle_check_scheduler(scheduler_sa, function)
return {
"function_service_account": function_sa,
"scheduler_service_account": scheduler_sa,
"permissions": permissions,
"source_bucket": bucket,
"source_archive": source_archive,
"function": function,
"public_access": public_access,
"scheduler_invoker": scheduler_invoker,
"scheduler_job": scheduler_job,
}