mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary
The encoding-worker idle-shutdown Cloud Function only iterated
\`[primary_vm, secondary_vm]\` in the default zone — capacity-fallback
VMs in alternate zones (\`-a\`/\`-b\`) were never checked, so a fallback
started during a capacity event stayed RUNNING indefinitely after the
event resolved.
Observed 2026-05-07: \`encoding-worker-fallback-a\` in \`us-central1-a\`
ran for 13+ hours idle (~\$15 cost) before manual intervention.
## Changes
- **\`infrastructure/functions/encoding_worker_idle/main.py\`**:
- Read \`ENCODING_WORKER_FALLBACK_VMS\` env (same secret the backend
uses for capacity failover) and iterate fallbacks alongside
primary/secondary.
- Threaded \`zone\` through \`get_vm_status\` / \`get_vm_ip\` /
\`stop_vm\` so cross-zone shutdowns work.
- Keep-alive-on-recent-activity now follows \`active_override_vm\` (the
currently-routed VM during a capacity event), not always the primary.
Otherwise an active capacity-fallback would get killed while serving
real traffic.
- **\`infrastructure/modules/encoding_worker_manager.py\`** (Pulumi):
- Wire the \`encoding-worker-fallback-vms\` secret into the function via
\`secret_environment_variables\`.
- New \`gcp.secretmanager.SecretIamMember\` binding so the function's SA
can read that secret.
- **\`infrastructure/functions/encoding_worker_idle/deploy.sh\`** (new):
uploads source zip and triggers function rebuild. Was missing for this
function (other functions all have one); matches the pattern from
\`backup_to_aws/deploy.sh\` etc.
-
**\`infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py\`**:
9 new tests:
- Idle fallback gets stopped (in its own zone)
- Active-override fallback kept alive on recent activity (primary stops
instead)
- Fallback with active jobs not stopped
- Graceful behavior when env var missing or malformed JSON
- 4 unit tests on the \`_parse_fallback_vms\` helper
- All 18 tests pass.
## Already applied to prod
I ran \`pulumi up\` and \`deploy.sh\` from this branch to apply the fix.
CI's pulumi-up will be a no-op on merge.
**End-to-end verification:**
1. Started \`encoding-worker-fallback-a\` manually (RUNNING)
2. Triggered \`encoding-worker-idle-check\` scheduler
3. fallback-a went to STOPPING within ~20s ✅
Before fix: fallback-a would have stayed RUNNING (old code never checked
fallbacks).
After fix: fallback gets stopped on the next 5-min idle sweep.
## Test plan
- [x] 18 unit tests pass locally (9 existing + 9 new)
- [x] \`pulumi preview\`: 1 created (secret IAM), 1 updated (function
with secret env var), 263 unchanged
- [x] \`pulumi up\` applied successfully
- [x] \`deploy.sh\` deployed new function code
- [x] Manually verified: fallback-a started → triggered idle-check →
stopped
@coderabbitai ignore
178 lines
6.3 KiB
Python
178 lines
6.3 KiB
Python
"""
|
|
Encoding Worker Idle Shutdown — Cloud Function + Cloud Scheduler.
|
|
|
|
Automatically stops encoding worker VMs after 15 minutes of inactivity.
|
|
Checks every 5 minutes via Cloud Scheduler trigger.
|
|
"""
|
|
|
|
import pulumi
|
|
import pulumi_gcp as gcp
|
|
from config import PROJECT_ID, REGION, ENCODING_WORKER_ZONE, EncodingWorkerConfig
|
|
|
|
|
|
def create_idle_shutdown_service_account():
|
|
"""Create service account for the idle shutdown Cloud Function."""
|
|
return gcp.serviceaccount.Account(
|
|
"encoding-worker-idle-sa",
|
|
account_id="encoding-worker-idle",
|
|
display_name="Encoding Worker Idle Shutdown",
|
|
description="Service account for the encoding worker idle auto-shutdown Cloud Function",
|
|
)
|
|
|
|
|
|
def grant_idle_shutdown_permissions(service_account):
|
|
"""Grant permissions to the idle shutdown service account."""
|
|
bindings = {}
|
|
|
|
bindings["compute_viewer"] = gcp.projects.IAMMember(
|
|
"idle-shutdown-compute-viewer",
|
|
project=PROJECT_ID,
|
|
role="roles/compute.viewer",
|
|
member=service_account.email.apply(lambda e: f"serviceAccount:{e}"),
|
|
)
|
|
|
|
stop_role = gcp.projects.IAMCustomRole(
|
|
"encoding-worker-stop-role",
|
|
role_id="encodingWorkerStop",
|
|
title="Encoding Worker Stop",
|
|
description="Stop encoding worker VMs (for idle shutdown)",
|
|
permissions=["compute.instances.stop"],
|
|
project=PROJECT_ID,
|
|
)
|
|
|
|
bindings["compute_stop"] = gcp.projects.IAMMember(
|
|
"idle-shutdown-compute-stop",
|
|
project=PROJECT_ID,
|
|
role=stop_role.name,
|
|
member=service_account.email.apply(lambda e: f"serviceAccount:{e}"),
|
|
)
|
|
|
|
bindings["firestore"] = gcp.projects.IAMMember(
|
|
"idle-shutdown-firestore",
|
|
project=PROJECT_ID,
|
|
role="roles/datastore.user",
|
|
member=service_account.email.apply(lambda e: f"serviceAccount:{e}"),
|
|
)
|
|
|
|
# The function reads the multi-zone fallback list from Secret Manager
|
|
# (same secret the backend uses) so it knows which fallback VMs to
|
|
# check during idle-sweep. Without this, fallbacks left running after
|
|
# a capacity event leak indefinitely.
|
|
bindings["fallback_secret"] = gcp.secretmanager.SecretIamMember(
|
|
"idle-shutdown-fallback-secret",
|
|
project=PROJECT_ID,
|
|
secret_id="encoding-worker-fallback-vms",
|
|
role="roles/secretmanager.secretAccessor",
|
|
member=service_account.email.apply(lambda e: f"serviceAccount:{e}"),
|
|
)
|
|
|
|
return bindings
|
|
|
|
|
|
def create_idle_shutdown_resources():
|
|
"""
|
|
Create all resources for the encoding worker idle shutdown system.
|
|
|
|
Returns:
|
|
dict: Dictionary of all created resources.
|
|
"""
|
|
sa = create_idle_shutdown_service_account()
|
|
perms = grant_idle_shutdown_permissions(sa)
|
|
|
|
source_bucket = gcp.storage.Bucket(
|
|
"encoding-worker-idle-source",
|
|
name=f"encoding-worker-idle-source-{PROJECT_ID}",
|
|
location="US-CENTRAL1",
|
|
force_destroy=True,
|
|
uniform_bucket_level_access=True,
|
|
)
|
|
|
|
function = gcp.cloudfunctionsv2.Function(
|
|
"encoding-worker-idle-function",
|
|
name=EncodingWorkerConfig.FUNCTION_NAME,
|
|
location=REGION,
|
|
description="Auto-stop idle encoding worker VMs",
|
|
build_config=gcp.cloudfunctionsv2.FunctionBuildConfigArgs(
|
|
runtime="python312",
|
|
entry_point="idle_shutdown",
|
|
source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceArgs(
|
|
storage_source=gcp.cloudfunctionsv2.FunctionBuildConfigSourceStorageSourceArgs(
|
|
bucket=source_bucket.name,
|
|
object="encoding-worker-idle-source.zip",
|
|
),
|
|
),
|
|
),
|
|
service_config=gcp.cloudfunctionsv2.FunctionServiceConfigArgs(
|
|
available_memory=EncodingWorkerConfig.FUNCTION_MEMORY,
|
|
timeout_seconds=EncodingWorkerConfig.FUNCTION_TIMEOUT,
|
|
min_instance_count=0,
|
|
max_instance_count=1,
|
|
service_account_email=sa.email,
|
|
environment_variables={
|
|
"GCP_PROJECT": PROJECT_ID,
|
|
"GCP_ZONE": ENCODING_WORKER_ZONE,
|
|
"IDLE_TIMEOUT_MINUTES": str(EncodingWorkerConfig.IDLE_TIMEOUT_MINUTES),
|
|
},
|
|
# Multi-zone fallback list — same secret as the backend uses for
|
|
# capacity failover. Function reads this to know which fallback
|
|
# VMs to check during idle-sweep.
|
|
secret_environment_variables=[
|
|
gcp.cloudfunctionsv2.FunctionServiceConfigSecretEnvironmentVariableArgs(
|
|
key="ENCODING_WORKER_FALLBACK_VMS",
|
|
project_id=PROJECT_ID,
|
|
secret="encoding-worker-fallback-vms",
|
|
version="latest",
|
|
),
|
|
],
|
|
),
|
|
opts=pulumi.ResourceOptions(
|
|
depends_on=list(perms.values()),
|
|
),
|
|
)
|
|
|
|
scheduler_invoker = gcp.cloudfunctionsv2.FunctionIamMember(
|
|
"idle-shutdown-scheduler-invoker",
|
|
project=PROJECT_ID,
|
|
location=REGION,
|
|
cloud_function=function.name,
|
|
role="roles/cloudfunctions.invoker",
|
|
member=sa.email.apply(lambda e: f"serviceAccount:{e}"),
|
|
)
|
|
|
|
run_invoker = gcp.cloudrunv2.ServiceIamMember(
|
|
"idle-shutdown-run-invoker",
|
|
project=PROJECT_ID,
|
|
location=REGION,
|
|
name=function.name,
|
|
role="roles/run.invoker",
|
|
member=sa.email.apply(lambda e: f"serviceAccount:{e}"),
|
|
)
|
|
|
|
scheduler = gcp.cloudscheduler.Job(
|
|
"encoding-worker-idle-scheduler",
|
|
name="encoding-worker-idle-check",
|
|
description="Check and stop idle encoding worker VMs",
|
|
region=REGION,
|
|
schedule=EncodingWorkerConfig.IDLE_CHECK_SCHEDULE,
|
|
time_zone="UTC",
|
|
http_target=gcp.cloudscheduler.JobHttpTargetArgs(
|
|
uri=function.url,
|
|
http_method="POST",
|
|
oidc_token=gcp.cloudscheduler.JobHttpTargetOidcTokenArgs(
|
|
service_account_email=sa.email,
|
|
),
|
|
),
|
|
retry_config=gcp.cloudscheduler.JobRetryConfigArgs(
|
|
retry_count=1,
|
|
min_backoff_duration="60s",
|
|
max_backoff_duration="300s",
|
|
),
|
|
)
|
|
|
|
return {
|
|
"service_account": sa,
|
|
"function": function,
|
|
"scheduler": scheduler,
|
|
"source_bucket": source_bucket,
|
|
}
|