diff --git a/CLAUDE.md b/CLAUDE.md index 1e241c62..2e9d7955 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,8 +68,9 @@ This single command: When asked to "test it yourself in prod" or verify a fix in production: ```bash -# Get admin token -export KARAOKE_ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d',' -f1) +# Get admin token. claude-readonly@ can read all secrets (Pulumi: +# infrastructure/modules/iam/claude_readonly_sa.py), so no gcloud login is needed. +export KARAOKE_ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d',' -f1) # Option 1: Use the debug template directly cd frontend && node e2e/helpers/debug-prod-template.mjs @@ -128,7 +129,7 @@ Use the `X-Admin-Token` header with the token from Secret Manager: ```bash # Trigger video worker curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/video" \ - -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke)" \ + -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com)" \ -H "Content-Type: application/json" \ -d '{"job_id": "YOUR_JOB_ID"}' diff --git a/docs/STORAGE-RETENTION.md b/docs/STORAGE-RETENTION.md index 43d3fadf..955d57a5 100644 --- a/docs/STORAGE-RETENTION.md +++ b/docs/STORAGE-RETENTION.md @@ -13,7 +13,7 @@ Policy + design: `docs/ARCHITECTURE.md` § Storage Retention. Code: Manual trigger (admin token): ```bash -TOK=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d, -f1) +TOK=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d, -f1) curl -X POST https://api.nomadkaraoke.com/api/internal/storage-retention/run -H "X-Admin-Token: $TOK" curl -X POST https://api.nomadkaraoke.com/api/internal/storage-retention/orphans -H "X-Admin-Token: $TOK" # dry-run ``` diff --git a/docs/TESTING.md b/docs/TESTING.md index 164edfb8..2f4cf4b6 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -744,7 +744,7 @@ cd frontend && node test-my-issue.local.mjs ```bash # Get admin token from Secret Manager -export KARAOKE_ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d',' -f1) +export KARAOKE_ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d',' -f1) # Run the template directly cd frontend && node e2e/helpers/debug-prod-template.mjs diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index ce1c3ea0..22a86e3f 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -33,7 +33,7 @@ angle-brackets reach the shell): # should return 200 fast, not hang JOB_ID="d93747cd" # any in-review job curl -s -m 30 -o /dev/null -w "%{http_code} %{time_total}s\n" \ - -H "Authorization: Bearer $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d, -f1)" \ + -H "Authorization: Bearer $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d, -f1)" \ "https://api.nomadkaraoke.com/api/review/${JOB_ID}/correction-data" # and check signing is logging again gcloud logging read 'resource.type="cloud_run_revision" AND resource.labels.service_name="karaoke-backend" AND jsonPayload.message:"Generated signed"' \ @@ -222,16 +222,16 @@ the PR title before squash-merging. ```bash # 1. Check for stuck downloads curl -s "https://api.nomadkaraoke.com/api/health/job-consistency" \ - -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke)" \ + -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com)" \ | python3 -m json.tool | grep -A3 "downloading_audio_stuck" # 2. Trigger recovery manually (if scheduler hasn't run yet) curl -X POST "https://api.nomadkaraoke.com/api/internal/recover-stuck-jobs" \ - -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke)" + -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com)" # 3. Retry the failed job via admin dashboard or API curl -X POST "https://api.nomadkaraoke.com/api/jobs/YOUR_JOB_ID/retry" \ - -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke)" + -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com)" ``` --- @@ -302,7 +302,7 @@ bumping it. ```bash # 1. Confirm the job is flagged curl -s "https://api.nomadkaraoke.com/api/health/job-consistency" \ - -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke)" \ + -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com)" \ | python3 -m json.tool | grep -A3 "encoding_stuck" # 2. Check what the GCE worker knows about the job @@ -313,7 +313,7 @@ curl -s "$WORKER_URL/status/YOUR_JOB_ID" -H "X-API-Key: $WORKER_KEY" | python3 - # 3. Re-trigger the video worker — it will pick up the cached GCE result if encoding # already finished, or rejoin the poll if it's still running curl -X POST "https://api.nomadkaraoke.com/api/internal/workers/video" \ - -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke)" \ + -H "X-Admin-Token: $(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com)" \ -H "Content-Type: application/json" \ -d '{"job_id": "YOUR_JOB_ID"}' ``` @@ -406,7 +406,7 @@ gcloud compute ssh --zone= --project=nomadkaraoke --tunnel-t **Re-render a job that already failed past review:** a failed job is terminal, and admin `/jobs/{id}/reset` has no `review_complete` target. Reset to `awaiting_review`, then re-submit the review (reuses the existing GCS corrections): ```bash -ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d',' -f1) +ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d',' -f1) curl -s -X POST "https://api.nomadkaraoke.com/api/admin/jobs//reset" \ -H "X-Admin-Token: $ADMIN_TOKEN" -H "Content-Type: application/json" \ -d '{"target_state": "awaiting_review"}' @@ -644,7 +644,7 @@ print((d.get('state_data') or {}).get('render_pending_capacity')) **Force an immediate retry** (Cloud Scheduler runs every 10 min, but you can poke it manually): ```bash -ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d',' -f1) +ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d',' -f1) curl -X POST https://api.nomadkaraoke.com/api/internal/retry-pending-render-jobs \ -H "X-Admin-Token: $ADMIN_TOKEN" ``` @@ -716,7 +716,7 @@ Look for these markers: **Recovery (only if a job somehow still gets stuck — e.g. SIGKILL hit before park completed):** manually transition to `review_complete` and re-trigger the render. The instrumental selection in `state_data` is preserved so no user re-work is needed: ```bash -ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d',' -f1) +ADMIN_TOKEN=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke --account=claude-readonly@nomadkaraoke.iam.gserviceaccount.com | cut -d',' -f1) JID=YOUR_JOB_ID python3 << EOF import os; os.environ['GOOGLE_CLOUD_PROJECT']='nomadkaraoke' diff --git a/infrastructure/modules/iam/claude_readonly_sa.py b/infrastructure/modules/iam/claude_readonly_sa.py index 8599aa06..363d60dc 100644 --- a/infrastructure/modules/iam/claude_readonly_sa.py +++ b/infrastructure/modules/iam/claude_readonly_sa.py @@ -44,6 +44,11 @@ def grant_claude_readonly_permissions( "bigquery_job_user": "roles/bigquery.jobUser", "storage_viewer": "roles/storage.objectViewer", "secretmanager_viewer": "roles/secretmanager.viewer", + # Secret VALUES too (Andrew, 2026-11-19): sessions need the admin API + # token, OAuth creds etc. for remediation, and the gcloud CLI user login + # keeps lapsing. This makes the SA's key secret-equivalent: anyone with + # it can read every secret in the project. + "secretmanager_accessor": "roles/secretmanager.secretAccessor", "run_viewer": "roles/run.viewer", "compute_viewer": "roles/compute.viewer", "logging_viewer": "roles/logging.viewer",