diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca8987b3..3accc77d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1657,7 +1657,13 @@ jobs: # ============================================ # Deploy to Cloud Run # ============================================ - + # PREREQUISITE (edge-security): the `edge-origin-secret` Secret Manager + # secret is referenced in --set-secrets below. `gcloud run deploy` FAILS + # if that secret has no version. Before this change reaches main, add a + # value: printf '%s' "$(openssl rand -hex 32)" | \ + # gcloud secrets versions add edge-origin-secret --data-file=- --project=nomadkaraoke + # (and set the SAME value as Pulumi `edge:originSecret`). Until cutover the + # backend runs EDGE_AUTH_MODE=off, so the secret is loaded but not enforced. - name: Deploy to Cloud Run run: | VERSION="${{ steps.version_check.outputs.version }}" @@ -1683,8 +1689,8 @@ jobs: --min-instances 1 \ --vpc-connector projects/nomadkaraoke/locations/us-central1/connectors/cloud-run-connector \ --vpc-egress private-ranges-only \ - --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-4-8,AUTO_CORRECT_COMPARE_MODELS=claude-opus-4-8;gemini-3.1-pro-preview,AUTO_CORRECT_PROACTIVE_ENABLED=true,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \ - --set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest" + --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-4-8,AUTO_CORRECT_COMPARE_MODELS=claude-opus-4-8;gemini-3.1-pro-preview,AUTO_CORRECT_PROACTIVE_ENABLED=true,EDGE_AUTH_MODE=off,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \ + --set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest" - name: Update Cloud Run Jobs with new image run: | diff --git a/backend/main.py b/backend/main.py index 9fee3038..b53ff844 100644 --- a/backend/main.py +++ b/backend/main.py @@ -18,6 +18,7 @@ from backend.services.nltk_preloader import preload_all_nltk_resources from backend.services.langfuse_preloader import preload_langfuse_handler from backend.middleware.audit_logging import AuditLoggingMiddleware from backend.middleware.tenant import TenantMiddleware +from backend.middleware.edge_auth import EdgeAuthMiddleware from backend.workers.registry import worker_registry @@ -170,6 +171,12 @@ app.add_middleware(AuditLoggingMiddleware) # Add tenant detection middleware (extracts tenant from subdomain/headers) app.add_middleware(TenantMiddleware) +# Edge origin lock (added last = outermost, runs first). Rejects direct-to-origin +# requests that bypassed the Cloudflare edge, before tenant/audit middleware do +# any work. No-op unless EDGE_AUTH_MODE is warn/enforce. See +# backend/middleware/edge_auth.py. +app.add_middleware(EdgeAuthMiddleware) + # Include routers app.include_router(health.router, prefix="/api") app.include_router(jobs.router, prefix="/api") diff --git a/backend/middleware/edge_auth.py b/backend/middleware/edge_auth.py new file mode 100644 index 00000000..47ab05c7 --- /dev/null +++ b/backend/middleware/edge_auth.py @@ -0,0 +1,115 @@ +""" +Edge origin-lock middleware. + +Cloudflare fronts ``api.nomadkaraoke.com`` (WAF + rate limiting). To stop +attackers from bypassing the edge by hitting the Cloud Run origin directly +(``*.run.app`` / ``ghs.googlehosted.com``), Cloudflare injects a shared-secret +header (``X-Edge-Auth``) on every proxied request via a Transform Rule. This +middleware rejects public requests that lack the correct header. + +A Cloud Run *domain mapping* can't use ``ingress=internal-and-cloud-load-balancing`` +(that needs a load balancer), so this app-layer check is how we lock the origin. + +Modes (env ``EDGE_AUTH_MODE``, default ``off``): + off - passthrough, no checks (deploy-dormant; behaviour unchanged). + warn - log requests missing/failing the header, but ALLOW them. Use to + confirm nothing legit is missing the header before enforcing. + enforce - reject requests missing/failing the header with 403. + +Secret: env ``EDGE_ORIGIN_SECRET`` (same value Cloudflare injects). If enforce +mode is set but the secret is unconfigured, the middleware FAILS CLOSED (503) — +a security control must not silently allow direct-origin traffic. (Low risk: +Cloud Run won't start a revision whose secret ref is missing, and enforce is +only enabled after staging confirms the secret.) In warn mode it still allows. + +Exemptions (always allowed, even in enforce mode): + * Health / root endpoints and Cloud Run probes, which hit the origin + directly (not via Cloudflare) and legitimately lack the header. +""" + +import hmac +import logging +import os + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +logger = logging.getLogger(__name__) + +# Header Cloudflare injects (must match CloudflareConfig.ORIGIN_AUTH_HEADER in +# infrastructure/modules/edge_security.py). +EDGE_AUTH_HEADER = "X-Edge-Auth" + +# Paths always allowed without the edge header. These are non-sensitive and are +# probed directly on the origin by Cloud Run startup/liveness checks and GCP +# uptime monitoring, which don't traverse Cloudflare. +_EXEMPT_EXACT = {"/", "/api/health", "/api/health/"} +_EXEMPT_PREFIXES = ("/api/health/",) + +_VALID_MODES = {"off", "warn", "enforce"} + + +def _mode() -> str: + mode = os.environ.get("EDGE_AUTH_MODE", "off").lower().strip() + return mode if mode in _VALID_MODES else "off" + + +def _is_exempt(path: str) -> bool: + if path in _EXEMPT_EXACT: + return True + return any(path.startswith(p) for p in _EXEMPT_PREFIXES) + + +class EdgeAuthMiddleware(BaseHTTPMiddleware): + """Reject origin requests that didn't come through the Cloudflare edge.""" + + async def dispatch(self, request: Request, call_next) -> Response: + mode = _mode() + if mode == "off": + return await call_next(request) + + if _is_exempt(request.url.path): + return await call_next(request) + + secret = os.environ.get("EDGE_ORIGIN_SECRET", "") + if not secret: + # Misconfiguration: enforce is requested but no secret is wired up. + # In enforce mode, fail CLOSED (503) — a security control must not + # silently allow direct-origin traffic when it can't do its job. + # This is low-risk operationally: Cloud Run won't even start a + # revision whose --set-secrets ref is missing, and enforce is only + # turned on after staging confirms the secret. In warn mode we still + # allow (observational only). + logger.error( + "EDGE_AUTH_MODE=%s but EDGE_ORIGIN_SECRET is unset.", mode, + ) + if mode == "enforce": + return JSONResponse( + status_code=503, + content={"detail": "Origin authentication unavailable"}, + ) + return await call_next(request) + + provided = request.headers.get(EDGE_AUTH_HEADER, "") + # Constant-time comparison to avoid leaking the secret via timing. + ok = bool(provided) and hmac.compare_digest(provided, secret) + + if ok: + return await call_next(request) + + client = request.client.host if request.client else "?" + if mode == "warn": + logger.warning( + "edge-auth: request missing/invalid %s header (warn mode, " + "allowing). path=%s ip=%s", + EDGE_AUTH_HEADER, request.url.path, client, + ) + return await call_next(request) + + # enforce + logger.warning( + "edge-auth: blocked direct-to-origin request. path=%s ip=%s", + request.url.path, client, + ) + return JSONResponse(status_code=403, content={"detail": "Forbidden"}) diff --git a/backend/tests/test_edge_auth_middleware.py b/backend/tests/test_edge_auth_middleware.py new file mode 100644 index 00000000..4ce5088a --- /dev/null +++ b/backend/tests/test_edge_auth_middleware.py @@ -0,0 +1,139 @@ +""" +Unit tests for EdgeAuthMiddleware (Cloudflare origin lock). + +Exercises the off/warn/enforce modes, header validation, exemptions, and the +fail-open-on-misconfiguration behaviour end-to-end via a minimal ASGI app. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +from backend.middleware.edge_auth import EdgeAuthMiddleware, EDGE_AUTH_HEADER + +SECRET = "s3cr3t-origin-value" + + +@pytest.fixture +def client(): + """Minimal app with the middleware installed and a few routes.""" + app = FastAPI() + app.add_middleware(EdgeAuthMiddleware) + + @app.get("/") + async def root(): + return {"ok": "root"} + + @app.get("/api/health") + async def health(): + return {"ok": "health"} + + @app.get("/api/test") + async def protected(): + return {"ok": "protected"} + + @app.post("/api/internal/recover-stuck-jobs") + async def internal(): + return {"ok": "internal"} + + return TestClient(app) + + +def _set(monkeypatch, mode=None, secret=None): + for key, val in (("EDGE_AUTH_MODE", mode), ("EDGE_ORIGIN_SECRET", secret)): + if val is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, val) + + +# --------------------------------------------------------------------------- # +# off mode (default) — pure passthrough +# --------------------------------------------------------------------------- # +def test_off_mode_is_default_passthrough(client, monkeypatch): + _set(monkeypatch, mode=None, secret=SECRET) # unset mode -> defaults to off + assert client.get("/api/test").status_code == 200 + + +def test_invalid_mode_treated_as_off(client, monkeypatch): + _set(monkeypatch, mode="bogus", secret=SECRET) + assert client.get("/api/test").status_code == 200 + + +# --------------------------------------------------------------------------- # +# enforce mode +# --------------------------------------------------------------------------- # +def test_enforce_blocks_missing_header(client, monkeypatch): + _set(monkeypatch, mode="enforce", secret=SECRET) + resp = client.get("/api/test") + assert resp.status_code == 403 + assert resp.json()["detail"] == "Forbidden" + + +def test_enforce_blocks_wrong_header(client, monkeypatch): + _set(monkeypatch, mode="enforce", secret=SECRET) + resp = client.get("/api/test", headers={EDGE_AUTH_HEADER: "nope"}) + assert resp.status_code == 403 + + +def test_enforce_allows_valid_header(client, monkeypatch): + _set(monkeypatch, mode="enforce", secret=SECRET) + resp = client.get("/api/test", headers={EDGE_AUTH_HEADER: SECRET}) + assert resp.status_code == 200 + assert resp.json()["ok"] == "protected" + + +def test_enforce_internal_path_requires_header(client, monkeypatch): + """Internal endpoints traverse Cloudflare (public host) so they DO get the + header — they are not exempt from the origin lock.""" + _set(monkeypatch, mode="enforce", secret=SECRET) + assert client.post("/api/internal/recover-stuck-jobs").status_code == 403 + ok = client.post( + "/api/internal/recover-stuck-jobs", headers={EDGE_AUTH_HEADER: SECRET} + ) + assert ok.status_code == 200 + + +# --------------------------------------------------------------------------- # +# exemptions — health/root probes hit the origin directly, no header +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("path", ["/", "/api/health"]) +def test_enforce_exempts_health_and_root(client, monkeypatch, path): + _set(monkeypatch, mode="enforce", secret=SECRET) + assert client.get(path).status_code == 200 + + +# --------------------------------------------------------------------------- # +# warn mode — logs but allows +# --------------------------------------------------------------------------- # +def test_warn_allows_missing_header(client, monkeypatch): + _set(monkeypatch, mode="warn", secret=SECRET) + assert client.get("/api/test").status_code == 200 + + +def test_warn_allows_valid_header(client, monkeypatch): + _set(monkeypatch, mode="warn", secret=SECRET) + assert client.get("/api/test", headers={EDGE_AUTH_HEADER: SECRET}).status_code == 200 + + +# --------------------------------------------------------------------------- # +# misconfiguration — enforce but no secret set +# --------------------------------------------------------------------------- # +def test_enforce_without_secret_fails_closed(client, monkeypatch): + # A security control must fail closed: enforce with no secret → 503, not open. + _set(monkeypatch, mode="enforce", secret=None) + resp = client.get("/api/test") + assert resp.status_code == 503 + assert resp.json()["detail"] == "Origin authentication unavailable" + + +def test_enforce_without_secret_still_exempts_health(client, monkeypatch): + # Health/root stay reachable even in the misconfigured state (probes). + _set(monkeypatch, mode="enforce", secret=None) + assert client.get("/api/health").status_code == 200 + + +def test_warn_without_secret_allows(client, monkeypatch): + # warn mode is observational only — never blocks, even with no secret. + _set(monkeypatch, mode="warn", secret=None) + assert client.get("/api/test").status_code == 200 diff --git a/docs/archive/2026-07-20-edge-security-cutover-runbook.md b/docs/archive/2026-07-20-edge-security-cutover-runbook.md new file mode 100644 index 00000000..8dbc0995 --- /dev/null +++ b/docs/archive/2026-07-20-edge-security-cutover-runbook.md @@ -0,0 +1,242 @@ +# Edge Security — Cutover Runbook + +Companion to `2026-07-20-edge-security-hardening-plan.md`. Step-by-step ops +sequence to put `api.nomadkaraoke.com` behind the Cloudflare edge with **near-zero +downtime**. Every prod-affecting action is a single, instantly-reversible flip. + +**Who runs what:** all `pulumi`, `gcloud`, and Cloudflare-token steps are run by +**Andrew** (agent has read-only ADC). The code (Pulumi module, middleware, alert, +CI env) is already in this branch. + +--- + +## Legend +- 🟢 non-disruptive (no prod impact) +- 🔴 prod-affecting (but instantly reversible) +- ✅ verification gate — do not proceed until green + +--- + +## Prereqs (one-time) 🟢 + +1. **Cloudflare token** — the `.envrc` `CLOUDFLARE_API_TOKEN` (`nomadkaraoke-claude`) + now has the required zone scopes (**verified 2026-07-20**: DNS record + create+delete OK, rulesets list OK, zone settings editable). Point Pulumi at it: + ```bash + cd infrastructure + pulumi config set --secret cloudflare:apiToken "$CLOUDFLARE_API_TOKEN" + # zone id (807f07f458f9cd38251f3b7948d55172) is baked in as the default; + # only set these if overriding: + # pulumi config set edge:cloudflareZoneId + # pulumi config set edge:cloudflareAccountId + pulumi config set edge:rolloutStage staging # default anyway + # Zone is on the FREE plan → managed OWASP ruleset stays OFF (Pro+ only). + # Leave edge:managedWafEnabled unset; set true only after upgrading to Pro. + ``` + > (Optional least-privilege: mint a dedicated token scoped to nomadkaraoke.com + > with DNS:Edit, Zone WAF:Edit, Zone Settings:Edit, Transform Rules:Edit, + > Zone:Read, and use that instead of the broad account token.) + > **Re-check any token before enabling** (should print `success: True`): + > ```bash + > curl -s -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + > "https://api.cloudflare.com/client/v4/zones/807f07f458f9cd38251f3b7948d55172/dns_records?per_page=1" \ + > | python3 -c "import sys,json;print('success:',json.load(sys.stdin)['success'])" + > ``` +2. **Create the secret container + other additive infra first** (edge module is + still OFF — `edge:enabled` defaults to false, so NO Cloudflare calls happen): + ```bash + cd infrastructure && pulumi preview # expect ONLY additions (incl. edge-origin-secret); no Cloudflare + pulumi up + ``` + ✅ No Cloudflare resources, no staging mapping, prod `api` untouched. + +3. **Origin secret** — generate once, store in BOTH Secret Manager (backend reads + it) and Pulumi config (Cloudflare injects it). Must be identical. + ```bash + SECRET=$(openssl rand -hex 32) + printf '%s' "$SECRET" | gcloud secrets versions add edge-origin-secret \ + --data-file=- --project=nomadkaraoke # container created in step 2 + pulumi config set --secret edge:originSecret "$SECRET" + unset SECRET + ``` + > ⚠️ The CI deploy references `edge-origin-secret:latest` — a backend deploy + > will fail until this version exists. Add it **before** this branch merges. + +4. **Activate + apply the edge module (staging scope)** — creates the staging + Cloud Run mapping + staging DNS + WAF/rate-limit/header-transform/zone-settings, + all scoped to `api-edge-test.nomadkaraoke.com` only. The live prod `api` record + is NOT managed at this stage (the module only manages it once rolloutStage=prod), + so there is nothing to import yet and zero risk to prod DNS: + ```bash + pulumi config set --secret cloudflare:apiToken "$CLOUDFLARE_API_TOKEN" + pulumi config set edge:enabled true # rolloutStage defaults to "staging" + pulumi preview # expect: NEW staging resources only; prod `api` record NOT in the plan + pulumi up # (use --target on the edge URNs if the stack has unrelated drift) + ``` + > The staging `api-edge-test` record is created **grey-cloud (proxied=False)** + > on purpose — see step 6 (the SSL dragon). It gets flipped to orange after the + > cert issues. + > Zone `security_level` (already "medium") is set via `cloudflare.ZoneSetting` + > — it adopts the current value (no-op). If preview errors that it "exists", + > import it: `pulumi import cloudflare:index/zoneSetting:ZoneSetting zone-security-level 807f07f458f9cd38251f3b7948d55172/security_level`. + > + > **Free-plan gotchas already handled in code (learned 2026-07-20 apply):** + > WAF uses the `contains` operator not regex `matches` (regex = Business+); + > rate-limit `period` must be 10s and `characteristics` MUST include + > `cf.colo.id`. Managed OWASP ruleset stays off (Pro+). + +--- + +## Phase A — Ship code dormant 🟢 + +5. Merge this branch (or deploy backend) with `EDGE_AUTH_MODE=off`. The + middleware is a no-op; the alert retune and staging infra are live. + ✅ Prod behaves exactly as before: `curl -sI https://api.nomadkaraoke.com/` + still shows `server: Google Frontend` (no cf-ray). + +--- + +## Phase B — Validate everything on staging 🟢 ✅ + +6. **The SSL dragon (grey → provision → orange).** With the record grey-cloud, + wait for the Cloud Run managed cert to provision (DNS must resolve to `ghs` + for ACME — a proxied record blocks issuance): + ```bash + gcloud beta run domain-mappings describe --domain api-edge-test.nomadkaraoke.com \ + --region us-central1 --project nomadkaraoke \ + --format='value(status.conditions[].type, status.conditions[].status)' + # wait until Ready=True / CertificateProvisioned=True (~15 min) + ``` + Then flip the staging record to **orange-cloud** and choose SSL mode: + ```bash + pulumi config set edge:proxyStaging true + pulumi up --target 'urn:...:DnsRecord::api-edge-test-dns' + ``` + Zone SSL/TLS mode should be **Full (strict)**; if the proxied host then throws + 525/handshake, step down the ladder (step 8). +7. Run the verification suite against **staging** (`HOST=api-edge-test.nomadkaraoke.com`): + ```bash + HOST=api-edge-test.nomadkaraoke.com + curl -sI https://$HOST/ | grep -iE 'server|cf-ray' # expect server: cloudflare + cf-ray + for p in '.env' 'home/root/.ssh/id_rsa' 'wp-config.php' 'etc/passwd'; do + echo -n "$p -> "; curl -s -o /dev/null -w '%{http_code}\n' "https://$HOST/$p"; done # expect 403 + curl -s -o /dev/null -w 'health %{http_code}\n' https://$HOST/api/health # expect 200 + for i in $(seq 1 120); do curl -s -o /dev/null -w '%{http_code} ' https://$HOST/api/health; done; echo # rate-limit trips + ``` + ✅ cf-ray present, exploit paths 403, health 200, rate-limit engages, **no + 525/SSL handshake errors**. + +8. **If SSL errors appear**, step down the ladder (edit `edge_security.py` / + zone SSL mode) and re-test: + - `Full (strict)` → `Full` (skip strict cert validation), or + - add a Cloudflare **Origin Rule** overriding origin host + SNI to + `karaoke-backend-ipzqd2k4yq-uc.a.run.app` (always-valid `*.run.app` cert). + +9. **Origin-lock check** — deploy backend with `EDGE_AUTH_MODE=enforce` (test + env / temporary override), then hit a **non-exempt** path directly on the + origin. Exempt paths are only `/`, `/api/health`, `/api/health/*` (probes) — + testing those would return 200 and prove nothing, so use a real endpoint: + ```bash + RUNAPP=karaoke-backend-ipzqd2k4yq-uc.a.run.app + curl -s -o /dev/null -w '%{http_code}\n' https://$RUNAPP/api/jobs # expect 403 (direct, non-exempt) + curl -s -o /dev/null -w '%{http_code}\n' https://$RUNAPP/api/health # expect 200 (exempt probe path) + curl -s -o /dev/null -w '%{http_code}\n' https://api-edge-test.nomadkaraoke.com/api/jobs # expect NOT 403 (via edge → has header) + ``` + ✅ Non-exempt path blocked direct-to-origin (403); same path via the edge is + allowed; health/root stay 200 direct (exempt for probes). + +10. **Soak** staging proxied for a few days; watch for delayed cert/525 issues + and any WAF false positives (Cloudflare Firewall Events / Security dashboard). + +--- + +## Phase C — (Pro only) enable + block the managed WAF ruleset 🟢 + +11. **Skip on the Free plan** — the managed OWASP ruleset is Pro+ and is gated + off (`edge:managedWafEnabled` default false). If/when the zone is upgraded to + Pro: `pulumi config set edge:managedWafEnabled true`, `pulumi up` (deploys it + in **log** mode), review Firewall Events for false positives, then change + `_managed_action = "log"` → `"block"` in `edge_security.py` and `pulumi up`. + +--- + +## Phase D — Prod cutover (two isolated steps) + +Pick a low-traffic window. `gen.nomadkaraoke.com` depends on this API. The prod +edge rules and the DNS proxy flip are now **separate** switches +(`edge:rolloutStage=prod` vs `edge:proxyProdApi=true`) so we provision + verify +the rules while the API is still DNS-only, then flip the proxy on its own. + +12. **Provision prod edge rules (still DNS-only)** 🟢 — extends the WAF / + rate-limit / header-transform rules to the prod host AND brings the prod + `api` record under management (still proxied=False). Because the record + already exists in Cloudflare, IMPORT it first so Pulumi adopts it instead of + creating a duplicate: + ```bash + cd infrastructure + pulumi config set edge:rolloutStage prod + # Import the existing prod `api` CNAME (only now does the module manage it): + pulumi import cloudflare:index/dnsRecord:DnsRecord api-dns \ + 807f07f458f9cd38251f3b7948d55172/6c31cba0080ff334a85cfff6c2927219 + pulumi preview # expect: edge rules add api.nomadkaraoke.com; api DnsRecord = NO diff (proxied=False, ttl=1) + pulumi up + ``` + ✅ `api` still resolves direct to `ghs` (no cf-ray); prod WAF/header rules now + exist; the `api` record is managed with proxied=False. Nothing user-facing changed. + +13. Set backend `EDGE_AUTH_MODE=warn` in prod and redeploy (observe, don't + block). This is inert until step 14 makes CF inject the header, but staging + it now means the very first proxied requests are only warned, not blocked. + +14. **Flip the prod DNS proxy** 🔴 (the single reversible change): + ```bash + pulumi config set edge:proxyProdApi true + pulumi preview # expect: ONLY the api DnsRecord proxied False->True; no other diffs + pulumi up + ``` +15. Verify against **prod** (`HOST=api.nomadkaraoke.com`) — same suite as step 7, + plus a real product smoke test (load `gen.nomadkaraoke.com`, start a job, + confirm signed-URL upload to GCS still works — uploads bypass CF so must be + unaffected). Watch backend logs for `edge-auth` warn lines from any legit + caller missing the header (incl. tenants) before enforcing. + ✅ cf-ray present, exploit paths 403, product works end-to-end, no warn spam. +16. Once stable (minutes-hours) and warn logs are clean, set + `EDGE_AUTH_MODE=enforce` in prod + redeploy. + ✅ `curl -s -o /dev/null -w '%{http_code}\n' https://karaoke-backend-ipzqd2k4yq-uc.a.run.app/api/jobs` → 403. + +### 🔴 Rollback (any time within Phase D) +```bash +cd infrastructure +pulumi config set edge:proxyProdApi false # prod api record → proxied=False (rules stay provisioned) +pulumi up # reverts within one DNS TTL +``` +And/or set `EDGE_AUTH_MODE=off` + redeploy backend. Uploads (signed-URL→GCS) are +never affected either way. + +--- + +## Phase E — Cleanup 🟢 + +17. After prod is stable for ~1 week, remove the staging scaffold: delete + `create_staging_domain_mapping()` + the staging DnsRecord from + `edge_security.py` (and drop `STAGING_API_HOST` from the rule host lists), + `pulumi up`. Keep everything else. + +--- + +## Notes / gotchas captured +- **Uploads bypass the API** (signed-URL direct-to-GCS) → Cloudflare's 100MB + proxy cap is a non-issue. +- **Schedulers** hit the public host `api.nomadkaraoke.com/api/internal/*` → they + traverse CF and get the header automatically; they're **excluded from rate + limiting** but **not** from the origin-lock (they're expected to carry the + header). Their OIDC auth is unchanged. +- **Tenant subdomains** (`singa`, `vocalstar`) — live but idle. They're Cloudflare + Pages frontends that call `api.nomadkaraoke.com` with an `X-Tenant-ID` header + (their own `/api/*` 404s at Cloudflare — no backend passthrough). So they get + the `X-Edge-Auth` header via the proxied `api` host like everyone else and are + NOT expected to break. `warn` mode (steps 13/15) is the safety net — if any + tenant request shows up in the missing-header logs, fix before `enforce`. + Downtime tolerable per Andrew. +- The error-rate alert is a **rate** (req/s), not a true %. Now scoped to 5xx so + client/bot 404s never page. diff --git a/docs/archive/2026-07-20-edge-security-hardening-plan.md b/docs/archive/2026-07-20-edge-security-hardening-plan.md new file mode 100644 index 00000000..b8f04c71 --- /dev/null +++ b/docs/archive/2026-07-20-edge-security-hardening-plan.md @@ -0,0 +1,261 @@ +# Edge Security Hardening — Cloudflare WAF for `api.nomadkaraoke.com` + +**Date:** 2026-07-20 +**Branch:** `feat/sess-20260720-1407-edge-security-hardening` +**Trigger:** Prod alert "Karaoke Backend - High Error Rate" fired 2026-07-20 17:43 UTC (0.167 vs 0.1 threshold). +**Prime directive:** *No extended downtime on the product.* `gen.nomadkaraoke.com` +(frontend) depends on `api.nomadkaraoke.com`. Validate every unknown on a +throwaway staging hostname first; the only prod-affecting action is a single, +instantly-reversible DNS proxy flip. + +--- + +## 1. Incident recap & root cause + +A single IP (`195.178.110.105`) ran an automated vulnerability scan against +`api.nomadkaraoke.com` — 323 requests/24h, ~1 every 2s — probing secret-exposure +paths (`/.env`, `/etc/passwd`, `/home/*/.ssh/id_rsa`, `wp-config.php`, +`.aws/credentials`, `/proc/self/environ`, …). **Every request returned 404** — +nothing exposed, no functional impact. + +It tripped the alert only because on a low-traffic backend, 312/348 requests that +hour were bot 404s (~90%), and the alert counts **all non-2xx** with **no minimum +request-count floor**. + +**Why we were reachable — the real gap:** + +| Surface | Edge today | +|---|---| +| `nomadkaraoke.com`, `www`, `decide`, `gen` (frontend) | ✅ Proxied through Cloudflare (orange-cloud Pages) | +| **`api.nomadkaraoke.com`** (karaoke-backend) | ❌ **DNS-only** CNAME → `ghs.googlehosted.com`; Cloud Run **ingress=`all`**; no WAF, no rate limit | + +Confirmed live: `curl -sI https://api.nomadkaraoke.com/` → `server: Google +Frontend`, no `cf-ray`. `infrastructure/__main__.py:665-673` even exports the +intended DNS record with `"proxied": False`. The API origin is our one +internet-facing surface with zero edge protection, served via a Cloud Run +**domain mapping** (not a Load Balancer) — which is why Cloud Armor can't attach +without re-architecting. + +## 2. Decisions (confirmed with Andrew) + +- **Architecture:** Cloudflare edge (orange-cloud the API + Cloudflare WAF). +- **Management:** Pulumi IaC (`pulumi_cloudflare`), version-controlled. +- **Scope:** Full — block + WAF/rate-limiting + origin lockdown + alert tuning. +- **Constraint:** ≤1h total downtime, ideally ~zero; test all unknowns off-prod. + +## 3. Confirmed facts (from investigation) + +- **Backend = FastAPI** (`backend/main.py:147`); existing middleware chain + (`AuditLoggingMiddleware`, `TenantMiddleware`) — clean insertion point. +- **Tenant subdomains — investigated 2026-07-20.** `singa.nomadkaraoke.com` and + `vocalstar.nomadkaraoke.com` are **live but idle**. They are **Cloudflare Pages + frontends** (already proxied; `/api/*` on them 404s at Cloudflare — no backend + passthrough). All their API traffic goes to `api.nomadkaraoke.com` with an + **`X-Tenant-ID` header** (`frontend/lib/tenant.ts`, `lib/api.ts` → + `API_BASE_URL=api.nomadkaraoke.com`), incl. tenant logo assets. So once `api` + is proxied they get the `X-Edge-Auth` header like everyone else — **edge-auth + should NOT break them.** `TenantMiddleware`'s Host-based detection path is + dormant (the backend never receives Host=`*.nomadkaraoke.com` tenant hosts). + Idle + downtime-tolerant per Andrew; `warn` mode catches any missed path. +- **Schedulers call the public host** `https://api.nomadkaraoke.com/api/internal/*` + (`__main__.py:347-445`) via OIDC. Once proxied they traverse Cloudflare and get + the injected secret header automatically — **but must be excluded from rate + limits** (cron bursts) and their OIDC auth is unchanged. +- **Uploads bypass the API** — signed-URL direct-to-GCS + (`karaoke_gen/utils/remote_cli.py:440`, "bypass Cloud Run's 32MB request body + limit"). So Cloudflare's 100MB proxy cap is a **non-issue**; the API is a + control plane (small JSON, async jobs via Cloud Tasks). +- **Only one domain mapping today** (`api.nomadkaraoke.com → karaoke-backend`, + Ready) → we can add a **second staging mapping to the same service** for + end-to-end edge testing without touching prod. +- **Cloudflare token — RESOLVED 2026-07-20.** The `.envrc` `CLOUDFLARE_API_TOKEN` + is the account token `nomadkaraoke-claude` (account `a7dd2a2b…`). It was + originally missing zone DNS/WAF/settings access (DNS records + rulesets + + settings all 403); Andrew added the scopes. **Re-verified:** DNS record + create+delete succeeds, rulesets list 200, zone settings readable/editable. So + `cloudflare:apiToken` can reuse this token. (It's a broad account-admin token; + a dedicated least-privilege zone-scoped token would be nicer but isn't required.) +- **Zone plan = Free** (verified). The **Cloudflare Managed (OWASP) ruleset is + Pro+ only**, so it's **gated off by default** (`edge:managedWafEnabled`, + default false) — deploying it on Free errors. Free DOES support everything + else we use: custom firewall rules (≤5), one rate-limit rule, Transform Rules + (header inject), Bot Fight Mode, Security Level — enough for the path-scanning + threat. Pro (~$20/mo) later unlocks OWASP CRS. +- **Zone id:** `807f07f458f9cd38251f3b7948d55172` (baked in as the default in + `config.py`; overridable via `edge:cloudflareZoneId`). +- **Activation gate:** the whole module is a no-op until `edge:enabled=true` + (default false), so merging it can't make `pulumi up` attempt Cloudflare calls + with the wrong token. + +## 4. The dragon: Cloud Run domain mapping × Cloudflare proxy (TLS) + +The reason the API is grey-cloud today is almost certainly the **managed-cert × +proxy interaction**: + +- A Cloud Run domain mapping provisions a **Google-managed TLS cert** via ACME, + which needs the hostname to resolve to `ghs.googlehosted.com` for validation. +- With Cloudflare orange-cloud + **Full (strict)**, Cloudflare connects to the + origin over TLS and validates the origin cert for `api.nomadkaraoke.com`. Works + **while the cert is valid**. +- **The delayed failure:** the managed cert must **renew** (~every 90d). Renewal + validation traffic can be broken by the proxy terminating TLS/ALPN → renewal + fails weeks later → **525/526 SSL errors** even though it "worked at first." + This matches the "it worked then broke" memory. + +**Mitigation = an SSL fallback ladder, proven on staging (§5, Phase B):** + +1. **Full (strict) on the existing mapping** — simplest; test first. +2. **Full (not strict)** — Cloudflare encrypts to origin but skips strict cert + validation, so a renewal hiccup won't 525. Slightly weaker (CF↔Google hop), + low risk on Google's network. +3. **SNI-override to the `*.run.app` origin** (most robust): a Cloudflare **Origin + Rule** sets origin host + SNI to `karaoke-backend-…-uc.a.run.app` (whose + Google cert is always valid and never our concern), while the public host stays + `api.nomadkaraoke.com`. Sidesteps the managed-cert dependency entirely. Requires + the backend to accept the run.app Host (ingress=all already does) and CORS to + allow the public origin (already `*`). + +We pick the highest rung that passes on staging and **leave staging proxied for a +few days** to smoke-test before prod cutover. If we want to fully eliminate the +cert dragon long-term, the follow-up is a GCP HTTPS LB (deferred — cost/scope). + +## 5. Phased plan (zero-downtime by construction) + +### Phase 0 — Immediate mitigation (ship now, independent) +No dependency on the rest; stops paging + blocks the active scanner. +- **0a. Alert retune** (`monitoring.py`, §Phase E) — merge/apply first so scanner + 404s stop paging regardless of the WAF timeline. +- **0b. Temporary block** — Cloudflare IP Access Rule / WAF custom rule for + `195.178.110.105` + exploit-path signature (dashboard now, codified in Phase D). + +### Phase A — Pulumi Cloudflare bootstrap (no prod effect) +- Add `pulumi-cloudflare` to `infrastructure/requirements.txt`. +- Token as **Pulumi config secret**: `pulumi config set --secret cloudflare:apiToken `. +- Add `CLOUDFLARE_ZONE_ID`, `CLOUDFLARE_ACCOUNT_ID` to `config.py`. +- New module `infrastructure/modules/edge_security.py` (all CF resources; nothing + proxied yet). `pulumi preview` must show only *additive* resources. + +### Phase B — Staging validation (the core de-risking; no prod effect) +Prove the entire edge stack on a throwaway host pointing at the **same** service. +1. Add **staging domain mapping** `api-edge-test.nomadkaraoke.com → karaoke-backend`. +2. Add staging **DNS record** grey-cloud (`proxied=False`) → `ghs.googlehosted.com`; + **wait for the Cloud Run managed cert to provision** (proves ACME path). +3. Flip staging DNS to **orange-cloud** with **Full (strict)**; run the + verification suite (§6) against `api-edge-test`: + - `cf-ray` present, 200 on `/`, real endpoints work. + - Exploit paths return **403 from Cloudflare** (not 404 from origin). + - Rate-limit rule trips under a burst; `/api/internal/*` excluded. + - Origin-secret-header: requests via CF pass; direct-to-`*.run.app` gets 403 + once middleware is in enforce mode. + - No TLS errors; if 525/handshake issues → step down the SSL ladder (§4) and + re-test until green. +4. **Soak:** leave staging proxied a few days; watch for delayed cert/525 issues. +5. Tear down staging mapping/record after cutover (documented). + +### Phase C — Backend origin-lockdown middleware (deploy dormant) +- New `backend/middleware/edge_auth.py` (`BaseHTTPMiddleware`), added after CORS. +- Modes via env: `EDGE_AUTH_MODE=off|warn|enforce` (default `off`), secret from + `EDGE_ORIGIN_SECRET` (Secret Manager). In `warn` it logs missing/invalid header + but allows; in `enforce` it 403s. **Exempt** Cloud Run health/startup probes and + keep it a no-op when `off`, so deploying it changes nothing until we flip modes. +- Unit tests for off/warn/enforce + exempt paths + tenant-host requests. +- Deploy to prod in `off`/`warn` — zero behavior change — before any DNS flip. + +### Phase D — WAF / rate-limit / bot rules (staging-scoped, then global) +`cloudflare.Ruleset` resources, initially matched to the **staging host only**, +promoted to zone-wide after staging passes: +1. **Custom firewall** (`http_request_firewall_custom`): block exploit paths + (`\.(env|git|aws)|/\.ssh/|wp-(config|login|admin)|/etc/|/proc/|configuration\.php|\.(mysql|bash)_history`). +2. **Rate limiting** (`http_ratelimit`): >~60 req/min per IP → block; exclude + `/api/internal/*` and known tenant hosts. +3. **Managed ruleset** (`http_request_firewall_managed`): Cloudflare core managed + rules in **log** mode → review → `block`. +4. **Bot Fight Mode** + zone Security Level ≥ Medium. + +### Phase E — Alert retune (`infrastructure/modules/monitoring.py`, ~72-105) +Current: counts all non-2xx, `threshold=0.1`, no volume floor. +- Scope to **`response_code_class="5xx"`** (real server errors), or exclude 4xx. +- Add a **minimum-volume gate** (e.g. AND a condition on absolute 5xx count > N/5m) + so a few requests can't trip the ratio. +- Update/add regression test if alert tests exist. + +### Phase F — Prod cutover (two isolated steps; instant rollback) +Preconditions: Phases B–E green; middleware live in `warn` in prod. The prod +edge rules and the DNS proxy flip are **separate** switches so we never change +both at once (per CodeRabbit review): +1. **Provision prod rules, still DNS-only** (non-disruptive): `edge:rolloutStage=prod` + + `pulumi up`. Prod WAF/rate-limit/header rules now exist; `api` still resolves + direct to `ghs` (unchanged). +2. **Flip the DNS proxy** (the only prod-affecting change): `edge:proxyProdApi=true` + + `pulumi up` — this changes ONLY the `api` record to proxied. +3. Run the §6 verification suite against `api.nomadkaraoke.com`; watch `edge-auth` + warn logs for legit callers missing the header. +4. Rollback if needed → **`edge:proxyProdApi=false` + `pulumi up`** reverts within + one TTL (`ttl=1`/auto), leaving the edge rules provisioned. Upload flow + (signed-URL→GCS) is unaffected throughout. +5. Once stable + warn logs clean, flip middleware to `enforce`; remove the + temporary Phase-0b block (now covered by the codified rules). + +## 6. Verification suite (run against staging, then prod) +```bash +HOST=api-edge-test.nomadkaraoke.com # then api.nomadkaraoke.com at cutover +RUNAPP=karaoke-backend-ipzqd2k4yq-uc.a.run.app + +# Proxied by Cloudflare (expect server: cloudflare + cf-ray) +curl -sI https://$HOST/ | grep -iE 'server|cf-ray' + +# Exploit paths blocked at edge (expect 403 from Cloudflare, not 404) +for p in '.env' 'home/root/.ssh/id_rsa' 'wp-config.php' 'etc/passwd'; do + echo -n "$p -> "; curl -s -o /dev/null -w '%{http_code}\n' "https://$HOST/$p"; done + +# Legit control-plane still works (expect 200) +curl -s -o /dev/null -w '%{http_code}\n' https://$HOST/api/health + +# Rate limit trips under burst (expect some 429/403 after threshold) +for i in $(seq 1 120); do curl -s -o /dev/null -w '%{http_code} ' https://$HOST/api/health; done; echo + +# Origin bypass rejected once middleware enforced (expect 403) +curl -sI https://$RUNAPP/ | head -1 + +# No long (>100s) synchronous endpoints behind proxy — review slow-request logs +``` + +## 7. Files to change +1. `infrastructure/requirements.txt` — add `pulumi-cloudflare`. +2. `infrastructure/config.py` — CF zone/account ids, token secret ref. +3. `infrastructure/modules/edge_security.py` — **new**: staging+prod DnsRecord, + Rulesets (custom/rate-limit/managed), Transform Rule (secret header), staging + domain mapping, zone settings. +4. `infrastructure/__main__.py` — wire module; replace `proxied:False` export + (~665-673); staging mapping export. +5. `infrastructure/modules/monitoring.py` — retune error-rate alert (~72-105). +6. `backend/middleware/edge_auth.py` — **new** middleware + register in + `backend/main.py` after CORS; env-driven modes. +7. `backend/config` / deploy env — `EDGE_AUTH_MODE`, `EDGE_ORIGIN_SECRET`. +8. Tests: `tests/.../test_edge_auth_middleware.py`; monitoring test if present. +9. Docs: `infrastructure/README.md` + this runbook. + +## 8. Rollback summary (per phase) +- A/C/D/E deploy dormant or additive → revert = normal `pulumi up` / redeploy. +- **B** staging is throwaway → delete mapping+record, zero prod impact. +- **F** cutover → `proxied=False` + `pulumi up` (≤ one TTL). Middleware → set + `EDGE_AUTH_MODE=off`. + +## 9. Open questions +Resolved: +- ✅ **Zone id** = `807f07f458f9cd38251f3b7948d55172` (baked in). +- ✅ **Plan tier** = Free → managed WAF ruleset gated off (Pro+ only); rest works. +- ✅ **Token** — the `.envrc` `CLOUDFLARE_API_TOKEN` now has DNS/WAF/settings + scopes (verified via live create+delete). Reuse it for `cloudflare:apiToken`. +- ✅ **Existing `api` DNS record** (id `6c31cba0080ff334a85cfff6c2927219`, + proxied=False, ttl=1) — must be `pulumi import`ed, not created (runbook step 4a). + +- ✅ **Tenant subdomains** (`singa`, `vocalstar`) — live but idle Pages frontends + that call the proxied `api` host with `X-Tenant-ID`; not broken by edge-auth + (see §3). Downtime-tolerant per Andrew. + +Still open: +- Confirm **no synchronous endpoint** routinely runs >100s under proxy (checked + during staging soak via slow-request logs). +- Do we also want to protect other direct GCP origins (audio-separator, + flacfetch) as a follow-up — same exposure class. diff --git a/infrastructure/Pulumi.prod.yaml b/infrastructure/Pulumi.prod.yaml index 34d80596..61f00bbe 100644 --- a/infrastructure/Pulumi.prod.yaml +++ b/infrastructure/Pulumi.prod.yaml @@ -2,3 +2,8 @@ config: gcp:project: nomadkaraoke gcp:region: us-central1 karaoke-gen-infrastructure:backupEncryptionPubkey: 1e966a0693d4ab12f5ef570dd60b35137ec12c42cc12424e13e8c1e8e2dbc914 + edge:originSecret: + secure: AAABAPgh/fn2A5urhLq8nN0zegTSPD4073aN3P+2Jqyg+xexxpq+QMWVfZbYOxfa6WpFXLtCFfVv/TlpjL+gmJ9xOtyBdWXF+g9fu29Cf64RJxGRKLo34eMMZ3MGguKx + cloudflare:apiToken: + secure: AAABAGX6zpucCw1NvaYSCCg5YDFVuf5Thw+HL8njNfGiLoT8qpY9UhleyJy6rsnTwFo42AKUntWTK2hCAZq7VN55vBKiuHZa3NqJ37/yjpoz4EAssA== + edge:enabled: "true" diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index 073c5523..669c76f9 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -40,6 +40,7 @@ from modules import database, storage as storage_module, artifact_registry, secr from modules import cloud_tasks, cloud_run, monitoring, networking, runner_manager from modules import divebar_mirror, kn_data_sync, divebar_lookup, backup from modules import audio_separator_service +from modules import edge_security from modules import gpu_artifact_registry from modules import encoding_worker_manager from modules import error_monitor as error_monitor_module @@ -174,6 +175,13 @@ github_runner_iam_bindings = worker_sas.grant_github_runner_permissions(github_r # Domain mapping for api.nomadkaraoke.com domain_mapping = cloud_run.create_domain_mapping() +# Cloudflare edge security (WAF / rate limiting / origin lock) for the backend. +# No-op until `edge:cloudflareZoneId` + `cloudflare:apiToken` are set in Pulumi +# config. Rollout is staged via `edge:rolloutStage` (staging|cutover); the +# default "staging" is non-disruptive to the live `api` record. See +# docs/archive/2026-07-20-edge-security-hardening-plan.md. +edge_resources = edge_security.configure_edge_security() + # Cloud Run Jobs for batch processing # These use Cloud Run Jobs instead of Cloud Tasks to avoid instance termination # during long-running processing (the BackgroundTasks issue) @@ -662,16 +670,18 @@ pulumi.export("bulk_search_job", bulk_search_job.name) pulumi.export("backend_url", "https://api.nomadkaraoke.com") pulumi.export("backend_default_url", "https://karaoke-backend-ipzqd2k4yq-uc.a.run.app") -# DNS configuration for Cloudflare -domain_mapping.statuses.apply( - lambda statuses: pulumi.export("dns_records", { - "type": "CNAME", - "name": "api", - "value": "ghs.googlehosted.com", - "ttl": 300, - "proxied": False, - }) -) +# Cloudflare edge (DNS records, WAF, rate limiting, origin lock) — now managed +# in modules/edge_security.py rather than hand-configured in the dashboard. The +# `api` CNAME is proxied only when `edge:proxyProdApi=true`; otherwise it stays +# DNS-only, exactly as before. Edge rules scope to prod via `edge:rolloutStage`. +if edge_resources: + pulumi.export("edge_rollout_stage", edge_resources.get("stage")) + pulumi.export("edge_proxy_prod", edge_resources.get("proxy_prod")) + pulumi.export("edge_protected_hosts", edge_resources.get("hosts")) + pulumi.export( + "edge_staging_domain", + edge_resources["staging_domain_mapping"].name, + ) # Alert policies pulumi.export("error_rate_alert_id", alert_policies["error_rate"].name) diff --git a/infrastructure/config.py b/infrastructure/config.py index 2eeeb221..5ed70bc0 100644 --- a/infrastructure/config.py +++ b/infrastructure/config.py @@ -118,6 +118,142 @@ class SecretNames: GITHUB_RUNNER_PAT = "github-runner-pat" GITHUB_WEBHOOK_SECRET = "github-webhook-secret" + # Shared secret Cloudflare injects as a header on proxied requests; the + # backend edge-auth middleware rejects requests to public routes that lack + # it (blocks direct-to-origin bypass of the Cloudflare edge). See + # modules/edge_security.py and backend/middleware/edge_auth.py. + EDGE_ORIGIN_SECRET = "edge-origin-secret" + + +class CloudflareConfig: + """ + Configuration for the Cloudflare edge (WAF / rate limiting / origin lock). + + Non-secret values (zone/account ids, hostnames) are read from Pulumi config + so they are version-controlled per stack without hardcoding account details. + The API token is a Pulumi *secret* config under the provider key + ``cloudflare:apiToken`` (set with ``pulumi config set --secret``). + + Set the ids once known: + pulumi config set edge:cloudflareZoneId + pulumi config set edge:cloudflareAccountId + pulumi config set --secret cloudflare:apiToken # Zone WAF:Edit + DNS:Edit + """ + + # Production backend host (already a Cloud Run domain mapping today). + PROD_API_HOST = "api.nomadkaraoke.com" + # Throwaway staging host used to validate the full edge stack against the + # SAME Cloud Run service before touching the prod `api` record. Torn down + # after cutover. See docs/archive/2026-07-20-edge-security-hardening-plan.md. + STAGING_API_HOST = "api-edge-test.nomadkaraoke.com" + # Cloud Run domain mappings both target this service origin. + DOMAIN_MAPPING_ROUTE = "karaoke-backend" + ORIGIN_CNAME_TARGET = "ghs.googlehosted.com" + + # Header name Cloudflare injects and the backend checks (value = secret). + ORIGIN_AUTH_HEADER = "X-Edge-Auth" + + # Rate limit: block a client IP that exceeds this many requests/period. + # Free plan constraints (enforced by the Cloudflare API): period must be 10s, + # and mitigation_timeout must equal the period (10s). A single flood control + # rule — the WAF path block (below) is the primary defense against the + # scanner class; this catches volumetric abuse. ~50 req / 10s = 5 req/s per + # IP, comfortably above legit page-load bursts to a control-plane API. + RATE_LIMIT_REQUESTS = 50 + RATE_LIMIT_PERIOD_SECONDS = 10 + RATE_LIMIT_MITIGATION_SECONDS = 10 + + # Paths that must never be rate-limited / header-gated at the edge: + # scheduler cron hits (OIDC-authed) to /api/internal/*. + INTERNAL_PATH_PREFIX = "/api/internal/" + + @staticmethod + def managed_waf_enabled(): + """ + Whether to deploy the Cloudflare Managed (OWASP-style) Ruleset. + + The nomadkaraoke.com zone is on the **Free** plan, where managed + rulesets are NOT available (Pro+ only) — deploying one errors. So this + defaults to False. The custom exploit-path rules + rate limiting + + origin lock + bot mitigation (all Free-tier) already cover the + path-scanning threat this was built for. If the zone is upgraded to + Pro, set ``edge:managedWafEnabled true`` to turn on the OWASP ruleset. + """ + return pulumi.Config("edge").get_bool("managedWafEnabled") or False + + # Cloudflare zone id for nomadkaraoke.com (not sensitive). Overridable via + # Pulumi config `edge:cloudflareZoneId`; defaults to the known zone so the + # edge module activates as soon as a WAF/DNS-scoped `cloudflare:apiToken` + # is set. NOTE: the zone is on the **Free** plan (managed WAF ruleset is + # Pro+; see managed_waf_enabled). + DEFAULT_ZONE_ID = "807f07f458f9cd38251f3b7948d55172" + + @staticmethod + def enabled(): + """ + Master activation switch for the Cloudflare edge module + (``edge:enabled`` bool, default False). + + MUST stay False until a WAF/DNS-scoped ``cloudflare:apiToken`` is set — + otherwise `pulumi up` would try to create Cloudflare resources with an + unauthorized token and fail, blocking all infra deploys. Flip to True + only after the token + config are in place (see the cutover runbook). + """ + return pulumi.Config("edge").get_bool("enabled") or False + + @staticmethod + def zone_id(): + """Cloudflare zone id for nomadkaraoke.com.""" + return pulumi.Config("edge").get("cloudflareZoneId") or CloudflareConfig.DEFAULT_ZONE_ID + + @staticmethod + def account_id(): + """Cloudflare account id (from Pulumi config).""" + return pulumi.Config("edge").get("cloudflareAccountId") + + @staticmethod + def rollout_stage(): + """ + Which hosts the edge WAF/rate-limit/header rules apply to (from Pulumi + config ``edge:rolloutStage``): + - "staging" : rules scoped to the staging host only. + - "prod" : rules apply to the prod host too (staging kept for soak). + Defaults to "staging" so a first apply is non-disruptive to prod. + + NOTE: this does NOT flip the prod DNS proxy — that's a SEPARATE switch + (``proxy_prod_api``), so production edge rules can be provisioned and + verified while the API is still DNS-only, then the proxy flipped in an + isolated second apply (and rolled back without tearing down the rules). + """ + stage = pulumi.Config("edge").get("rolloutStage") or "staging" + return "prod" if stage in ("prod", "cutover") else "staging" + + @staticmethod + def proxy_staging_api(): + """ + Whether the staging ``api-edge-test`` record is proxied (``edge:proxyStaging`` + bool, default False). + + THE SSL DRAGON: a Cloud Run domain mapping provisions its Google-managed + cert via ACME, which needs the hostname to resolve DIRECTLY to + ``ghs.googlehosted.com``. While the record is proxied (orange-cloud) DNS + resolves to Cloudflare, so ACME can't validate and the cert never issues. + Sequence: create grey (proxied=False) → wait for cert → set this True. + """ + return pulumi.Config("edge").get_bool("proxyStaging") or False + + @staticmethod + def proxy_prod_api(): + """ + Whether the prod ``api`` DNS record is proxied through Cloudflare + (``edge:proxyProdApi`` bool, default False). + + This is the ONE prod-affecting, instantly-reversible flip. Sequence: + 1. rolloutStage=prod, apply → prod edge rules live; api still DNS-only. + 2. proxyProdApi=true, apply → api proxied (only this record changes). + Rollback = proxyProdApi=false, apply (edge rules stay provisioned). + """ + return pulumi.Config("edge").get_bool("proxyProdApi") or False # GitHub repository for runner registration GITHUB_REPO_OWNER = "nomadkaraoke" diff --git a/infrastructure/modules/edge_security.py b/infrastructure/modules/edge_security.py new file mode 100644 index 00000000..7a0e2fcd --- /dev/null +++ b/infrastructure/modules/edge_security.py @@ -0,0 +1,412 @@ +""" +Cloudflare edge security for the karaoke backend. + +Puts Cloudflare's WAF, rate limiting and bot mitigation in front of +``api.nomadkaraoke.com`` (Cloud Run) and locks the origin so it only accepts +traffic that came through Cloudflare (via a secret header the edge injects). + +WHY THIS EXISTS + The backend was reachable directly at ``ghs.googlehosted.com`` (Cloud Run + domain mapping, DNS-only / grey-cloud) with no WAF — a vuln scanner hammered + it with ``/.env`` / ``/.ssh/id_rsa`` style probes on 2026-07-20. This module + moves the API behind the Cloudflare edge that already fronts the rest of the + domain. + +ZERO-DOWNTIME ROLLOUT (see docs/archive/2026-07-20-edge-security-hardening-plan.md) + Driven by TWO independent switches so provisioning and the DNS flip are + separate (``CloudflareConfig.rollout_stage()`` + ``.proxy_prod_api()``): + + rolloutStage="staging" (default, NON-disruptive): + * adds a *second* Cloud Run domain mapping + DNS record for + ``api-edge-test.nomadkaraoke.com`` (proxied) → same karaoke-backend + * edge rules (WAF / rate-limit / header inject) are scoped to the + staging host ONLY + * the prod ``api`` record is left exactly as today (DNS-only) + → validate the whole Cloud-Run-×-Cloudflare interaction (incl. the + managed-cert / SSL "dragon") without any prod impact. + + rolloutStage="prod" (still non-disruptive): + * edge rules additionally apply to the prod host, but the prod ``api`` + record is STILL DNS-only (proxied stays False). + → provision + verify prod WAF/header rules before touching DNS. + + proxyProdApi=true (the single prod-affecting change, instantly reversible): + * flips ONLY the prod ``api`` record to proxied=True. + * requires rolloutStage=prod first (guarded) so the header transform + + rules cover the prod host before traffic is routed through the edge. + → rollback = proxyProdApi=false + `pulumi up` (edge rules stay in place). + +PROVIDER + pulumi-cloudflare 6.x (see infrastructure/requirements.txt). Resource arg + shapes were introspected against 6.18.0. Run ``pulumi preview`` after any + provider bump — Cloudflare restructures ruleset schemas between majors. + +CREDENTIALS / CONFIG (per stack, not committed) + pulumi config set --secret cloudflare:apiToken # Zone WAF:Edit + DNS:Edit + Zone Settings:Edit + pulumi config set edge:cloudflareZoneId + pulumi config set edge:cloudflareAccountId + pulumi config set edge:rolloutStage staging # then later: cutover +""" + +from typing import Optional + +import pulumi +import pulumi_cloudflare as cloudflare +import pulumi_gcp as gcp +from pulumi_gcp import cloudrun + +from config import CloudflareConfig, REGION, PROJECT_ID + + +# Cloudflare's account-level "Cloudflare Managed Ruleset" (OWASP-style core +# protections). This id is stable across all Cloudflare accounts. +CLOUDFLARE_MANAGED_RULESET_ID = "efb7b8c949ac4650a09736fc376e9aee" + +# Exploit / secret-exposure path substrings to hard-block at the edge regardless +# of source (2026-07-20 scanner signature + common web probes). We use the +# `contains` operator (not regex `matches`) because the regex operator requires a +# Business/WAF-Advanced plan; `contains` works on Free. Scanners use lowercase +# paths, matching these lowercase tokens. +EXPLOIT_PATH_TOKENS = [ + "/.env", + "/.git/", + "/.aws/", + "/.ssh/", + "/etc/passwd", + "/etc/shadow", + "/etc/hosts", + "/proc/self", + "/proc/net", + "/var/log/", + "wp-config", + "wp-login", + "wp-admin", + "configuration.php", + ".mysql_history", + ".bash_history", +] + + +def _exploit_path_expression(host_expr: str) -> str: + """Free-plan-safe firewall expression: host match AND any exploit substring.""" + ors = " or ".join( + f'http.request.uri.path contains "{t}"' for t in EXPLOIT_PATH_TOKENS + ) + return f"{host_expr} and ({ors})" + + +def _hosts_for_stage() -> list[str]: + """Which hostnames the edge rules apply to, per rollout stage.""" + if CloudflareConfig.rollout_stage() == "prod": + return [CloudflareConfig.PROD_API_HOST, CloudflareConfig.STAGING_API_HOST] + return [CloudflareConfig.STAGING_API_HOST] + + +def _host_match_expression(hosts: list[str]) -> str: + """Cloudflare filter expression matching any of the given hosts.""" + quoted = " ".join(f'"{h}"' for h in hosts) + return f"(http.host in {{{quoted}}})" + + +# --------------------------------------------------------------------------- # +# Staging Cloud Run domain mapping (same service as prod api) +# --------------------------------------------------------------------------- # +def create_staging_domain_mapping() -> cloudrun.DomainMapping: + """ + Second Cloud Run domain mapping so ``api-edge-test.nomadkaraoke.com`` serves + the SAME karaoke-backend service as prod. Lets us validate the full edge + stack end-to-end without touching the live ``api`` mapping. Torn down after + cutover (delete this resource + its DNS record, then `pulumi up`). + """ + return cloudrun.DomainMapping( + "karaoke-backend-edge-test-domain", + location=REGION, + name=CloudflareConfig.STAGING_API_HOST, + metadata=cloudrun.DomainMappingMetadataArgs(namespace=PROJECT_ID), + spec=cloudrun.DomainMappingSpecArgs( + route_name=CloudflareConfig.DOMAIN_MAPPING_ROUTE, + ), + ) + + +# --------------------------------------------------------------------------- # +# DNS records (Cloudflare) +# --------------------------------------------------------------------------- # +def create_dns_records(zone_id: str) -> dict[str, cloudflare.DnsRecord]: + """ + Manage the ``api`` (prod) and ``api-edge-test`` (staging) CNAMEs. + + - staging: always proxied=True (that's the whole point — test the proxy). + - prod ``api``: proxied only once ``proxy_prod_api()`` is true; until then it + stays DNS-only, byte-for-byte the behaviour we have today. + + NOTE: ``ttl`` must be 1 (automatic) whenever proxied=True — Cloudflare + rejects an explicit TTL on a proxied record. + """ + records: dict[str, cloudflare.DnsRecord] = {} + proxy_prod = CloudflareConfig.proxy_prod_api() + + # Created grey-cloud (proxied=False) so the Cloud Run managed cert can + # provision via ACME; flip to proxied=True (edge:proxyStaging) once the cert + # is issued. See CloudflareConfig.proxy_staging_api (the SSL "dragon"). + records["staging"] = cloudflare.DnsRecord( + "api-edge-test-dns", + zone_id=zone_id, + name="api-edge-test", + type="CNAME", + content=CloudflareConfig.ORIGIN_CNAME_TARGET, + ttl=1, + proxied=CloudflareConfig.proxy_staging_api(), + comment="Edge-security staging host → karaoke-backend (throwaway; see edge_security.py)", + ) + + # The prod `api` record is only brought under Pulumi management once we're + # provisioning for prod (rollout_stage="prod") — during staging validation + # we leave the live record completely untouched (no import needed yet). + if CloudflareConfig.rollout_stage() != "prod": + return records + + # The prod `api` record ALREADY EXISTS in Cloudflare (id + # 6c31cba0080ff334a85cfff6c2927219, ttl=1/auto, proxied=False). Pulumi must + # IMPORT it into this resource before the first apply, otherwise it tries to + # create a duplicate / errors. Runbook has the import command. ttl is kept at + # 1 (auto) to match the live record and because a proxied record requires it. + # `proxied` is gated on its OWN switch (proxy_prod_api), independent of + # rollout_stage, so prod edge rules can be provisioned first and the proxy + # flip done (and rolled back) as an isolated one-record change. + records["prod"] = cloudflare.DnsRecord( + "api-dns", + zone_id=zone_id, + name="api", + type="CNAME", + content=CloudflareConfig.ORIGIN_CNAME_TARGET, + ttl=1, + proxied=proxy_prod, + comment="karaoke-backend (Cloud Run). Proxied via Cloudflare when proxyProdApi=true.", + ) + + return records + + +# --------------------------------------------------------------------------- # +# WAF: custom exploit-path block + Cloudflare Managed Ruleset +# --------------------------------------------------------------------------- # +def create_waf_rulesets(zone_id: str, hosts: list[str]) -> dict[str, cloudflare.Ruleset]: + host_expr = _host_match_expression(hosts) + rulesets: dict[str, cloudflare.Ruleset] = {} + + # Custom firewall rules (evaluated before managed rules). + rulesets["custom"] = cloudflare.Ruleset( + "edge-waf-custom", + zone_id=zone_id, + name="karaoke-backend edge WAF (custom)", + description="Block secret-exposure / exploit path probes at the edge.", + kind="zone", + phase="http_request_firewall_custom", + rules=[ + cloudflare.RulesetRuleArgs( + action="block", + description="Block exploit / secret-exposure paths", + expression=_exploit_path_expression(host_expr), + enabled=True, + ), + ], + ) + + # Cloudflare Managed Ruleset (OWASP-style). Only available on Pro+ plans — + # the zone is currently Free, so this is gated off by default (see + # CloudflareConfig.managed_waf_enabled). Deploy in LOG mode first so we can + # review false positives before switching the override action to "block". + if CloudflareConfig.managed_waf_enabled(): + _managed_action = "log" + rulesets["managed"] = cloudflare.Ruleset( + "edge-waf-managed", + zone_id=zone_id, + name="karaoke-backend edge WAF (managed)", + description="Cloudflare Managed Ruleset (log first, then block).", + kind="zone", + phase="http_request_firewall_managed", + rules=[ + cloudflare.RulesetRuleArgs( + action="execute", + description="Execute Cloudflare Managed Ruleset", + expression=host_expr, + enabled=True, + action_parameters=cloudflare.RulesetRuleActionParametersArgs( + id=CLOUDFLARE_MANAGED_RULESET_ID, + overrides=cloudflare.RulesetRuleActionParametersOverridesArgs( + action=_managed_action, + ), + ), + ), + ], + ) + + return rulesets + + +# --------------------------------------------------------------------------- # +# Rate limiting (per-IP), excluding scheduler cron paths +# --------------------------------------------------------------------------- # +def create_rate_limit_ruleset(zone_id: str, hosts: list[str]) -> cloudflare.Ruleset: + host_expr = _host_match_expression(hosts) + # Don't rate-limit OIDC-authed scheduler hits to /api/internal/*. + expression = ( + f'{host_expr} and ' + f'(not starts_with(http.request.uri.path, "{CloudflareConfig.INTERNAL_PATH_PREFIX}"))' + ) + return cloudflare.Ruleset( + "edge-ratelimit", + zone_id=zone_id, + name="karaoke-backend edge rate limit", + description="Throttle abusive clients by IP; exclude internal cron paths.", + kind="zone", + phase="http_ratelimit", + rules=[ + cloudflare.RulesetRuleArgs( + action="block", + description=( + f"Block >{CloudflareConfig.RATE_LIMIT_REQUESTS} req / " + f"{CloudflareConfig.RATE_LIMIT_PERIOD_SECONDS}s per IP" + ), + expression=expression, + enabled=True, + ratelimit=cloudflare.RulesetRuleRatelimitArgs( + # cf.colo.id is REQUIRED by Cloudflare (counting is processed + # per-colocation); ip.src makes it per source IP. + characteristics=["ip.src", "cf.colo.id"], + period=CloudflareConfig.RATE_LIMIT_PERIOD_SECONDS, + requests_per_period=CloudflareConfig.RATE_LIMIT_REQUESTS, + mitigation_timeout=CloudflareConfig.RATE_LIMIT_MITIGATION_SECONDS, + ), + ), + ], + ) + + +# --------------------------------------------------------------------------- # +# Origin lockdown: inject a secret header on every proxied request +# --------------------------------------------------------------------------- # +def create_origin_header_transform( + zone_id: str, hosts: list[str], secret: pulumi.Input[str] +) -> cloudflare.Ruleset: + """ + Add ``X-Edge-Auth: `` to requests as they leave the Cloudflare edge + for the origin. The backend edge-auth middleware rejects public requests + that lack it — so anyone hitting the raw ``*.run.app`` / ghs origin directly + (bypassing Cloudflare) is refused. A Cloud Run domain mapping can't use + ingress=internal, so this app-layer check is how we lock the origin. + """ + host_expr = _host_match_expression(hosts) + return cloudflare.Ruleset( + "edge-origin-header", + zone_id=zone_id, + name="karaoke-backend origin auth header", + description="Inject shared-secret header so origin can reject non-edge traffic.", + kind="zone", + phase="http_request_late_transform", + rules=[ + cloudflare.RulesetRuleArgs( + action="rewrite", + description="Set origin auth header", + expression=host_expr, + enabled=True, + action_parameters=cloudflare.RulesetRuleActionParametersArgs( + headers={ + CloudflareConfig.ORIGIN_AUTH_HEADER: cloudflare.RulesetRuleActionParametersHeadersArgs( + operation="set", + value=secret, + ), + }, + ), + ), + ], + ) + + +# --------------------------------------------------------------------------- # +# Zone-level bot mitigation / security level +# --------------------------------------------------------------------------- # +def create_zone_security_settings(zone_id: str) -> dict[str, cloudflare.ZoneSetting]: + """ + Raise baseline zone protections. These are zone-wide (Cloudflare doesn't + scope Security Level / Bot Fight Mode per-host), so they also benefit the + marketing site and other subdomains. + """ + settings: dict[str, cloudflare.ZoneSetting] = {} + settings["security_level"] = cloudflare.ZoneSetting( + "zone-security-level", + zone_id=zone_id, + setting_id="security_level", + value="medium", + ) + return settings + + +# --------------------------------------------------------------------------- # +# Orchestrator +# --------------------------------------------------------------------------- # +def configure_edge_security() -> dict: + """ + Wire up the whole Cloudflare edge for the backend, honouring the rollout + stage. Returns a dict of created resources for export/inspection. + + Safe no-op until explicitly enabled: returns ``{}`` and creates nothing + unless ``edge:enabled`` is True. This MUST gate on the opt-in flag (not the + zone id, which has a default) so that merging the module before a + WAF/DNS-scoped ``cloudflare:apiToken`` exists can't make `pulumi up` try to + create Cloudflare resources with an unauthorized token. + + The origin-lock header value comes from the Pulumi secret config + ``edge:originSecret`` (same string stored in Secret Manager as + ``edge-origin-secret`` for the backend to read). If it isn't set, the WAF / + rate-limit / DNS resources are still created but the origin-header transform + is skipped (so the backend can stay in EDGE_AUTH_MODE=off/warn safely). + """ + if not CloudflareConfig.enabled(): + pulumi.log.info( + "edge_security: edge:enabled is false — Cloudflare edge resources " + "not created. Set a WAF/DNS-scoped cloudflare:apiToken, then " + "`pulumi config set edge:enabled true` to activate." + ) + return {} + + zone_id = CloudflareConfig.zone_id() + # Guard the cutover ordering: proxying prod before the prod host is covered + # by the edge rules (esp. the X-Edge-Auth header transform) would send + # header-less requests to the origin — enforce mode would then 403 all prod + # traffic. Require rollout_stage=prod (rules provisioned + verified) first. + if CloudflareConfig.proxy_prod_api() and CloudflareConfig.rollout_stage() != "prod": + raise Exception( + "edge_security: edge:proxyProdApi=true requires edge:rolloutStage=prod " + "first (so prod-host WAF + X-Edge-Auth header rules exist before the " + "DNS proxy flip). Set rolloutStage=prod, apply, verify, THEN proxyProdApi=true." + ) + + hosts = _hosts_for_stage() + resources: dict = { + "stage": CloudflareConfig.rollout_stage(), + "proxy_prod": CloudflareConfig.proxy_prod_api(), + "hosts": hosts, + } + + # Staging domain mapping only needs to exist while we're validating; keep it + # through cutover for soak, remove later. + resources["staging_domain_mapping"] = create_staging_domain_mapping() + resources["dns"] = create_dns_records(zone_id) + resources["waf"] = create_waf_rulesets(zone_id, hosts) + resources["rate_limit"] = create_rate_limit_ruleset(zone_id, hosts) + resources["zone_settings"] = create_zone_security_settings(zone_id) + + origin_secret = pulumi.Config("edge").get_secret("originSecret") + if origin_secret is not None: + resources["origin_header"] = create_origin_header_transform( + zone_id, hosts, origin_secret + ) + else: + pulumi.log.warn( + "edge_security: edge:originSecret not set — origin-header transform " + "skipped. Keep backend EDGE_AUTH_MODE=off/warn until it's set." + ) + + return resources diff --git a/infrastructure/modules/monitoring.py b/infrastructure/modules/monitoring.py index 2287f439..7b5b68d2 100644 --- a/infrastructure/modules/monitoring.py +++ b/infrastructure/modules/monitoring.py @@ -68,16 +68,30 @@ def create_alert_policies( notification_channels = _channel_names(channels) alerts = {} - # Alert: High Error Rate (>10% of requests returning errors) + # Alert: Sustained server errors (5xx). + # + # NOTE on semantics: with ALIGN_RATE + REDUCE_SUM the threshold is a request + # *rate* (requests/second), NOT a true percentage — the old "Error rate > + # 10%" name was a misnomer. It also matched ALL non-2xx codes, so a burst of + # 404s from an internet vuln-scanner (2026-07-20: 0.167 req/s of 404s from a + # single IP hammering /.env, /.ssh/... ) tripped it despite the service being + # perfectly healthy. + # + # Fix: scope to `response_code_class="5xx"` so only genuine *server* errors + # count — client/bot 404s never page us. WAF/rate-limiting at the Cloudflare + # edge (see modules/edge_security.py) is the primary defense that keeps that + # scan traffic off the metric entirely. Threshold = >0.1 5xx req/s sustained + # over 5 min. With COMPARISON_GT the boundary is strict: 6 errors/min + # (=0.1/s) does NOT fire; >=7/min sustained for 5 min does → real regression. alerts["error_rate"] = gcp.monitoring.AlertPolicy( "high-error-rate-alert", - display_name="Karaoke Backend - High Error Rate", + display_name="Karaoke Backend - High Server-Error Rate (5xx)", combiner="OR", conditions=[ gcp.monitoring.AlertPolicyConditionArgs( - display_name="Error rate > 10%", + display_name="Sustained 5xx server errors > 0.1/s (5 min)", condition_threshold=gcp.monitoring.AlertPolicyConditionConditionThresholdArgs( - filter='resource.type="cloud_run_revision" AND resource.labels.service_name="karaoke-backend" AND metric.type="run.googleapis.com/request_count" AND metric.labels.response_code_class!="2xx"', + filter='resource.type="cloud_run_revision" AND resource.labels.service_name="karaoke-backend" AND metric.type="run.googleapis.com/request_count" AND metric.labels.response_code_class="5xx"', comparison="COMPARISON_GT", threshold_value=0.1, duration="300s", @@ -98,7 +112,7 @@ def create_alert_policies( auto_close="3600s", ), documentation=gcp.monitoring.AlertPolicyDocumentationArgs( - content="High error rate detected on karaoke-backend. Check Cloud Run logs for details.\n\nDashboard: https://console.cloud.google.com/run/detail/us-central1/karaoke-backend/logs?project=nomadkaraoke", + content="Sustained 5xx server errors on karaoke-backend (not client/bot 404s). Check Cloud Run logs for crashes, timeouts, or dependency failures.\n\nDashboard: https://console.cloud.google.com/run/detail/us-central1/karaoke-backend/logs?project=nomadkaraoke", mime_type="text/markdown", ), enabled=True, diff --git a/infrastructure/modules/secrets.py b/infrastructure/modules/secrets.py index a25de819..442d7120 100644 --- a/infrastructure/modules/secrets.py +++ b/infrastructure/modules/secrets.py @@ -74,6 +74,14 @@ def create_secrets() -> dict[str, secretmanager.Secret]: # an otherwise-public endpoint that re-runs the mirror/sync/xref # scheduler jobs on demand, so it's anti-abuse, not data-sensitive. "divebar-refresh-token", + + # Cloudflare edge origin lock: shared secret Cloudflare injects as the + # X-Edge-Auth header on proxied requests; the backend edge-auth + # middleware rejects public requests that lack it (blocks direct-to- + # origin bypass of the WAF). Value added manually; also provided to + # Pulumi as `edge:originSecret` for the Cloudflare header transform. + # See modules/edge_security.py. + "edge-origin-secret", ] secrets = {} diff --git a/infrastructure/requirements.txt b/infrastructure/requirements.txt index e4ec0f03..e367c4a0 100644 --- a/infrastructure/requirements.txt +++ b/infrastructure/requirements.txt @@ -5,3 +5,8 @@ pulumi>=3.0.0,<4.0.0 # REPLACEMENT of the encoding-worker VMs on `pulumi up`. Bump this deliberately # (and handle any VM replacement in a maintenance window) — never let it float. pulumi-gcp==9.22.0 +# Cloudflare edge (WAF / rate limiting / origin lock) for api.nomadkaraoke.com. +# Pin exactly — Cloudflare restructures ruleset schemas between majors; bump +# deliberately and re-run `pulumi preview`. Resource arg shapes in +# modules/edge_security.py were introspected against 6.18.0. +pulumi-cloudflare==6.18.0 diff --git a/pyproject.toml b/pyproject.toml index 4cb5e227..33d9652e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.191.3" +version = "0.192.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"