mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Puts `api.nomadkaraoke.com` behind the Cloudflare edge that already fronts the rest of the domain, after a vuln-scanner hammered the directly-exposed Cloud Run origin on 2026-07-20 (`/.env`, `/.ssh/id_rsa`, `wp-config.php`… — all 404'd, but the burst of 404s tripped the "High Error Rate" alert). The API was the one internet-facing surface with **no WAF, no rate limiting** — DNS-only CNAME straight to `ghs.googlehosted.com`, Cloud Run ingress `all`. **Zero-downtime by construction:** everything is gated behind `edge:enabled` (default off) and staged via two independent switches (`edge:rolloutStage` for rule scope, `edge:proxyProdApi` for the DNS flip). The whole Cloud-Run×Cloudflare interaction is validated on a throwaway staging host first; the only prod-affecting action is a single, instantly-reversible DNS proxy flip. Full sequence in `docs/archive/2026-07-20-edge-security-cutover-runbook.md`. ## Changes - **`infrastructure/modules/edge_security.py`** (new) — `pulumi_cloudflare` WAF (exploit-path block), per-IP rate limiting (excludes `/api/internal/*`), origin-lock header transform, zone security; staging + prod DNS records. Managed OWASP ruleset gated off (zone is Free plan). - **`backend/middleware/edge_auth.py`** (new) — off/warn/enforce origin lock via a secret header Cloudflare injects; **fails closed (503) in enforce** when the secret is missing; health/root exempt. +13 unit tests. - **`monitoring.py`** — retune the high-error-rate alert to **5xx-only** (+ fix the rate-vs-ratio naming) so client/bot 404 bursts no longer page. - Wiring: `secrets.py` (edge-origin-secret container), `config.py`, `__main__.py`, pinned `pulumi-cloudflare==6.18.0`, `ci.yml` (`EDGE_AUTH_MODE=off` + secret ref), `Pulumi.prod.yaml` (encrypted `edge:originSecret`). - Docs: hardening plan + cutover runbook in `docs/archive/`. ## Already applied to prod (targeted `pulumi up`) - ✅ `edge-origin-secret` Secret Manager container + v1 value - ✅ 5xx error-rate alert retune (fixes the original paging issue immediately) - Everything else stays dormant: `edge:enabled=false`, backend `EDGE_AUTH_MODE=off`. ## Testing - [x] `make test` — full backend + frontend suite green (1085 frontend tests; backend incl. 13 new edge-auth tests) - [x] Cloudflare token verified against live API (DNS create+delete, rulesets, zone settings) - [x] `pulumi preview` clean (targeted apply of only the 2 additive resources) - [ ] Staging validation + prod cutover — follow the runbook (separate, gated) ## Review - [x] CodeRabbit CLI review completed locally; findings addressed (fail-closed enforce, decoupled cutover, fixed origin-lock test, threshold wording). One pre-existing PR_TITLE shell-interp finding in the deploy step left out of scope. ## Notes for reviewer - Nothing here touches prod behavior until someone sets `edge:enabled true` and walks the runbook. The prod `api` DNS record must be `pulumi import`ed (already exists) — documented in runbook step 4a. - Pre-existing cross-worktree Cloud Function source drift is visible in a full `pulumi preview`; not introduced here (applied via `--target` to avoid it). @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
239 lines
9.6 KiB
Python
239 lines
9.6 KiB
Python
"""
|
|
FastAPI application entry point for karaoke generation backend.
|
|
|
|
(Trivial comment touch to trigger backend CI on ephemeral runners after
|
|
the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.)
|
|
"""
|
|
import logging
|
|
from contextlib import asynccontextmanager
|
|
from fastapi import FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
from backend.config import settings
|
|
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, rate_limits, push, catalog, encoding_worker, client_errors, bulk, parse_titles
|
|
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
|
|
from backend.services.structured_logging import setup_structured_logging
|
|
from backend.services.spacy_preloader import preload_spacy_model
|
|
from backend.services.nltk_preloader import preload_all_nltk_resources
|
|
from backend.services.langfuse_preloader import preload_langfuse_handler
|
|
from backend.middleware.audit_logging import AuditLoggingMiddleware
|
|
from backend.middleware.tenant import TenantMiddleware
|
|
from backend.middleware.edge_auth import EdgeAuthMiddleware
|
|
from backend.workers.registry import worker_registry
|
|
|
|
|
|
from backend.version import VERSION
|
|
|
|
|
|
# Configure structured logging (JSON in Cloud Run, human-readable locally)
|
|
# This must happen before any logging calls
|
|
setup_structured_logging()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Initialize OpenTelemetry tracing (must happen before app creation)
|
|
tracing_enabled = setup_tracing(
|
|
service_name="karaoke-backend",
|
|
service_version=VERSION,
|
|
enable_in_dev=False, # Set to True to enable tracing locally
|
|
)
|
|
|
|
|
|
async def validate_credentials_on_startup():
|
|
"""Validate OAuth credentials on startup and send alerts if needed."""
|
|
try:
|
|
from backend.services.credential_manager import get_credential_manager, CredentialStatus
|
|
|
|
manager = get_credential_manager()
|
|
results = manager.check_all_credentials()
|
|
|
|
invalid_services = [
|
|
result for result in results.values()
|
|
if result.status in (CredentialStatus.INVALID, CredentialStatus.EXPIRED)
|
|
]
|
|
|
|
if invalid_services:
|
|
logger.warning(f"Some OAuth credentials need attention:")
|
|
for result in invalid_services:
|
|
logger.warning(f" - {result.service}: {result.message}")
|
|
|
|
# Try to send Discord alert
|
|
discord_url = settings.get_secret("discord-alert-webhook") if hasattr(settings, 'get_secret') else None
|
|
if discord_url:
|
|
manager.send_credential_alert(invalid_services, discord_url)
|
|
logger.info("Sent credential alert to Discord")
|
|
else:
|
|
logger.info("All OAuth credentials validated successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to validate credentials on startup: {e}")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
"""Lifespan event handler for startup and shutdown."""
|
|
# Startup
|
|
logger.info("Starting karaoke generation backend")
|
|
logger.info(f"Environment: {settings.environment}")
|
|
logger.info(f"GCS Bucket: {settings.gcs_bucket_name}")
|
|
logger.info(f"Tracing enabled: {tracing_enabled}")
|
|
|
|
# Preload NLP models and resources to avoid cold start delays
|
|
# See docs/archive/2026-01-08-performance-investigation.md for background
|
|
|
|
# 1. SpaCy model (60+ second delay without preload)
|
|
try:
|
|
preload_spacy_model("en_core_web_sm")
|
|
except Exception as e:
|
|
logger.warning(f"SpaCy preload failed (will load lazily): {e}")
|
|
|
|
# 2. NLTK cmudict (50-100+ second delay without preload)
|
|
try:
|
|
preload_all_nltk_resources()
|
|
except Exception as e:
|
|
logger.warning(f"NLTK preload failed (will load lazily): {e}")
|
|
|
|
# 3. Langfuse callback handler (200+ second delay without preload)
|
|
try:
|
|
preload_langfuse_handler()
|
|
except Exception as e:
|
|
logger.warning(f"Langfuse preload failed (will initialize lazily): {e}")
|
|
|
|
# Validate OAuth credentials (non-blocking)
|
|
try:
|
|
await validate_credentials_on_startup()
|
|
except Exception as e:
|
|
logger.error(f"Credential validation failed: {e}")
|
|
|
|
yield
|
|
|
|
# Shutdown - wait for any active workers to complete before terminating
|
|
# This prevents Cloud Run from killing workers mid-processing.
|
|
#
|
|
# Cloud Run gen2 default termination grace period is 600s. We wait up to
|
|
# 480s, leaving ~120s for the cleanup pass below to write Firestore state
|
|
# before SIGKILL. Without that headroom, render jobs that miss the wait
|
|
# would sit in `rendering_video` forever waiting for an operator.
|
|
logger.info("Shutdown requested, checking for active workers...")
|
|
if worker_registry.has_active_workers():
|
|
active = worker_registry.get_active_workers()
|
|
logger.info(f"Active workers found: {active}")
|
|
logger.info("Waiting for workers to complete (timeout: 480s)...")
|
|
completed = await worker_registry.wait_for_completion(timeout=480)
|
|
if not completed:
|
|
logger.error(
|
|
"Shutdown timeout - some workers may not have completed cleanly. "
|
|
f"Remaining workers: {worker_registry.get_active_workers()}"
|
|
)
|
|
|
|
# Park any still-active render jobs so the auto-retry scheduler can
|
|
# recover them. Safe to call unconditionally — it's a no-op if nothing
|
|
# render-related is in flight.
|
|
try:
|
|
from backend.workers.render_video_worker import park_active_render_jobs_for_shutdown
|
|
parked = park_active_render_jobs_for_shutdown()
|
|
if parked:
|
|
logger.warning(
|
|
f"Parked {parked} in-flight render job(s) for auto-retry "
|
|
f"due to shutdown"
|
|
)
|
|
except Exception as exc:
|
|
logger.error(f"Render-job shutdown park failed: {exc!r}")
|
|
else:
|
|
logger.info("No active workers, proceeding with shutdown")
|
|
|
|
logger.info("Shutting down karaoke generation backend")
|
|
|
|
|
|
# Create FastAPI app
|
|
app = FastAPI(
|
|
title="Karaoke Generator API",
|
|
description="Backend API for web-based karaoke video generation",
|
|
version=VERSION,
|
|
lifespan=lifespan
|
|
)
|
|
|
|
# Instrument FastAPI with OpenTelemetry (adds automatic spans for all requests)
|
|
if tracing_enabled:
|
|
instrument_app(app)
|
|
|
|
# Configure CORS
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=False,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Add audit logging middleware (captures all requests with request_id for correlation)
|
|
app.add_middleware(AuditLoggingMiddleware)
|
|
|
|
# Add tenant detection middleware (extracts tenant from subdomain/headers)
|
|
app.add_middleware(TenantMiddleware)
|
|
|
|
# Edge origin lock (added last = outermost, runs first). Rejects direct-to-origin
|
|
# requests that bypassed the Cloudflare edge, before tenant/audit middleware do
|
|
# any work. No-op unless EDGE_AUTH_MODE is warn/enforce. See
|
|
# backend/middleware/edge_auth.py.
|
|
app.add_middleware(EdgeAuthMiddleware)
|
|
|
|
# Include routers
|
|
app.include_router(health.router, prefix="/api")
|
|
app.include_router(jobs.router, prefix="/api")
|
|
app.include_router(file_upload.router, prefix="/api") # File upload endpoint
|
|
app.include_router(internal.router, prefix="/api") # Internal worker endpoints
|
|
app.include_router(encoding_worker.router, prefix="/api") # Encoding worker lifecycle
|
|
app.include_router(review.router, prefix="/api") # Review UI compatibility endpoints
|
|
app.include_router(auth.router, prefix="/api") # OAuth credential management
|
|
app.include_router(audio_search.router, prefix="/api") # Audio search (artist+title mode)
|
|
app.include_router(themes.router, prefix="/api") # Theme selection for styles
|
|
app.include_router(users.router, prefix="/api") # User auth, credits, and Stripe webhooks
|
|
from backend.api.routes import referrals
|
|
app.include_router(referrals.router, prefix="/api") # Referral links and payouts
|
|
app.include_router(admin.router, prefix="/api") # Admin dashboard and management
|
|
app.include_router(rate_limits.router, prefix="/api") # Rate limits admin management
|
|
app.include_router(push.router, prefix="/api") # Push notification subscription management
|
|
app.include_router(catalog.router, prefix="/api") # Catalog proxy for song/artist autocomplete
|
|
app.include_router(parse_titles.router, prefix="/api") # kjbox karaoke-filename parser
|
|
app.include_router(bulk.router, prefix="/api") # Bulk Mode: multi-job submission
|
|
app.include_router(client_errors.router, prefix="/api") # Frontend crash reports
|
|
app.include_router(tenant.router) # Tenant/white-label configuration (no /api prefix, router has it)
|
|
|
|
|
|
# Exception handlers
|
|
from fastapi import Request
|
|
from fastapi.responses import JSONResponse
|
|
from backend.exceptions import InsufficientCreditsError
|
|
from backend.i18n import t, get_locale_from_request
|
|
|
|
|
|
@app.exception_handler(InsufficientCreditsError)
|
|
async def insufficient_credits_exception_handler(request: Request, exc: InsufficientCreditsError):
|
|
"""Handle insufficient credits errors with 402 Payment Required status."""
|
|
locale = get_locale_from_request(request)
|
|
return JSONResponse(
|
|
status_code=402,
|
|
content={
|
|
"detail": t(locale, "errors.insufficientCredits"),
|
|
"credits_available": exc.credits_available,
|
|
"credits_required": exc.credits_required,
|
|
"buy_url": t(locale, "errors.insufficientCreditsBuyUrl"),
|
|
},
|
|
)
|
|
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
"""Root endpoint."""
|
|
return {
|
|
"service": "karaoke-gen-backend",
|
|
"version": VERSION,
|
|
"status": "running"
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=8080)
|
|
|