Files
karaoke-gen/backend/main.py
T
Andrew Beveridge ☄️andClaude Opus 5.5 290f927175 fix(backend): move remaining event-loop blockers off the loop (v0.278.7) (#1185)
## Summary
Found while triaging Discord #alerts and 14 days of freeze telemetry
(2026-10-18). The event-loop watchdog still caught real freezes that hit
users:

- **151s freeze, 32 requests affected (2026-10-17 20:10 UTC).** `POST
/api/review/{id}/custom-lyrics/generate` called
`CustomLyricsService.generate` (a sync Gemini call) on the loop. Every
user on the instance saw "server unavailable". It now runs via
`asyncio.to_thread`.
- **14.9s freeze, 7 requests (2026-10-16).** The first `from
google.cloud import run_v2` in
`worker_service._trigger_worker_cloud_run_job` ran on the loop.
`google.cloud.run_v2` and `httpcore` (imported lazily by httpx) are now
in the startup `_LAZY_ROUTE_IMPORTS`, so they are imported off-loop
before readiness.
- **GCS on the loop in async routes.** `_prepare_theme_for_job` ran on
the loop in `upload_and_create_job`, `create_job_with_upload_urls`,
`create_job_from_url` and audio-search `search_audio`;
`submit_audio_edit` also uploaded `edit_log.json` on the loop. These now
go through `to_thread`.
- **Guard.** `upload_json`, `_prepare_theme_for_job` and `generate` are
added to the `BLOCKING_CALLS` denylist in
`test_no_blocking_async_routes.py`. The denylist caught the two extra
routes above.

## Testing
- `pytest backend/tests --ignore=backend/tests/emulator`: 6418 passed,
36 skipped.
- `test_audio_search` source check updated to accept the
`to_thread(_prepare_theme_for_job, …)` form.
- Local CodeRabbit review: 0 findings.

## Verify after deploy
No new `EVENT_LOOP_STALL ended … requests_affected>0` lines with
culprits `_call_gemini`, `run_v2` or `upload_json`.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_019pmG2XAgtUguEJWLFpkKHN

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 01:56:13 +00:00

394 lines
17 KiB
Python

"""
FastAPI application entry point for karaoke generation backend.
(Trivial comment touch to trigger backend CI on ephemeral runners after
the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.)
"""
import asyncio
import logging
import threading
import time
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from backend.config import settings, validate_production_config
from backend.services.fake_workers import fake_workers_enabled
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, tenant_admin, tenant_bulk, tenant_theme, rate_limits, push, catalog, encoding_worker, client_errors, client_events, bulk, parse_titles
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
from backend.services.structured_logging import setup_structured_logging
from backend.services.spacy_preloader import preload_spacy_model
from backend.services.nltk_preloader import preload_all_nltk_resources
from backend.services.langfuse_preloader import preload_langfuse_handler
from backend.middleware.audit_logging import AuditLoggingMiddleware
from backend.middleware.tenant import TenantMiddleware
from backend.middleware.edge_auth import EdgeAuthMiddleware
from backend.workers.registry import worker_registry
from backend.version import VERSION
# Configure structured logging (JSON in Cloud Run, human-readable locally)
# This must happen before any logging calls
setup_structured_logging()
logger = logging.getLogger(__name__)
# Initialize OpenTelemetry tracing (must happen before app creation)
tracing_enabled = setup_tracing(
service_name="karaoke-backend",
service_version=VERSION,
enable_in_dev=False, # Set to True to enable tracing locally
)
def validate_credentials_on_startup():
"""Validate OAuth credentials on startup and send alerts if needed."""
try:
from backend.services.credential_manager import get_credential_manager, CredentialStatus
manager = get_credential_manager()
results = manager.check_all_credentials()
invalid_services = [
result for result in results.values()
if result.status in (CredentialStatus.INVALID, CredentialStatus.EXPIRED)
]
if invalid_services:
logger.warning(f"Some OAuth credentials need attention:")
for result in invalid_services:
logger.warning(f" - {result.service}: {result.message}")
# Try to send Discord alert
discord_url = settings.get_secret("discord-alert-webhook") if hasattr(settings, 'get_secret') else None
if discord_url:
manager.send_credential_alert(invalid_services, discord_url)
logger.info("Sent credential alert to Discord")
else:
logger.info("All OAuth credentials validated successfully")
except Exception as e:
logger.error(f"Failed to validate credentials on startup: {e}")
# Modules that route handlers import lazily. On a fresh instance the first such
# import ran ON the event loop: httpx/httpcore (via flacfetch_client /
# worker_service) froze it 3-15s on cold disk reads (EVENT_LOOP_STALL in
# importlib, 2026-10-08). Lifespan startup imports them in a worker thread before
# readiness — a background thread alone would still race the first request.
_LAZY_ROUTE_IMPORTS = (
"httpx",
# Imported lazily by httpx / worker_service on first use; still froze the loop
# after the list above existed (httpcore ~1s; google.cloud.run_v2 14.9s with
# 7 requests in flight, 2026-10-16).
"httpcore",
"google.cloud.run_v2",
"backend.services.worker_service",
"backend.services.flacfetch_client",
"backend.services.catalog_proxy_service",
"backend.services.ip_geolocation_service",
"backend.services.email_validation_service",
"backend.services.musicbrainz_service",
)
LAZY_ROUTE_IMPORT_TIMEOUT_S = 30
def _import_lazy_route_modules() -> None:
import importlib
for module in _LAZY_ROUTE_IMPORTS:
try:
importlib.import_module(module)
except Exception as e:
logger.warning(f"Startup import of {module} failed (will import lazily): {e}")
def _run_background_warmup():
"""Warm caches that used to block startup (runs in a daemon thread).
Historically these preloads ran inline in lifespan startup, which meant
Cloud Run held every routed request for the full ~7.5s they take (spaCy
~1s, NLTK ~2.5s, Langfuse ~3s, credential checks ~1.5s) on top of import
time. None of them are needed to serve HTTP: they exist to make the FIRST
lyrics-processing job fast (see docs/archive/2026-01-08-performance-
investigation.md). Running them in a background thread right after boot
keeps that warm-cache property (they finish within seconds, long before
any real job runs) without gating readiness. Workers that race the warmup
simply fall back to the preloaders' lazy paths.
"""
warmup_start = time.time()
# 1. SpaCy model (60+ second delay without preload)
try:
preload_spacy_model("en_core_web_sm")
except Exception as e:
logger.warning(f"SpaCy preload failed (will load lazily): {e}")
# 2. NLTK cmudict (50-100+ second delay without preload)
try:
preload_all_nltk_resources()
except Exception as e:
logger.warning(f"NLTK preload failed (will load lazily): {e}")
# 3. Langfuse callback handler (200+ second delay without preload)
try:
preload_langfuse_handler()
except Exception as e:
logger.warning(f"Langfuse preload failed (will initialize lazily): {e}")
if fake_workers_enabled(settings):
# Local stack: no encoding VM and no OAuth credentials to validate.
logger.info(f"Background warmup complete in {time.time() - warmup_start:.2f}s (fake workers)")
return
# 4. Encoding service singleton — its first construction imports
# google.cloud.compute_v1 (~10s on a fresh instance). Build it here, off the
# event loop, before /api/health/detailed or a preview encode needs it.
try:
from backend.services.encoding_service import get_encoding_service
get_encoding_service()
except Exception as e:
logger.warning(f"Encoding service warmup failed (will build lazily): {e}")
# Validate OAuth credentials (alerting only, nothing depends on it)
try:
validate_credentials_on_startup()
except Exception as e:
logger.error(f"Credential validation failed: {e}")
logger.info(f"Background warmup complete in {time.time() - warmup_start:.2f}s")
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Lifespan event handler for startup and shutdown."""
# Startup
logger.info("Starting karaoke generation backend")
logger.info(f"Environment: {settings.environment}")
logger.info(f"GCS Bucket: {settings.gcs_bucket_name}")
logger.info(f"Tracing enabled: {tracing_enabled}")
# Fail fast if required production config is missing, rather than silently
# running on dev defaults (wrong GCP project / GCS bucket / localhost worker
# URL). No-op outside production. (Fallback audit 2026-06-09, Theme 7.)
validate_production_config()
# Off the loop, but before readiness (Cloud Run holds requests meanwhile).
# Bounded so a hung cold-disk read can't keep the instance from ever going
# ready; the import keeps running in its thread and routes fall back to it.
try:
await asyncio.wait_for(
asyncio.to_thread(_import_lazy_route_modules), timeout=LAZY_ROUTE_IMPORT_TIMEOUT_S
)
except asyncio.TimeoutError:
logger.error(
f"Startup import of lazy route modules exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; "
"continuing startup (first requests may import them on the event loop)"
)
# NLP model / credential warmup runs in the background so it doesn't gate
# readiness — Cloud Run holds all routed requests until lifespan startup
# returns, and these preloads were ~7.5s of the ~16s cold start.
threading.Thread(
target=_run_background_warmup, name="startup-warmup", daemon=True
).start()
# asyncio.to_thread's default executor is min(32, cpu+4) ≈ 6 threads on our
# 2 vCPUs. The screens worker, stale-review cron, auto-correct and many
# review endpoints all offload into it (mostly I/O-bound), so size it up —
# otherwise review-page requests queue behind a few long-running jobs.
import concurrent.futures
asyncio.get_running_loop().set_default_executor(
concurrent.futures.ThreadPoolExecutor(max_workers=48, thread_name_prefix="to-thread")
)
# Event-loop stall watchdog: makes "every request froze for 30s" episodes
# self-diagnosing (logs the blocking stack; records >=5s stalls in prod).
watchdog = None
if settings.loop_watchdog_enabled:
from backend.services.loop_watchdog import (
LoopWatchdog,
firestore_stall_recorder,
request_tracker,
)
watchdog = LoopWatchdog(
recorder=firestore_stall_recorder if settings.environment == "production" else None,
# Stalls nobody waited on (idle, CPU-throttled autoscaled instances)
# stay WARNING so they don't trip the >=10s stall alert.
request_tracker=request_tracker,
)
watchdog.start()
yield
if watchdog is not None:
await watchdog.stop()
# Shutdown - best-effort parking of any still-registered workers.
#
# Reality check (incident 2026-09-13, job 41e06b90): Cloud Run services
# SIGKILL instances ~10 seconds after SIGTERM — NOT the 600s this comment
# previously claimed — and uvicorn only runs this lifespan-shutdown code
# AFTER all BackgroundTasks have completed ("Waiting for background tasks
# to complete"). A long-running worker task therefore keeps uvicorn stuck
# in that wait until SIGKILL, and this hook never executes at all. It can
# NOT be relied on to protect in-flight renders; that protection comes
# from running them as Cloud Run Jobs instead (USE_CLOUD_RUN_JOBS_FOR_RENDER,
# mirroring USE_CLOUD_RUN_JOBS_FOR_VIDEO from incident 2026-03-08).
#
# This hook is kept as a cheap safety net for the legacy flag-off path and
# for the narrow case where workers finish right as SIGTERM lands. The wait
# is capped well inside the 10s kill window so the parking pass below still
# gets a chance to write Firestore state.
logger.info("Shutdown requested, checking for active workers...")
if worker_registry.has_active_workers():
active = worker_registry.get_active_workers()
logger.info(f"Active workers found: {active}")
logger.info("Waiting for workers to complete (timeout: 5s)...")
completed = await worker_registry.wait_for_completion(timeout=5)
if not completed:
logger.error(
"Shutdown timeout - some workers may not have completed cleanly. "
f"Remaining workers: {worker_registry.get_active_workers()}"
)
# Park any still-active render jobs so the auto-retry scheduler can
# recover them. Safe to call unconditionally — it's a no-op if nothing
# render-related is in flight.
try:
from backend.workers.render_video_worker import park_active_render_jobs_for_shutdown
parked = park_active_render_jobs_for_shutdown()
if parked:
logger.warning(
f"Parked {parked} in-flight render job(s) for auto-retry "
f"due to shutdown"
)
except Exception as exc:
logger.error(f"Render-job shutdown park failed: {exc!r}")
else:
logger.info("No active workers, proceeding with shutdown")
logger.info("Shutting down karaoke generation backend")
# Create FastAPI app
app = FastAPI(
title="Karaoke Generator API",
description="Backend API for web-based karaoke video generation",
version=VERSION,
lifespan=lifespan
)
# Instrument FastAPI with OpenTelemetry (adds automatic spans for all requests)
if tracing_enabled:
instrument_app(app)
# Configure CORS
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=False,
allow_methods=["*"],
allow_headers=["*"],
)
# Add audit logging middleware (captures all requests with request_id for correlation)
app.add_middleware(AuditLoggingMiddleware)
# Add tenant detection middleware (extracts tenant from subdomain/headers)
app.add_middleware(TenantMiddleware)
# Edge origin lock (added last = outermost, runs first). Rejects direct-to-origin
# requests that bypassed the Cloudflare edge, before tenant/audit middleware do
# any work. No-op unless EDGE_AUTH_MODE is warn/enforce. See
# backend/middleware/edge_auth.py.
app.add_middleware(EdgeAuthMiddleware)
# Counts in-flight requests for the loop watchdog (outermost, so every request that
# waited on a stalled loop counts — even ones the edge lock rejects).
from backend.services.loop_watchdog import RequestTrackingMiddleware # noqa: E402
app.add_middleware(RequestTrackingMiddleware)
# Include routers
app.include_router(health.router, prefix="/api")
app.include_router(jobs.router, prefix="/api")
app.include_router(file_upload.router, prefix="/api") # File upload endpoint
app.include_router(internal.router, prefix="/api") # Internal worker endpoints
app.include_router(encoding_worker.router, prefix="/api") # Encoding worker lifecycle
app.include_router(review.router, prefix="/api") # Review UI compatibility endpoints
app.include_router(auth.router, prefix="/api") # OAuth credential management
app.include_router(audio_search.router, prefix="/api") # Audio search (artist+title mode)
app.include_router(themes.router, prefix="/api") # Theme selection for styles
app.include_router(users.router, prefix="/api") # User auth, credits, and Stripe webhooks
from backend.api.routes import referrals
app.include_router(referrals.router, prefix="/api") # Referral links and payouts
app.include_router(admin.router, prefix="/api") # Admin dashboard and management
app.include_router(rate_limits.router, prefix="/api") # Rate limits admin management
app.include_router(push.router, prefix="/api") # Push notification subscription management
app.include_router(catalog.router, prefix="/api") # Catalog proxy for song/artist autocomplete
app.include_router(parse_titles.router, prefix="/api") # kjbox karaoke-filename parser
app.include_router(bulk.router, prefix="/api") # Bulk Mode: multi-job submission
from backend.api.routes import requests_board
app.include_router(requests_board.router, prefix="/api") # Public song-request voting board
app.include_router(client_errors.router, prefix="/api") # Frontend crash reports
app.include_router(client_events.router, prefix="/api") # Frontend degradation telemetry (banner/unavailable/waveform events)
from backend.api.routes import karaokehunt
app.include_router(karaokehunt.router, prefix="/api") # Retired KaraokeHunt app request interceptor
from backend.api.routes import kjbox
app.include_router(kjbox.router, prefix="/api") # kjbox partner API: singer email-code sign-in + show credits
from backend.api.routes import login_code
app.include_router(login_code.router, prefix="/api") # gen email-code sign-in (/users/auth/send-code, verify-code)
app.include_router(tenant.router) # Tenant/white-label configuration (no /api prefix, router has it)
app.include_router(tenant_admin.router) # Admin tenant provisioning (router has /api prefix)
app.include_router(tenant_bulk.router) # Tenant bulk-upload filename analysis (router has /api prefix)
app.include_router(tenant_theme.router) # Tenant self-service theme editor (router has /api prefix)
if fake_workers_enabled(settings):
# Local E2E stack only (startup refuses FAKE_WORKERS in production).
from backend.api.routes import test_support
app.include_router(test_support.router, prefix="/api")
# Exception handlers
from fastapi import Request
from fastapi.responses import JSONResponse
from backend.exceptions import InsufficientCreditsError
from backend.i18n import t, get_locale_from_request
@app.exception_handler(InsufficientCreditsError)
async def insufficient_credits_exception_handler(request: Request, exc: InsufficientCreditsError):
"""Handle insufficient credits errors with 402 Payment Required status."""
locale = get_locale_from_request(request)
return JSONResponse(
status_code=402,
content={
"detail": t(locale, "errors.insufficientCredits"),
"credits_available": exc.credits_available,
"credits_required": exc.credits_required,
"buy_url": t(locale, "errors.insufficientCreditsBuyUrl"),
},
)
@app.get("/")
async def root():
"""Root endpoint."""
return {
"service": "karaoke-gen-backend",
"version": VERSION,
"status": "running"
}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8080)