mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - **Local stack:** `scripts/local-stack.sh up --frontend` runs the **real** backend on the Firestore + GCS emulators with `FAKE_WORKERS=1`. In-process fake workers move jobs through their real states in seconds, using tiny real media, so web Playwright and mobile Maestro full-journey tests can run offline against the same endpoints. - **Email-code sign-in** (nomad-mobile spec §4.1): `POST /api/users/auth/send-code` + `verify-code`. The code logic is shared with kjbox via `LoginCodeStore`, and sends are throttled per email and per IP. - **Bug fix found by the new suite:** `confirm-duration` moved jobs to `separating_stage1`, from which they can never reach review. It now resumes at `downloading`. No prod job had taken this path yet. ## Changes - `backend/services/fake_workers/`: dispatch seam in `WorkerService.trigger_*`, plus fake search, download, checkout and outbox, scenarios, and per-user config. The real stem upload, transcode, backing-vocals analysis and preview render are reused. Every check is strict (`is True`), so a MagicMock settings object never enables the fakes, and startup **refuses FAKE_WORKERS in production**. - `/api/test/*` router (mounted only in that mode): `status`, `scenario`, `fake-config`, `emails`, `reset`, plus the fake Stripe checkout page. Paying it runs the real webhook handler. - `frontend/e2e/local-stack/` (9 specs, ~50s): - sign-up → make → review → instrumental → download - buy and cancel credits - duration confirm, failed, complete and review states - code sign-in and lockout - New required CI job `local-stack-e2e`. - Docs: TESTING.md § Local stack, DEVELOPMENT.md, API.md, README, LESSONS-LEARNED. - `MAX_SIGNUPS_PER_IP` can now be set by env. Only the local stack raises it; production keeps the default of 2. ## Testing - [x] Backend unit: 6175 passed - [x] Emulator: 152 passed - [x] Frontend Jest: 1706 passed - [x] Local-stack Playwright: 9/9 passed (CI mode) ## Review - [x] CodeRabbit CLI review completed locally. 1 minor finding declined: it suggested swallowing infrastructure errors in send-code as fake "sent" responses, which would hide real outages from users. @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01AGuxtgoscF6RJGtpjHPg3W --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
389 lines
17 KiB
Python
389 lines
17 KiB
Python
"""
|
|
FastAPI application entry point for karaoke generation backend.
|
|
|
|
(Trivial comment touch to trigger backend CI on ephemeral runners after
|
|
the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.)
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
import threading
|
|
import time
|
|
from contextlib import asynccontextmanager
|
|
from fastapi import FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
from backend.config import settings, validate_production_config
|
|
from backend.services.fake_workers import fake_workers_enabled
|
|
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, tenant_admin, tenant_bulk, tenant_theme, rate_limits, push, catalog, encoding_worker, client_errors, client_events, bulk, parse_titles
|
|
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
|
|
from backend.services.structured_logging import setup_structured_logging
|
|
from backend.services.spacy_preloader import preload_spacy_model
|
|
from backend.services.nltk_preloader import preload_all_nltk_resources
|
|
from backend.services.langfuse_preloader import preload_langfuse_handler
|
|
from backend.middleware.audit_logging import AuditLoggingMiddleware
|
|
from backend.middleware.tenant import TenantMiddleware
|
|
from backend.middleware.edge_auth import EdgeAuthMiddleware
|
|
from backend.workers.registry import worker_registry
|
|
|
|
|
|
from backend.version import VERSION
|
|
|
|
|
|
# Configure structured logging (JSON in Cloud Run, human-readable locally)
|
|
# This must happen before any logging calls
|
|
setup_structured_logging()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Initialize OpenTelemetry tracing (must happen before app creation)
|
|
tracing_enabled = setup_tracing(
|
|
service_name="karaoke-backend",
|
|
service_version=VERSION,
|
|
enable_in_dev=False, # Set to True to enable tracing locally
|
|
)
|
|
|
|
|
|
def validate_credentials_on_startup():
|
|
"""Validate OAuth credentials on startup and send alerts if needed."""
|
|
try:
|
|
from backend.services.credential_manager import get_credential_manager, CredentialStatus
|
|
|
|
manager = get_credential_manager()
|
|
results = manager.check_all_credentials()
|
|
|
|
invalid_services = [
|
|
result for result in results.values()
|
|
if result.status in (CredentialStatus.INVALID, CredentialStatus.EXPIRED)
|
|
]
|
|
|
|
if invalid_services:
|
|
logger.warning(f"Some OAuth credentials need attention:")
|
|
for result in invalid_services:
|
|
logger.warning(f" - {result.service}: {result.message}")
|
|
|
|
# Try to send Discord alert
|
|
discord_url = settings.get_secret("discord-alert-webhook") if hasattr(settings, 'get_secret') else None
|
|
if discord_url:
|
|
manager.send_credential_alert(invalid_services, discord_url)
|
|
logger.info("Sent credential alert to Discord")
|
|
else:
|
|
logger.info("All OAuth credentials validated successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to validate credentials on startup: {e}")
|
|
|
|
|
|
# Modules that route handlers import lazily. On a fresh instance the first such
|
|
# import ran ON the event loop: httpx/httpcore (via flacfetch_client /
|
|
# worker_service) froze it 3-15s on cold disk reads (EVENT_LOOP_STALL in
|
|
# importlib, 2026-10-08). Lifespan startup imports them in a worker thread before
|
|
# readiness — a background thread alone would still race the first request.
|
|
_LAZY_ROUTE_IMPORTS = (
|
|
"httpx",
|
|
"backend.services.worker_service",
|
|
"backend.services.flacfetch_client",
|
|
"backend.services.catalog_proxy_service",
|
|
"backend.services.ip_geolocation_service",
|
|
"backend.services.email_validation_service",
|
|
"backend.services.musicbrainz_service",
|
|
)
|
|
|
|
|
|
LAZY_ROUTE_IMPORT_TIMEOUT_S = 30
|
|
|
|
|
|
def _import_lazy_route_modules() -> None:
|
|
import importlib
|
|
|
|
for module in _LAZY_ROUTE_IMPORTS:
|
|
try:
|
|
importlib.import_module(module)
|
|
except Exception as e:
|
|
logger.warning(f"Startup import of {module} failed (will import lazily): {e}")
|
|
|
|
|
|
def _run_background_warmup():
|
|
"""Warm caches that used to block startup (runs in a daemon thread).
|
|
|
|
Historically these preloads ran inline in lifespan startup, which meant
|
|
Cloud Run held every routed request for the full ~7.5s they take (spaCy
|
|
~1s, NLTK ~2.5s, Langfuse ~3s, credential checks ~1.5s) on top of import
|
|
time. None of them are needed to serve HTTP: they exist to make the FIRST
|
|
lyrics-processing job fast (see docs/archive/2026-01-08-performance-
|
|
investigation.md). Running them in a background thread right after boot
|
|
keeps that warm-cache property (they finish within seconds, long before
|
|
any real job runs) without gating readiness. Workers that race the warmup
|
|
simply fall back to the preloaders' lazy paths.
|
|
"""
|
|
warmup_start = time.time()
|
|
|
|
# 1. SpaCy model (60+ second delay without preload)
|
|
try:
|
|
preload_spacy_model("en_core_web_sm")
|
|
except Exception as e:
|
|
logger.warning(f"SpaCy preload failed (will load lazily): {e}")
|
|
|
|
# 2. NLTK cmudict (50-100+ second delay without preload)
|
|
try:
|
|
preload_all_nltk_resources()
|
|
except Exception as e:
|
|
logger.warning(f"NLTK preload failed (will load lazily): {e}")
|
|
|
|
# 3. Langfuse callback handler (200+ second delay without preload)
|
|
try:
|
|
preload_langfuse_handler()
|
|
except Exception as e:
|
|
logger.warning(f"Langfuse preload failed (will initialize lazily): {e}")
|
|
|
|
if fake_workers_enabled(settings):
|
|
# Local stack: no encoding VM and no OAuth credentials to validate.
|
|
logger.info(f"Background warmup complete in {time.time() - warmup_start:.2f}s (fake workers)")
|
|
return
|
|
|
|
# 4. Encoding service singleton — its first construction imports
|
|
# google.cloud.compute_v1 (~10s on a fresh instance). Build it here, off the
|
|
# event loop, before /api/health/detailed or a preview encode needs it.
|
|
try:
|
|
from backend.services.encoding_service import get_encoding_service
|
|
|
|
get_encoding_service()
|
|
except Exception as e:
|
|
logger.warning(f"Encoding service warmup failed (will build lazily): {e}")
|
|
|
|
# Validate OAuth credentials (alerting only, nothing depends on it)
|
|
try:
|
|
validate_credentials_on_startup()
|
|
except Exception as e:
|
|
logger.error(f"Credential validation failed: {e}")
|
|
|
|
logger.info(f"Background warmup complete in {time.time() - warmup_start:.2f}s")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
"""Lifespan event handler for startup and shutdown."""
|
|
# Startup
|
|
logger.info("Starting karaoke generation backend")
|
|
logger.info(f"Environment: {settings.environment}")
|
|
logger.info(f"GCS Bucket: {settings.gcs_bucket_name}")
|
|
logger.info(f"Tracing enabled: {tracing_enabled}")
|
|
|
|
# Fail fast if required production config is missing, rather than silently
|
|
# running on dev defaults (wrong GCP project / GCS bucket / localhost worker
|
|
# URL). No-op outside production. (Fallback audit 2026-06-09, Theme 7.)
|
|
validate_production_config()
|
|
|
|
# Off the loop, but before readiness (Cloud Run holds requests meanwhile).
|
|
# Bounded so a hung cold-disk read can't keep the instance from ever going
|
|
# ready; the import keeps running in its thread and routes fall back to it.
|
|
try:
|
|
await asyncio.wait_for(
|
|
asyncio.to_thread(_import_lazy_route_modules), timeout=LAZY_ROUTE_IMPORT_TIMEOUT_S
|
|
)
|
|
except asyncio.TimeoutError:
|
|
logger.error(
|
|
f"Startup import of lazy route modules exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; "
|
|
"continuing startup (first requests may import them on the event loop)"
|
|
)
|
|
|
|
# NLP model / credential warmup runs in the background so it doesn't gate
|
|
# readiness — Cloud Run holds all routed requests until lifespan startup
|
|
# returns, and these preloads were ~7.5s of the ~16s cold start.
|
|
threading.Thread(
|
|
target=_run_background_warmup, name="startup-warmup", daemon=True
|
|
).start()
|
|
|
|
# asyncio.to_thread's default executor is min(32, cpu+4) ≈ 6 threads on our
|
|
# 2 vCPUs. The screens worker, stale-review cron, auto-correct and many
|
|
# review endpoints all offload into it (mostly I/O-bound), so size it up —
|
|
# otherwise review-page requests queue behind a few long-running jobs.
|
|
import concurrent.futures
|
|
|
|
asyncio.get_running_loop().set_default_executor(
|
|
concurrent.futures.ThreadPoolExecutor(max_workers=48, thread_name_prefix="to-thread")
|
|
)
|
|
|
|
# Event-loop stall watchdog: makes "every request froze for 30s" episodes
|
|
# self-diagnosing (logs the blocking stack; records >=5s stalls in prod).
|
|
watchdog = None
|
|
if settings.loop_watchdog_enabled:
|
|
from backend.services.loop_watchdog import (
|
|
LoopWatchdog,
|
|
firestore_stall_recorder,
|
|
request_tracker,
|
|
)
|
|
|
|
watchdog = LoopWatchdog(
|
|
recorder=firestore_stall_recorder if settings.environment == "production" else None,
|
|
# Stalls nobody waited on (idle, CPU-throttled autoscaled instances)
|
|
# stay WARNING so they don't trip the >=10s stall alert.
|
|
request_tracker=request_tracker,
|
|
)
|
|
watchdog.start()
|
|
|
|
yield
|
|
|
|
if watchdog is not None:
|
|
await watchdog.stop()
|
|
|
|
# Shutdown - best-effort parking of any still-registered workers.
|
|
#
|
|
# Reality check (incident 2026-09-13, job 41e06b90): Cloud Run services
|
|
# SIGKILL instances ~10 seconds after SIGTERM — NOT the 600s this comment
|
|
# previously claimed — and uvicorn only runs this lifespan-shutdown code
|
|
# AFTER all BackgroundTasks have completed ("Waiting for background tasks
|
|
# to complete"). A long-running worker task therefore keeps uvicorn stuck
|
|
# in that wait until SIGKILL, and this hook never executes at all. It can
|
|
# NOT be relied on to protect in-flight renders; that protection comes
|
|
# from running them as Cloud Run Jobs instead (USE_CLOUD_RUN_JOBS_FOR_RENDER,
|
|
# mirroring USE_CLOUD_RUN_JOBS_FOR_VIDEO from incident 2026-03-08).
|
|
#
|
|
# This hook is kept as a cheap safety net for the legacy flag-off path and
|
|
# for the narrow case where workers finish right as SIGTERM lands. The wait
|
|
# is capped well inside the 10s kill window so the parking pass below still
|
|
# gets a chance to write Firestore state.
|
|
logger.info("Shutdown requested, checking for active workers...")
|
|
if worker_registry.has_active_workers():
|
|
active = worker_registry.get_active_workers()
|
|
logger.info(f"Active workers found: {active}")
|
|
logger.info("Waiting for workers to complete (timeout: 5s)...")
|
|
completed = await worker_registry.wait_for_completion(timeout=5)
|
|
if not completed:
|
|
logger.error(
|
|
"Shutdown timeout - some workers may not have completed cleanly. "
|
|
f"Remaining workers: {worker_registry.get_active_workers()}"
|
|
)
|
|
|
|
# Park any still-active render jobs so the auto-retry scheduler can
|
|
# recover them. Safe to call unconditionally — it's a no-op if nothing
|
|
# render-related is in flight.
|
|
try:
|
|
from backend.workers.render_video_worker import park_active_render_jobs_for_shutdown
|
|
parked = park_active_render_jobs_for_shutdown()
|
|
if parked:
|
|
logger.warning(
|
|
f"Parked {parked} in-flight render job(s) for auto-retry "
|
|
f"due to shutdown"
|
|
)
|
|
except Exception as exc:
|
|
logger.error(f"Render-job shutdown park failed: {exc!r}")
|
|
else:
|
|
logger.info("No active workers, proceeding with shutdown")
|
|
|
|
logger.info("Shutting down karaoke generation backend")
|
|
|
|
|
|
# Create FastAPI app
|
|
app = FastAPI(
|
|
title="Karaoke Generator API",
|
|
description="Backend API for web-based karaoke video generation",
|
|
version=VERSION,
|
|
lifespan=lifespan
|
|
)
|
|
|
|
# Instrument FastAPI with OpenTelemetry (adds automatic spans for all requests)
|
|
if tracing_enabled:
|
|
instrument_app(app)
|
|
|
|
# Configure CORS
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=False,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Add audit logging middleware (captures all requests with request_id for correlation)
|
|
app.add_middleware(AuditLoggingMiddleware)
|
|
|
|
# Add tenant detection middleware (extracts tenant from subdomain/headers)
|
|
app.add_middleware(TenantMiddleware)
|
|
|
|
# Edge origin lock (added last = outermost, runs first). Rejects direct-to-origin
|
|
# requests that bypassed the Cloudflare edge, before tenant/audit middleware do
|
|
# any work. No-op unless EDGE_AUTH_MODE is warn/enforce. See
|
|
# backend/middleware/edge_auth.py.
|
|
app.add_middleware(EdgeAuthMiddleware)
|
|
|
|
# Counts in-flight requests for the loop watchdog (outermost, so every request that
|
|
# waited on a stalled loop counts — even ones the edge lock rejects).
|
|
from backend.services.loop_watchdog import RequestTrackingMiddleware # noqa: E402
|
|
|
|
app.add_middleware(RequestTrackingMiddleware)
|
|
|
|
# Include routers
|
|
app.include_router(health.router, prefix="/api")
|
|
app.include_router(jobs.router, prefix="/api")
|
|
app.include_router(file_upload.router, prefix="/api") # File upload endpoint
|
|
app.include_router(internal.router, prefix="/api") # Internal worker endpoints
|
|
app.include_router(encoding_worker.router, prefix="/api") # Encoding worker lifecycle
|
|
app.include_router(review.router, prefix="/api") # Review UI compatibility endpoints
|
|
app.include_router(auth.router, prefix="/api") # OAuth credential management
|
|
app.include_router(audio_search.router, prefix="/api") # Audio search (artist+title mode)
|
|
app.include_router(themes.router, prefix="/api") # Theme selection for styles
|
|
app.include_router(users.router, prefix="/api") # User auth, credits, and Stripe webhooks
|
|
from backend.api.routes import referrals
|
|
app.include_router(referrals.router, prefix="/api") # Referral links and payouts
|
|
app.include_router(admin.router, prefix="/api") # Admin dashboard and management
|
|
app.include_router(rate_limits.router, prefix="/api") # Rate limits admin management
|
|
app.include_router(push.router, prefix="/api") # Push notification subscription management
|
|
app.include_router(catalog.router, prefix="/api") # Catalog proxy for song/artist autocomplete
|
|
app.include_router(parse_titles.router, prefix="/api") # kjbox karaoke-filename parser
|
|
app.include_router(bulk.router, prefix="/api") # Bulk Mode: multi-job submission
|
|
from backend.api.routes import requests_board
|
|
app.include_router(requests_board.router, prefix="/api") # Public song-request voting board
|
|
app.include_router(client_errors.router, prefix="/api") # Frontend crash reports
|
|
app.include_router(client_events.router, prefix="/api") # Frontend degradation telemetry (banner/unavailable/waveform events)
|
|
from backend.api.routes import karaokehunt
|
|
app.include_router(karaokehunt.router, prefix="/api") # Retired KaraokeHunt app request interceptor
|
|
from backend.api.routes import kjbox
|
|
app.include_router(kjbox.router, prefix="/api") # kjbox partner API: singer email-code sign-in + show credits
|
|
from backend.api.routes import login_code
|
|
app.include_router(login_code.router, prefix="/api") # gen email-code sign-in (/users/auth/send-code, verify-code)
|
|
app.include_router(tenant.router) # Tenant/white-label configuration (no /api prefix, router has it)
|
|
app.include_router(tenant_admin.router) # Admin tenant provisioning (router has /api prefix)
|
|
app.include_router(tenant_bulk.router) # Tenant bulk-upload filename analysis (router has /api prefix)
|
|
app.include_router(tenant_theme.router) # Tenant self-service theme editor (router has /api prefix)
|
|
if fake_workers_enabled(settings):
|
|
# Local E2E stack only (startup refuses FAKE_WORKERS in production).
|
|
from backend.api.routes import test_support
|
|
app.include_router(test_support.router, prefix="/api")
|
|
|
|
|
|
# Exception handlers
|
|
from fastapi import Request
|
|
from fastapi.responses import JSONResponse
|
|
from backend.exceptions import InsufficientCreditsError
|
|
from backend.i18n import t, get_locale_from_request
|
|
|
|
|
|
@app.exception_handler(InsufficientCreditsError)
|
|
async def insufficient_credits_exception_handler(request: Request, exc: InsufficientCreditsError):
|
|
"""Handle insufficient credits errors with 402 Payment Required status."""
|
|
locale = get_locale_from_request(request)
|
|
return JSONResponse(
|
|
status_code=402,
|
|
content={
|
|
"detail": t(locale, "errors.insufficientCredits"),
|
|
"credits_available": exc.credits_available,
|
|
"credits_required": exc.credits_required,
|
|
"buy_url": t(locale, "errors.insufficientCreditsBuyUrl"),
|
|
},
|
|
)
|
|
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
"""Root endpoint."""
|
|
return {
|
|
"service": "karaoke-gen-backend",
|
|
"version": VERSION,
|
|
"status": "running"
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=8080)
|
|
|