mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Fixes three cutover bugs that left the ephemeral runner dispatcher silently broken since 2026-05-17T04:56Z: 1. **`on_host_maintenance="TERMINATE"` set unconditionally** — e2 machine types reject TERMINATE unless preemptible, so 100% of general/build VM creates failed. Fix: only set TERMINATE on GPU (n1) families where it's required because attached GPUs can't live-migrate. 3 regression tests added in `test_ephemeral.py`. 2. **GPU `disk_size_gb=150` but image is 200GB** — GCE rejects boot disks smaller than the source image. Fix: raise GPU disk to 200GB. 3. **Runner user has no passwordless sudo** — workflow steps like `sudo apt-get install -y google-cloud-cli-firestore-emulator` (backend-emulator-tests) and `sudo apt-get install -y ffmpeg` (package-*, GPU integration tests) failed with "a terminal is required to read the password". The legacy GHA runner VMs had NOPASSWD sudo. Fix: add `/etc/sudoers.d/runner` in the image provision script. Fixes 1 + 2 are dispatcher code (already applied via `pulumi up --target ...:runner-manager-source ...:runner-manager-function` from this branch). Fix 3 requires a new image build (triggered: build-runner-images.yml on this branch). ## Why the trivial file touches `backend/main.py` and `karaoke_gen/__init__.py` carry one-line comments to flip the `backend` and `package` `dorny/paths-filter` outputs so this PR exercises **all five PR-triggered self-hosted jobs** end-to-end on ephemeral runners. Verification PR (paired): nomadkaraoke/python-audio-separator#286. ## Test plan - [x] `pytest test_ephemeral.py` — 26 passed locally (3 new scheduling tests) - [x] Pulumi applied locally (both iterations, function update verified) - [x] First ephemeral general VM create succeeded (4 RUNNING; previously 100% failure) - [ ] Image rebuild completes for general+build+gpu with new sudo provision - [ ] PR CI passes with all 5 self-hosted jobs running on the new image - [ ] python-audio-separator#286 GPU integration tests pass on the new GPU image - [ ] After merge: `deploy-backend` succeeds on ephemeral build runner; prod /api/health/detailed reports new version - [ ] Phase 4 decommission of 7 legacy VMs+disks (separate PR, ~$220/mo saving) ## Context - `karaoke-gen/docs/EPHEMERAL-GHA-RUNNERS.md` - `docs/archive/2026-05-16-ephemeral-gha-runners-plan.md` (workspace) - Memory: `project_cost_sprint_may2026.md` @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
230 lines
9.1 KiB
Python
230 lines
9.1 KiB
Python
"""
|
|
FastAPI application entry point for karaoke generation backend.
|
|
|
|
(Trivial comment touch to trigger backend CI on ephemeral runners after
|
|
the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.)
|
|
"""
|
|
import logging
|
|
from contextlib import asynccontextmanager
|
|
from fastapi import FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
from backend.config import settings
|
|
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, rate_limits, push, catalog, encoding_worker, client_errors
|
|
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
|
|
from backend.services.structured_logging import setup_structured_logging
|
|
from backend.services.spacy_preloader import preload_spacy_model
|
|
from backend.services.nltk_preloader import preload_all_nltk_resources
|
|
from backend.services.langfuse_preloader import preload_langfuse_handler
|
|
from backend.middleware.audit_logging import AuditLoggingMiddleware
|
|
from backend.middleware.tenant import TenantMiddleware
|
|
from backend.workers.registry import worker_registry
|
|
|
|
|
|
from backend.version import VERSION
|
|
|
|
|
|
# Configure structured logging (JSON in Cloud Run, human-readable locally)
|
|
# This must happen before any logging calls
|
|
setup_structured_logging()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Initialize OpenTelemetry tracing (must happen before app creation)
|
|
tracing_enabled = setup_tracing(
|
|
service_name="karaoke-backend",
|
|
service_version=VERSION,
|
|
enable_in_dev=False, # Set to True to enable tracing locally
|
|
)
|
|
|
|
|
|
async def validate_credentials_on_startup():
|
|
"""Validate OAuth credentials on startup and send alerts if needed."""
|
|
try:
|
|
from backend.services.credential_manager import get_credential_manager, CredentialStatus
|
|
|
|
manager = get_credential_manager()
|
|
results = manager.check_all_credentials()
|
|
|
|
invalid_services = [
|
|
result for result in results.values()
|
|
if result.status in (CredentialStatus.INVALID, CredentialStatus.EXPIRED)
|
|
]
|
|
|
|
if invalid_services:
|
|
logger.warning(f"Some OAuth credentials need attention:")
|
|
for result in invalid_services:
|
|
logger.warning(f" - {result.service}: {result.message}")
|
|
|
|
# Try to send Discord alert
|
|
discord_url = settings.get_secret("discord-alert-webhook") if hasattr(settings, 'get_secret') else None
|
|
if discord_url:
|
|
manager.send_credential_alert(invalid_services, discord_url)
|
|
logger.info("Sent credential alert to Discord")
|
|
else:
|
|
logger.info("All OAuth credentials validated successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to validate credentials on startup: {e}")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
"""Lifespan event handler for startup and shutdown."""
|
|
# Startup
|
|
logger.info("Starting karaoke generation backend")
|
|
logger.info(f"Environment: {settings.environment}")
|
|
logger.info(f"GCS Bucket: {settings.gcs_bucket_name}")
|
|
logger.info(f"Tracing enabled: {tracing_enabled}")
|
|
|
|
# Preload NLP models and resources to avoid cold start delays
|
|
# See docs/archive/2026-01-08-performance-investigation.md for background
|
|
|
|
# 1. SpaCy model (60+ second delay without preload)
|
|
try:
|
|
preload_spacy_model("en_core_web_sm")
|
|
except Exception as e:
|
|
logger.warning(f"SpaCy preload failed (will load lazily): {e}")
|
|
|
|
# 2. NLTK cmudict (50-100+ second delay without preload)
|
|
try:
|
|
preload_all_nltk_resources()
|
|
except Exception as e:
|
|
logger.warning(f"NLTK preload failed (will load lazily): {e}")
|
|
|
|
# 3. Langfuse callback handler (200+ second delay without preload)
|
|
try:
|
|
preload_langfuse_handler()
|
|
except Exception as e:
|
|
logger.warning(f"Langfuse preload failed (will initialize lazily): {e}")
|
|
|
|
# Validate OAuth credentials (non-blocking)
|
|
try:
|
|
await validate_credentials_on_startup()
|
|
except Exception as e:
|
|
logger.error(f"Credential validation failed: {e}")
|
|
|
|
yield
|
|
|
|
# Shutdown - wait for any active workers to complete before terminating
|
|
# This prevents Cloud Run from killing workers mid-processing.
|
|
#
|
|
# Cloud Run gen2 default termination grace period is 600s. We wait up to
|
|
# 480s, leaving ~120s for the cleanup pass below to write Firestore state
|
|
# before SIGKILL. Without that headroom, render jobs that miss the wait
|
|
# would sit in `rendering_video` forever waiting for an operator.
|
|
logger.info("Shutdown requested, checking for active workers...")
|
|
if worker_registry.has_active_workers():
|
|
active = worker_registry.get_active_workers()
|
|
logger.info(f"Active workers found: {active}")
|
|
logger.info("Waiting for workers to complete (timeout: 480s)...")
|
|
completed = await worker_registry.wait_for_completion(timeout=480)
|
|
if not completed:
|
|
logger.error(
|
|
"Shutdown timeout - some workers may not have completed cleanly. "
|
|
f"Remaining workers: {worker_registry.get_active_workers()}"
|
|
)
|
|
|
|
# Park any still-active render jobs so the auto-retry scheduler can
|
|
# recover them. Safe to call unconditionally — it's a no-op if nothing
|
|
# render-related is in flight.
|
|
try:
|
|
from backend.workers.render_video_worker import park_active_render_jobs_for_shutdown
|
|
parked = park_active_render_jobs_for_shutdown()
|
|
if parked:
|
|
logger.warning(
|
|
f"Parked {parked} in-flight render job(s) for auto-retry "
|
|
f"due to shutdown"
|
|
)
|
|
except Exception as exc:
|
|
logger.error(f"Render-job shutdown park failed: {exc!r}")
|
|
else:
|
|
logger.info("No active workers, proceeding with shutdown")
|
|
|
|
logger.info("Shutting down karaoke generation backend")
|
|
|
|
|
|
# Create FastAPI app
|
|
app = FastAPI(
|
|
title="Karaoke Generator API",
|
|
description="Backend API for web-based karaoke video generation",
|
|
version=VERSION,
|
|
lifespan=lifespan
|
|
)
|
|
|
|
# Instrument FastAPI with OpenTelemetry (adds automatic spans for all requests)
|
|
if tracing_enabled:
|
|
instrument_app(app)
|
|
|
|
# Configure CORS
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=False,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Add audit logging middleware (captures all requests with request_id for correlation)
|
|
app.add_middleware(AuditLoggingMiddleware)
|
|
|
|
# Add tenant detection middleware (extracts tenant from subdomain/headers)
|
|
app.add_middleware(TenantMiddleware)
|
|
|
|
# Include routers
|
|
app.include_router(health.router, prefix="/api")
|
|
app.include_router(jobs.router, prefix="/api")
|
|
app.include_router(file_upload.router, prefix="/api") # File upload endpoint
|
|
app.include_router(internal.router, prefix="/api") # Internal worker endpoints
|
|
app.include_router(encoding_worker.router, prefix="/api") # Encoding worker lifecycle
|
|
app.include_router(review.router, prefix="/api") # Review UI compatibility endpoints
|
|
app.include_router(auth.router, prefix="/api") # OAuth credential management
|
|
app.include_router(audio_search.router, prefix="/api") # Audio search (artist+title mode)
|
|
app.include_router(themes.router, prefix="/api") # Theme selection for styles
|
|
app.include_router(users.router, prefix="/api") # User auth, credits, and Stripe webhooks
|
|
from backend.api.routes import referrals
|
|
app.include_router(referrals.router, prefix="/api") # Referral links and payouts
|
|
app.include_router(admin.router, prefix="/api") # Admin dashboard and management
|
|
app.include_router(rate_limits.router, prefix="/api") # Rate limits admin management
|
|
app.include_router(push.router, prefix="/api") # Push notification subscription management
|
|
app.include_router(catalog.router, prefix="/api") # Catalog proxy for song/artist autocomplete
|
|
app.include_router(client_errors.router, prefix="/api") # Frontend crash reports
|
|
app.include_router(tenant.router) # Tenant/white-label configuration (no /api prefix, router has it)
|
|
|
|
|
|
# Exception handlers
|
|
from fastapi import Request
|
|
from fastapi.responses import JSONResponse
|
|
from backend.exceptions import InsufficientCreditsError
|
|
from backend.i18n import t, get_locale_from_request
|
|
|
|
|
|
@app.exception_handler(InsufficientCreditsError)
|
|
async def insufficient_credits_exception_handler(request: Request, exc: InsufficientCreditsError):
|
|
"""Handle insufficient credits errors with 402 Payment Required status."""
|
|
locale = get_locale_from_request(request)
|
|
return JSONResponse(
|
|
status_code=402,
|
|
content={
|
|
"detail": t(locale, "errors.insufficientCredits"),
|
|
"credits_available": exc.credits_available,
|
|
"credits_required": exc.credits_required,
|
|
"buy_url": t(locale, "errors.insufficientCreditsBuyUrl"),
|
|
},
|
|
)
|
|
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
"""Root endpoint."""
|
|
return {
|
|
"service": "karaoke-gen-backend",
|
|
"version": VERSION,
|
|
"status": "running"
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=8080)
|
|
|