Files
karaoke-gen/backend/api/routes/client_errors.py
T
Andrew BeveridgeandClaude Opus 5.5 f006246bd6 fix(alerts): start-review 409 on finished jobs, per-cause crash patterns, one alert per job failure (v0.290.1)
From the 2026-11-03 #alerts triage.

- start-review returns 409 for a job not in review (and when losing the
  auto-approver race) instead of a 500; correction-data's same race too.
- The error monitor builds unhandled-exception patterns from the exception's
  cause, so a second crash on another endpoint is no longer swallowed by an
  already-open "Exception in ASGI application" pattern.
- YouTube video ids are normalized out of signatures; one job failure is one
  alert (fail_job and the known download-failure lines log WARNING).
- Suppressions: Zalo's injected script, python -c experiments on job images,
  decide-search health-probe aborts. Demotions: Spotify 403 on /lyrics, Dropbox
  token probe (ERROR moves to the refresh failing).
- Cold start: import the anyio backend and warm the middleware stack off-loop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
2026-11-03 17:53:57 -05:00

137 lines
5.0 KiB
Python

"""POST /api/client-errors — ingest frontend crash reports.
Unauthenticated (logged-out users' crashes still matter). Deduped and alerted
via the existing error-monitor pipeline — see
``backend/services/error_monitor``.
"""
from __future__ import annotations
import logging
import time
from typing import Optional
from fastapi import APIRouter, Request, HTTPException
from pydantic import BaseModel, Field, field_validator
from backend.services.error_monitor.firestore_adapter import ErrorPatternsAdapter
from backend.services.error_monitor.frontend_ingestion import (
FrontendErrorReport,
RateLimiter,
build_pattern_data,
is_bot_user_agent,
is_host_app_injection_error,
is_opaque_script_error,
sanitize_url,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/client-errors", tags=["client-errors"])
_limiter = RateLimiter(max_per_minute=60)
_adapter_singleton: ErrorPatternsAdapter | None = None
def _get_adapter() -> ErrorPatternsAdapter:
global _adapter_singleton
if _adapter_singleton is None:
_adapter_singleton = ErrorPatternsAdapter()
return _adapter_singleton
class ClientErrorPayload(BaseModel):
message: str = Field(..., min_length=1, max_length=4000)
stack: Optional[str] = Field(None, max_length=1_000_000)
url: str = Field("", max_length=2048)
user_agent: str = Field("", max_length=1024)
release: str = Field("", max_length=64)
user_email: Optional[str] = Field(None, max_length=320)
viewport: Optional[dict] = None
locale: str = Field("en", max_length=10)
source: str = Field("unknown", max_length=64)
# optional free-form extra for debugging. Capped server-side.
extra: Optional[dict] = None
@field_validator("message", "url", "user_agent", "locale", "source")
@classmethod
def strip(cls, v: str) -> str:
return (v or "").strip()
class ClientErrorResponse(BaseModel):
pattern_id: str
is_new: bool
@router.post("", response_model=ClientErrorResponse, status_code=202)
def report_client_error(payload: ClientErrorPayload, request: Request) -> ClientErrorResponse:
client_ip = request.client.host if request.client else "unknown"
if not _limiter.allow(client_ip, time.monotonic()):
raise HTTPException(status_code=429, detail="too many reports")
# Crawler/bot crashes are pure alert noise (bots run headless Chrome and trip
# DOM errors no real user hits). Accept the report so the client doesn't retry,
# but don't persist/alert on it.
if is_bot_user_agent(payload.user_agent):
# Log the bot UA (not sensitive) + sanitized URL (query/token stripped)
# for visibility into what's being dropped, without persisting/alerting.
logger.info(
"frontend_crash_ignored_bot ua=%r url=%s source=%s",
payload.user_agent[:120],
sanitize_url(payload.url),
payload.source,
)
return ClientErrorResponse(pattern_id="bot-ignored", is_new=False)
# Opaque cross-origin "Script error." — no file/line, not our code.
if is_opaque_script_error(payload.message, payload.source, payload.extra):
logger.info(
"frontend_crash_ignored_opaque_script_error url=%s source=%s",
sanitize_url(payload.url),
payload.source,
)
return ClientErrorResponse(pattern_id="opaque-script-error-ignored", is_new=False)
# A script the in-app browser's host app injected (Zalo), not our code.
if is_host_app_injection_error(payload.message, payload.user_agent):
logger.info(
"frontend_crash_ignored_host_app_injection url=%s source=%s",
sanitize_url(payload.url),
payload.source,
)
return ClientErrorResponse(pattern_id="host-app-injection-ignored", is_new=False)
report = FrontendErrorReport(
message=payload.message,
stack=payload.stack,
url=payload.url,
user_agent=payload.user_agent,
release=payload.release,
user_email=payload.user_email,
viewport=payload.viewport,
locale=payload.locale,
extra=payload.extra,
)
pattern_data = build_pattern_data(report)
try:
result = _get_adapter().upsert_pattern(pattern_data)
except Exception: # pragma: no cover - Firestore transient errors
logger.exception("Failed to upsert frontend error pattern")
raise HTTPException(status_code=503, detail="storage unavailable")
logger.info(
"frontend_crash_reported pattern_id=%s is_new=%s source=%s",
result.pattern_id,
result.is_new,
payload.source,
# Full client diagnostics + breadcrumbs, searchable in Cloud Logging
# (the alert sample only carries a trimmed summary).
extra={
"error_pattern_id": result.pattern_id,
"client_url": sanitize_url(payload.url),
"client_release": payload.release,
"client_extra": payload.extra,
},
)
return ClientErrorResponse(pattern_id=result.pattern_id, is_new=result.is_new)