mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(alerts): start-review 409 on finished jobs, per-cause crash patterns, one alert per job failure (v0.290.1)
From the 2026-11-03 #alerts triage. - start-review returns 409 for a job not in review (and when losing the auto-approver race) instead of a 500; correction-data's same race too. - The error monitor builds unhandled-exception patterns from the exception's cause, so a second crash on another endpoint is no longer swallowed by an already-open "Exception in ASGI application" pattern. - YouTube video ids are normalized out of signatures; one job failure is one alert (fail_job and the known download-failure lines log WARNING). - Suppressions: Zalo's injected script, python -c experiments on job images, decide-search health-probe aborts. Demotions: Spotify 403 on /lyrics, Dropbox token probe (ERROR moves to the refresh failing). - Cold start: import the anyio backend and warm the middleware stack off-loop. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
This commit is contained in:
1 parent
ea69386e50
commit
f006246bd6
25 files changed
+942
-26
No files matched your search
@@ -19,6 +19,7 @@ from backend.services.error_monitor.frontend_ingestion import (
|
||||
RateLimiter,
|
||||
build_pattern_data,
|
||||
is_bot_user_agent,
|
||||
is_host_app_injection_error,
|
||||
is_opaque_script_error,
|
||||
sanitize_url,
|
||||
)
|
||||
@@ -90,6 +91,15 @@ def report_client_error(payload: ClientErrorPayload, request: Request) -> Client
|
||||
)
|
||||
return ClientErrorResponse(pattern_id="opaque-script-error-ignored", is_new=False)
|
||||
|
||||
# A script the in-app browser's host app injected (Zalo), not our code.
|
||||
if is_host_app_injection_error(payload.message, payload.user_agent):
|
||||
logger.info(
|
||||
"frontend_crash_ignored_host_app_injection url=%s source=%s",
|
||||
sanitize_url(payload.url),
|
||||
payload.source,
|
||||
)
|
||||
return ClientErrorResponse(pattern_id="host-app-injection-ignored", is_new=False)
|
||||
|
||||
report = FrontendErrorReport(
|
||||
message=payload.message,
|
||||
stack=payload.stack,
|
||||
|
||||
@@ -1085,6 +1085,14 @@ def get_review_data(
|
||||
}
|
||||
|
||||
|
||||
def _job_not_in_review_conflict(locale: str, status: Any) -> HTTPException:
|
||||
"""409 for a review request on a job that isn't in review."""
|
||||
return HTTPException(
|
||||
status_code=409,
|
||||
detail=t(locale, "jobs.jobNotInReviewState", status=getattr(status, "value", status)),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{job_id}/start-review")
|
||||
def start_review(
|
||||
job_id: str,
|
||||
@@ -1109,10 +1117,21 @@ def start_review(
|
||||
# Idempotent: concurrent/duplicate opens (React double-mount, double-tap, two
|
||||
# tabs) must not fail with "in_review -> in_review". Already in review == success,
|
||||
# and the one-time side effects (record_review_started) are not repeated.
|
||||
# A job past review (e.g. a stale "review lyrics" push tapped on a finished song)
|
||||
# is the client's to handle: 409, no transition attempted, nothing logged at ERROR.
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise _job_not_in_review_conflict(locale, job.status)
|
||||
|
||||
was_awaiting = job.status == JobStatus.AWAITING_REVIEW
|
||||
result = ensure_in_review(
|
||||
job_manager, job_id, job.status, "User started reviewing lyrics"
|
||||
)
|
||||
try:
|
||||
result = ensure_in_review(
|
||||
job_manager, job_id, job.status, "User started reviewing lyrics"
|
||||
)
|
||||
except InvalidStateTransitionError:
|
||||
# Lost the race: the auto-approver (or another worker) moved the job on
|
||||
# between our read and the transition.
|
||||
current = job_manager.get_job(job_id)
|
||||
raise _job_not_in_review_conflict(locale, current.status if current else job.status)
|
||||
if result == "refused":
|
||||
raise HTTPException(status_code=400, detail=t(locale, "jobs.reviewCannotStart"))
|
||||
if result == "started" and was_awaiting:
|
||||
|
||||
@@ -32,6 +32,7 @@ from google.cloud.firestore_v1.field_path import FieldPath
|
||||
from backend.models.job import JobStatus
|
||||
from backend.models.review_session import ReviewSession, ReviewSessionSummary
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
from backend.services.review_state import ensure_in_review
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
from backend.services.storage_service import StorageService
|
||||
@@ -932,13 +933,23 @@ async def get_correction_data(
|
||||
if not replay and job.status == JobStatus.AWAITING_REVIEW:
|
||||
# Idempotent: iOS/in-app browsers fire two concurrent loads; the loser
|
||||
# must not 500 on "in_review -> in_review" nor repeat one-time side effects.
|
||||
started = "started" == await asyncio.to_thread(
|
||||
ensure_in_review,
|
||||
job_manager,
|
||||
job_id,
|
||||
job.status,
|
||||
"User opened combined review interface",
|
||||
)
|
||||
try:
|
||||
started = "started" == await asyncio.to_thread(
|
||||
ensure_in_review,
|
||||
job_manager,
|
||||
job_id,
|
||||
job.status,
|
||||
"User opened combined review interface",
|
||||
)
|
||||
except InvalidStateTransitionError:
|
||||
# Lost the race: the auto-approver moved the job on after the status
|
||||
# gate above. Not a load failure, so 409 rather than the 500 below.
|
||||
current = await asyncio.to_thread(job_manager.get_job, job_id)
|
||||
status = current.status if current else job.status
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=t("en", "review.reviewNotReady", status=getattr(status, "value", status)),
|
||||
)
|
||||
if started:
|
||||
# Set by require_review_auth (absent for review-token links).
|
||||
auth_state = getattr(request, "state", None)
|
||||
@@ -978,6 +989,8 @@ async def get_correction_data(
|
||||
logger.info(f"Job {job_id}: Serving correction data (replay={replay}) for combined review")
|
||||
return corrections_data
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Error loading corrections: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=t("en", "review.correctionsLoadFailed", error=str(e)))
|
||||
|
||||
@@ -9,6 +9,7 @@ import logging
|
||||
import threading
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
@@ -95,6 +96,9 @@ _LAZY_ROUTE_IMPORTS = (
|
||||
"flacfetch.core.manager",
|
||||
"flacfetch.downloaders.youtube",
|
||||
"yt_dlp",
|
||||
# starlette/middleware/base.py picks the anyio backend on the first request:
|
||||
# 17.3s on the loop with 18 requests in flight (2026-10-30).
|
||||
"anyio._backends._asyncio",
|
||||
)
|
||||
|
||||
|
||||
@@ -137,6 +141,29 @@ def _warm_route_contexts(application: FastAPI) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def _warm_middleware_stack(application: Any) -> None:
|
||||
"""Send one request through the app, so the first user's request isn't what builds
|
||||
the middleware stack and runs its first-use imports on the loop.
|
||||
|
||||
Blocking: runs its own event loop, so call it from a worker thread. /api/health is
|
||||
exempt from the edge lock, skips the audit log and touches no storage. Best-effort.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
async def _request() -> None:
|
||||
transport = httpx.ASGITransport(app=application)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client:
|
||||
await client.get("/api/health")
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
asyncio.run(_request())
|
||||
except Exception as e: # noqa: BLE001 - warmup must never block startup
|
||||
logger.warning(f"Middleware warm request failed (stack builds on first request): {e}")
|
||||
return
|
||||
logger.info(f"Warmed middleware stack in {time.time() - start:.2f}s")
|
||||
|
||||
|
||||
def _warm_route_contexts_safely(application: FastAPI) -> None:
|
||||
start = time.time()
|
||||
try:
|
||||
@@ -243,6 +270,16 @@ async def lifespan(app: FastAPI):
|
||||
f"Route-context warmup exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; continuing startup"
|
||||
)
|
||||
|
||||
# Same reason, for what only the first request through the middleware touches.
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.to_thread(_warm_middleware_stack, app), timeout=LAZY_ROUTE_IMPORT_TIMEOUT_S
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
f"Middleware warmup exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; continuing startup"
|
||||
)
|
||||
|
||||
# NLP model / credential warmup runs in the background so it doesn't gate
|
||||
# readiness — Cloud Run holds all routed requests until lifespan startup
|
||||
# returns, and these preloads were ~7.5s of the ~16s cold start.
|
||||
|
||||
@@ -222,7 +222,9 @@ class CredentialManager:
|
||||
message="Dropbox credentials refreshed successfully",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
else:
|
||||
logger.error("Dropbox access token rejected and no refresh credentials are stored")
|
||||
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.INVALID,
|
||||
@@ -410,7 +412,9 @@ class CredentialManager:
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Dropbox API test failed: {e}")
|
||||
# Short-lived tokens expire constantly and the caller refreshes them, so
|
||||
# the probe failing isn't page-worthy; the refresh failing is.
|
||||
logger.warning(f"Dropbox API test failed: {e}")
|
||||
return False
|
||||
|
||||
def _refresh_dropbox_token(self, creds: dict) -> bool:
|
||||
@@ -436,7 +440,8 @@ class CredentialManager:
|
||||
# Update in Secret Manager
|
||||
self._update_dropbox_credentials(creds)
|
||||
return True
|
||||
|
||||
|
||||
logger.error(f"Dropbox token refresh rejected ({response.status_code}): {response.text[:200]}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -74,6 +74,20 @@ def is_opaque_script_error(message: str | None, source: str | None = None, extra
|
||||
return not extra.get("filename") and not extra.get("lineno")
|
||||
|
||||
|
||||
# Zalo's in-app browser injects scripts into every page it opens; one references a
|
||||
# global it hasn't defined yet. Not our code, and nothing the user sees. Needs both
|
||||
# the host app's UA and its global, so a real error inside Zalo still reports.
|
||||
_HOST_APP_INJECTION_RE = re.compile(r"\bzaloJSV\d*\b is not defined")
|
||||
_ZALO_UA_RE = re.compile(r"\bZalo\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def is_host_app_injection_error(message: str | None, user_agent: str | None) -> bool:
|
||||
"""True for an error from a script the in-app browser's host app injected."""
|
||||
if not message or not user_agent:
|
||||
return False
|
||||
return bool(_HOST_APP_INJECTION_RE.search(message) and _ZALO_UA_RE.search(user_agent))
|
||||
|
||||
|
||||
# A chunk-load failure is a tab running an old build after a deploy: the client
|
||||
# auto-reloads (frontend/lib/version-check.ts), so one report is not actionable.
|
||||
# Mirrors isChunkLoadError() there. Reports collapse into one pattern per build
|
||||
|
||||
@@ -15,6 +15,7 @@ Usage:
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -129,18 +130,55 @@ _SERVICE_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str]] = [
|
||||
"while instances start, GitHub redelivers and the runners still dispatch "
|
||||
"(seen 2026-09-20, 09-23, 09-27, 10-08)",
|
||||
),
|
||||
(
|
||||
"audio_separation_experiment_traceback",
|
||||
frozenset({"audio-separation-job"}),
|
||||
# Two consecutive <string> frames: `python -c` running exec'd code, i.e. an
|
||||
# experiment on a prod job image. A multiprocessing-spawn traceback also
|
||||
# starts with a <string> frame, but its second frame is a real file.
|
||||
re.compile(r'File "<string>", line 1, in <module>\s+File "<string>", line \d+'),
|
||||
"Ad-hoc experiment run on a prod audio-separation-job image, not a job failure "
|
||||
"(seen 2026-10-31, 11-01)",
|
||||
),
|
||||
(
|
||||
"url_download_failed_duplicate",
|
||||
frozenset({"karaoke-backend"}),
|
||||
# file_upload.py logs this, then fail_job: ops_alerts' job-failed alert is the
|
||||
# one alert for the failure.
|
||||
re.compile(r"^URL download failed: "),
|
||||
"Same failure as the job-failed alert that fail_job sends right after this line",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def should_ignore(service: str, message: str) -> IgnoreReason | None:
|
||||
# Noise only for one service AND request URL path (Cloud Run request logs carry the
|
||||
# URL outside the message). (pattern_name, services, message regex, exact URL path, reason)
|
||||
_SERVICE_URL_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str, str]] = [
|
||||
(
|
||||
"decide_search_health_abort",
|
||||
frozenset({"karaoke-decide-search"}),
|
||||
re.compile(
|
||||
r"request was aborted after waiting too long for an available instance",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"/api/search/health",
|
||||
"The keep-warm ping waiting on a min-0 cold start (seen 2026-10-30). The same "
|
||||
"abort on a real search path drops a user's request and still alerts.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def should_ignore(service: str, message: str, request_url: str | None = None) -> IgnoreReason | None:
|
||||
"""Check whether a log entry matches a known infrastructure noise pattern.
|
||||
|
||||
``_IGNORE_PATTERNS`` match on the message alone; ``_SERVICE_IGNORE_PATTERNS``
|
||||
also require ``service`` to be one of the listed services.
|
||||
also require ``service`` to be one of the listed services, and
|
||||
``_SERVICE_URL_IGNORE_PATTERNS`` also require ``request_url``'s path.
|
||||
|
||||
Args:
|
||||
service: The originating service name (e.g. ``"karaoke-backend"``).
|
||||
message: The raw log message string to evaluate.
|
||||
request_url: The entry's request URL, if it is a request log.
|
||||
|
||||
Returns:
|
||||
An :class:`IgnoreReason` if the message matches a known noise pattern,
|
||||
@@ -157,4 +195,10 @@ def should_ignore(service: str, message: str) -> IgnoreReason | None:
|
||||
if service in services and pattern.search(message):
|
||||
return IgnoreReason(pattern_name=pattern_name, reason=reason)
|
||||
|
||||
if request_url:
|
||||
url_path = urlsplit(request_url).path
|
||||
for pattern_name, services, pattern, path, reason in _SERVICE_URL_IGNORE_PATTERNS:
|
||||
if service in services and url_path == path and pattern.search(message):
|
||||
return IgnoreReason(pattern_name=pattern_name, reason=reason)
|
||||
|
||||
return None
|
||||
@@ -15,6 +15,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -115,6 +116,39 @@ def _build_log_filter(
|
||||
)
|
||||
|
||||
|
||||
#: Fixed messages uvicorn and the audit middleware log for every unhandled exception.
|
||||
#: The cause is only in the entry's ``exception`` field.
|
||||
_UNHANDLED_EXCEPTION_MESSAGES = frozenset({"Exception in ASGI application", "request_audit_error"})
|
||||
|
||||
#: The margin of an ExceptionGroup traceback (" | ", " | | ", " + ").
|
||||
_TRACEBACK_MARGIN_RE = re.compile(r"^\s*(?:[|+] )+")
|
||||
#: Its separators (" +-+---- 1 ----", " +-------").
|
||||
_TRACEBACK_SEPARATOR_RE = re.compile(r"^\s*(?:\| )*\+[-+]")
|
||||
_TRACEBACK_FRAME_RE = re.compile(r'^\s*File ".*", line \d+')
|
||||
|
||||
|
||||
def _exception_summary(exception_text: str) -> str | None:
|
||||
"""The ``Type: message`` line a traceback ends on, without its group margin.
|
||||
|
||||
Looks after the last frame, not at the last line: a multi-line exception message
|
||||
(pydantic's ends in a docs link shared by every failure) would otherwise replace
|
||||
the cause. Frame source and caret lines are indented; the exception line isn't.
|
||||
"""
|
||||
lines = [
|
||||
_TRACEBACK_MARGIN_RE.sub("", line)
|
||||
for line in exception_text.splitlines()
|
||||
if line.strip() and not _TRACEBACK_SEPARATOR_RE.match(line)
|
||||
]
|
||||
if not lines:
|
||||
return None
|
||||
last_frame = max((i for i, line in enumerate(lines) if _TRACEBACK_FRAME_RE.match(line)), default=None)
|
||||
if last_frame is not None:
|
||||
for line in lines[last_frame + 1 :]:
|
||||
if not line[0].isspace():
|
||||
return line
|
||||
return lines[-1].strip()
|
||||
|
||||
|
||||
def _extract_message(entry: Any) -> str | None:
|
||||
"""Extract the log message text from a Cloud Logging entry.
|
||||
|
||||
@@ -125,6 +159,9 @@ def _extract_message(entry: Any) -> str | None:
|
||||
``"msg"`` keys in that order.
|
||||
* ``None`` payload — returns ``None``.
|
||||
|
||||
An unhandled-exception message becomes ``Unhandled exception: <cause>``, so
|
||||
each cause is its own pattern and both log lines of one crash share it.
|
||||
|
||||
Args:
|
||||
entry: A Cloud Logging ``LogEntry`` object.
|
||||
|
||||
@@ -140,13 +177,40 @@ def _extract_message(entry: Any) -> str | None:
|
||||
for key in ("message", "textPayload", "error", "msg"):
|
||||
value = payload.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
text = str(value)
|
||||
if text.strip() in _UNHANDLED_EXCEPTION_MESSAGES:
|
||||
summary = _exception_summary(str(payload.get("exception") or ""))
|
||||
if summary:
|
||||
return f"Unhandled exception: {summary}"
|
||||
return text
|
||||
return None
|
||||
# Fallback: convert to string representation
|
||||
text = str(payload)
|
||||
return text if text else None
|
||||
|
||||
|
||||
def _extract_sample(entry: Any, message: str) -> str:
|
||||
"""The alert's sample: the message, led by the request path when the entry has one.
|
||||
|
||||
The path stays out of the pattern (one cause on many jobs or endpoints is one
|
||||
alert) but the alert needs it to say where to look.
|
||||
"""
|
||||
payload = entry.payload
|
||||
path = payload.get("path") if isinstance(payload, dict) else None
|
||||
if not path:
|
||||
return message
|
||||
method = payload.get("method")
|
||||
return f"{method} {path}\n{message}" if method else f"{path}\n{message}"
|
||||
|
||||
|
||||
def _extract_request_url(entry: Any) -> str | None:
|
||||
"""The request URL on a Cloud Run request-log entry, if any."""
|
||||
http_request = getattr(entry, "http_request", None)
|
||||
if not isinstance(http_request, dict):
|
||||
return None
|
||||
return http_request.get("requestUrl") or None
|
||||
|
||||
|
||||
def _extract_service_name(entry: Any, resource_type: str) -> str:
|
||||
"""Extract the service/job/function/instance name from a log entry.
|
||||
|
||||
@@ -370,7 +434,7 @@ class ErrorMonitor:
|
||||
if not message:
|
||||
continue
|
||||
service = _extract_service_name(entry, resource_type)
|
||||
ignore_reason = should_ignore(service, message)
|
||||
ignore_reason = should_ignore(service, message, _extract_request_url(entry))
|
||||
if ignore_reason:
|
||||
logger.debug(
|
||||
"Ignoring entry from %s (%s): %s",
|
||||
@@ -387,7 +451,7 @@ class ErrorMonitor:
|
||||
"service": service,
|
||||
"resource_type": _classify_resource_type(resource_type),
|
||||
"normalized": normalized,
|
||||
"message": message,
|
||||
"message": _extract_sample(entry, message),
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
||||
@@ -74,6 +74,17 @@ _NORMALIZERS: list[tuple[re.Pattern, str]] = [
|
||||
re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"),
|
||||
r"https://\1",
|
||||
),
|
||||
# 8b. YouTube video IDs: yt-dlp's "[youtube] <id>: <error>" and bare watch links.
|
||||
# Otherwise the same error on each video is a new pattern and a "first time"
|
||||
# job-failure signature (ops_alerts._signature uses this normalizer).
|
||||
(
|
||||
re.compile(r"(\[youtube(?::[\w-]+)?\]) [A-Za-z0-9_-]{11}(?=:)"),
|
||||
r"\1 <VIDEO_ID>",
|
||||
),
|
||||
(
|
||||
re.compile(r"(youtube\.com/watch\?v=|youtu\.be/)[A-Za-z0-9_-]{11}(?![A-Za-z0-9_-])"),
|
||||
r"\1<VIDEO_ID>",
|
||||
),
|
||||
# 9. Job IDs in paths (/jobs/<alphanumeric>)
|
||||
(
|
||||
re.compile(r"/jobs/[A-Za-z0-9]+"),
|
||||
|
||||
@@ -1346,7 +1346,8 @@ class JobManager:
|
||||
message=error_message
|
||||
)
|
||||
|
||||
logger.error(f"Job {job_id} failed: {error_message}")
|
||||
# WARNING: the status write above sends the one job-failed alert (ops_alerts).
|
||||
logger.warning(f"Job {job_id} failed: {error_message}")
|
||||
|
||||
# Auto-refund credit on failure
|
||||
job = self.get_job(job_id)
|
||||
@@ -1693,7 +1694,8 @@ class JobManager:
|
||||
message=f"Failed: {error_message}",
|
||||
**updates
|
||||
)
|
||||
logger.error(f"Job {job_id} failed: {error_message}")
|
||||
# WARNING: the status write above sends the one job-failed alert (ops_alerts).
|
||||
logger.warning(f"Job {job_id} failed: {error_message}")
|
||||
if not already_notified:
|
||||
self._trigger_state_notifications(job_id, JobStatus.FAILED)
|
||||
|
||||
|
||||
@@ -142,3 +142,32 @@ def test_strips_sensitive_query_string(client):
|
||||
(pd,), _ = client.fake_adapter.upsert_pattern.call_args
|
||||
assert "SECRET" not in pd.sample_message
|
||||
assert "token=" not in pd.sample_message
|
||||
|
||||
|
||||
ZALO_UA = (
|
||||
"Mozilla/5.0 (Linux; Android 16; X Build/Y;) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Version/4.0 Chrome/156.0.8078.47 Mobile Safari/537.36 Zalo android/261002903 ZaloTheme/light"
|
||||
)
|
||||
ZALO_MESSAGE = "ReferenceError: zaloJSV2 is not defined"
|
||||
|
||||
|
||||
def test_zalo_host_app_injection_is_accepted_but_not_persisted(client):
|
||||
resp = client.post(
|
||||
"/api/client-errors",
|
||||
json=_payload(message=ZALO_MESSAGE, stack=f"{ZALO_MESSAGE}\n at <anonymous>:1:2", user_agent=ZALO_UA),
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
assert resp.json()["is_new"] is False
|
||||
client.fake_adapter.upsert_pattern.assert_not_called()
|
||||
|
||||
|
||||
def test_real_errors_inside_zalo_are_persisted(client):
|
||||
resp = client.post("/api/client-errors", json=_payload(user_agent=ZALO_UA))
|
||||
assert resp.status_code == 202
|
||||
client.fake_adapter.upsert_pattern.assert_called_once()
|
||||
|
||||
|
||||
def test_zalo_message_from_another_browser_is_persisted(client):
|
||||
resp = client.post("/api/client-errors", json=_payload(message=ZALO_MESSAGE))
|
||||
assert resp.status_code == 202
|
||||
client.fake_adapter.upsert_pattern.assert_called_once()
|
||||
@@ -1085,6 +1085,17 @@ class TestTorrentStallAwaitsUser:
|
||||
assert details["stall_minutes"] == 60
|
||||
assert details["keep_trying"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_known_download_failure_logs_warning_not_error(self, caplog):
|
||||
"""fail_job's job-failed alert is the one alert for this failure."""
|
||||
import logging
|
||||
|
||||
flacfetch = _mock_flacfetch(FlacfetchServiceError("Download failed: tracker 500"))
|
||||
with caplog.at_level(logging.INFO, logger="backend.workers.audio_download_worker"):
|
||||
await self._run(_red_job(), flacfetch, attempt="0")
|
||||
records = [r for r in caplog.records if "Download failed: Download failed" in r.getMessage()]
|
||||
assert [r.levelno for r in records] == [logging.WARNING]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stall_failure_still_marks_retry_pending(self):
|
||||
flacfetch = _mock_flacfetch(FlacfetchServiceError("Download failed: tracker 500"))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Tests for credential manager and auth endpoints.
|
||||
"""
|
||||
import logging
|
||||
import pytest
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -194,6 +195,65 @@ class TestCredentialManager:
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_expired_dropbox_token_probe_logs_warning_not_error(self, caplog):
|
||||
"""Short-lived tokens expire constantly and get refreshed; that isn't page-worthy."""
|
||||
manager = CredentialManager()
|
||||
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")):
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
|
||||
assert manager._test_dropbox_api({"access_token": "test"}) is False
|
||||
levels = [r.levelno for r in caplog.records if "Dropbox" in r.message]
|
||||
assert levels == [logging.WARNING]
|
||||
|
||||
def _dropbox_manager(self, creds):
|
||||
manager = CredentialManager()
|
||||
manager.settings = MagicMock()
|
||||
manager.settings.get_secret.return_value = json.dumps(creds)
|
||||
return manager
|
||||
|
||||
FULL_DROPBOX_CREDS = {"access_token": "a", "refresh_token": "r", "app_key": "k", "app_secret": "s"}
|
||||
|
||||
def test_dropbox_refresh_success_logs_no_error(self, caplog):
|
||||
manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS)
|
||||
ok = MagicMock(status_code=200)
|
||||
ok.json.return_value = {"access_token": "new"}
|
||||
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \
|
||||
patch('requests.post', return_value=ok), \
|
||||
patch.object(manager, "_update_dropbox_credentials", return_value=True):
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
|
||||
result = manager.check_dropbox_credentials()
|
||||
assert result.status == CredentialStatus.VALID
|
||||
assert not [r for r in caplog.records if r.levelno >= logging.ERROR]
|
||||
|
||||
def test_dropbox_refresh_rejected_logs_error(self, caplog):
|
||||
manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS)
|
||||
bad = MagicMock(status_code=400, text='{"error": "invalid_grant"}')
|
||||
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \
|
||||
patch('requests.post', return_value=bad):
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
|
||||
result = manager.check_dropbox_credentials()
|
||||
assert result.status == CredentialStatus.INVALID
|
||||
errors = [r.message for r in caplog.records if r.levelno >= logging.ERROR]
|
||||
assert len(errors) == 1
|
||||
assert "400" in errors[0]
|
||||
assert "invalid_grant" in errors[0]
|
||||
|
||||
def test_dropbox_refresh_raising_logs_one_error(self, caplog):
|
||||
manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS)
|
||||
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \
|
||||
patch('requests.post', side_effect=RuntimeError("network down")):
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
|
||||
result = manager.check_dropbox_credentials()
|
||||
assert result.status == CredentialStatus.INVALID
|
||||
assert len([r for r in caplog.records if r.levelno >= logging.ERROR]) == 1
|
||||
|
||||
def test_dropbox_invalid_with_no_refresh_token_logs_error(self, caplog):
|
||||
manager = self._dropbox_manager({"access_token": "a"})
|
||||
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")):
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
|
||||
result = manager.check_dropbox_credentials()
|
||||
assert result.status == CredentialStatus.INVALID
|
||||
assert len([r for r in caplog.records if r.levelno >= logging.ERROR]) == 1
|
||||
|
||||
def test_send_credential_alert_no_webhook(self):
|
||||
"""Test alert without webhook URL."""
|
||||
manager = CredentialManager()
|
||||
|
||||
@@ -290,6 +290,24 @@ class TestJobStatusTransitions:
|
||||
class TestJobFailure:
|
||||
"""Test job failure handling."""
|
||||
|
||||
def test_fail_job_logs_warning_not_error(self, job_manager, mock_firestore_service, caplog):
|
||||
"""ops_alerts' job-failed alert is the one alert; an ERROR here made a second one."""
|
||||
import logging
|
||||
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.job_manager"):
|
||||
job_manager.fail_job("job-1", "Download failed: boom")
|
||||
records = [r for r in caplog.records if "job-1 failed" in r.getMessage()]
|
||||
assert [r.levelno for r in records] == [logging.WARNING]
|
||||
|
||||
def test_mark_job_failed_logs_warning_not_error(self, job_manager, mock_firestore_service, caplog):
|
||||
import logging
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="backend.services.job_manager"):
|
||||
job_manager.mark_job_failed("job-2", "boom")
|
||||
records = [r for r in caplog.records if "job-2 failed" in r.getMessage()]
|
||||
assert [r.levelno for r in records] == [logging.WARNING]
|
||||
|
||||
def test_mark_job_as_failed(self, job_manager, mock_firestore_service):
|
||||
"""Test marking a job as failed."""
|
||||
error_message = "Audio separation failed"
|
||||
|
||||
@@ -440,12 +440,30 @@ class TestStartReview:
|
||||
assert response.json() == {"status": "success", "job_status": "in_review"}
|
||||
mock_job_manager.record_review_started.assert_not_called()
|
||||
|
||||
def test_start_review_invalid_transition_otherwise_still_errors(self, client, mock_job_manager, mock_job, auth_headers):
|
||||
def test_start_review_invalid_transition_race_returns_409(self, client, mock_job_manager, mock_job, auth_headers):
|
||||
"""The auto-approver moved the job past review between our read and the transition."""
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
mock_job.status = JobStatus.AWAITING_REVIEW
|
||||
moved_on = mock_job.model_copy(update={"status": JobStatus.REVIEW_COMPLETE})
|
||||
mock_job_manager.get_job.side_effect = [mock_job, moved_on, moved_on]
|
||||
mock_job_manager.transition_to_state.side_effect = InvalidStateTransitionError(message="nope")
|
||||
with pytest.raises(InvalidStateTransitionError):
|
||||
client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers)
|
||||
response = client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers)
|
||||
assert response.status_code == 409
|
||||
assert "review_complete" in response.json()["detail"]
|
||||
mock_job_manager.record_review_started.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("status", [JobStatus.COMPLETE, JobStatus.FAILED, JobStatus.PENDING])
|
||||
def test_start_review_outside_review_returns_409_without_transition(
|
||||
self, client, mock_job_manager, mock_job, auth_headers, status
|
||||
):
|
||||
"""A stale "review lyrics" push on a finished job: no 500, no transition, no ERROR log."""
|
||||
mock_job.status = status
|
||||
response = client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers)
|
||||
assert response.status_code == 409
|
||||
assert status.value in response.json()["detail"]
|
||||
assert "JobStatus." not in response.json()["detail"]
|
||||
mock_job_manager.transition_to_state.assert_not_called()
|
||||
mock_job_manager.record_review_started.assert_not_called()
|
||||
|
||||
def test_start_review_from_awaiting_records_once(self, client, mock_job_manager, mock_job, auth_headers):
|
||||
mock_job.status = JobStatus.AWAITING_REVIEW
|
||||
|
||||
@@ -181,3 +181,67 @@ def test_lifespan_warms_route_contexts_off_loop_before_readiness():
|
||||
|
||||
assert seen[0][0] is main_module.app
|
||||
assert seen[0][1].startswith("asyncio_"), seen
|
||||
|
||||
|
||||
def test_anyio_backend_is_a_lazy_route_import():
|
||||
"""starlette/middleware/base.py imports it on the first request: 17.3s on the loop
|
||||
with 18 requests in flight, 2026-10-30."""
|
||||
assert "anyio._backends._asyncio" in main_module._LAZY_ROUTE_IMPORTS
|
||||
|
||||
|
||||
def test_warm_request_runs_through_the_whole_middleware_stack_off_loop():
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import PlainTextResponse
|
||||
from starlette.routing import Route
|
||||
|
||||
seen = {}
|
||||
|
||||
class Spy(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request, call_next):
|
||||
seen["thread"] = threading.current_thread().name
|
||||
seen["path"] = request.url.path
|
||||
return await call_next(request)
|
||||
|
||||
async def health(_request):
|
||||
return PlainTextResponse("ok")
|
||||
|
||||
app = Starlette(routes=[Route("/api/health", health)], middleware=[Middleware(Spy)])
|
||||
assert app.middleware_stack is None
|
||||
|
||||
worker = threading.Thread(target=main_module._warm_middleware_stack, args=(app,), name="warm-test")
|
||||
worker.start()
|
||||
worker.join(timeout=10)
|
||||
|
||||
assert seen == {"thread": "warm-test", "path": "/api/health"}
|
||||
assert app.middleware_stack is not None
|
||||
|
||||
|
||||
def test_warm_request_failure_never_blocks_startup():
|
||||
async def broken_app(scope, receive, send):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
main_module._warm_middleware_stack(broken_app) # must not raise
|
||||
|
||||
|
||||
def test_lifespan_warms_the_middleware_stack_before_readiness():
|
||||
calls = []
|
||||
|
||||
def fake_warm(application):
|
||||
calls.append(application)
|
||||
|
||||
with patch.object(main_module, "_run_background_warmup"), \
|
||||
patch.object(main_module, "_warm_middleware_stack", fake_warm):
|
||||
with TestClient(main_module.app):
|
||||
assert calls == [main_module.app]
|
||||
|
||||
|
||||
def test_warm_request_on_the_real_app_returns_healthy_and_skips_the_audit_log(caplog):
|
||||
import logging
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
main_module._warm_middleware_stack(main_module.app)
|
||||
assert main_module.app.middleware_stack is not None
|
||||
assert not [r for r in caplog.records if r.name == "audit"]
|
||||
assert not [r for r in caplog.records if "warm request failed" in r.message.lower()]
|
||||
@@ -1738,6 +1738,35 @@ class TestReplayCorrectionData:
|
||||
finally:
|
||||
self._teardown()
|
||||
|
||||
def _awaiting_job(self):
|
||||
from backend.models.job import JobStatus
|
||||
job = self._completed_job()
|
||||
job.status = JobStatus.AWAITING_REVIEW
|
||||
job.state_data = {}
|
||||
return job
|
||||
|
||||
def test_auto_approver_race_returns_409_not_500(self):
|
||||
"""Job read as awaiting_review, then auto-approved before our transition."""
|
||||
from backend.exceptions import InvalidStateTransitionError
|
||||
from backend.models.job import JobStatus
|
||||
|
||||
client = self._client_with_auth("full")
|
||||
try:
|
||||
with patch("backend.api.routes.review.JobManager") as jm, \
|
||||
patch("backend.api.routes.review.StorageService") as storage_cls:
|
||||
moved_on = self._awaiting_job()
|
||||
moved_on.status = JobStatus.REVIEW_COMPLETE
|
||||
jm.return_value.get_job.side_effect = [self._awaiting_job(), moved_on, moved_on]
|
||||
jm.return_value.transition_to_state.side_effect = InvalidStateTransitionError(message="nope")
|
||||
storage_cls.return_value.file_exists.return_value = True
|
||||
storage_cls.return_value.download_json.return_value = {"corrected_segments": []}
|
||||
resp = client.get("/api/review/job1/correction-data")
|
||||
assert resp.status_code == 409
|
||||
assert "review_complete" in resp.json()["detail"]
|
||||
jm.return_value.record_review_started.assert_not_called()
|
||||
finally:
|
||||
self._teardown()
|
||||
|
||||
def test_replay_requires_full_auth(self):
|
||||
client = self._client_with_auth("review_token")
|
||||
try:
|
||||
|
||||
@@ -399,7 +399,8 @@ async def process_audio_download(job_id: str) -> bool:
|
||||
if not isinstance(e, (DownloadError, FlacfetchServiceError)):
|
||||
logger.error(f"[job:{job_id}] Download failed: {e}", exc_info=True)
|
||||
else:
|
||||
logger.error(f"[job:{job_id}] Download failed: {e}")
|
||||
# Expected failure type; fail_job below sends the one job-failed alert.
|
||||
logger.warning(f"[job:{job_id}] Download failed: {e}")
|
||||
_mark_retry_pending_if_attempts_remain(job_manager, job_id)
|
||||
job_manager.fail_job(job_id, f"Audio download failed: {e}",
|
||||
error_details={'stage': 'audio_download'})
|
||||
|
||||
@@ -117,7 +117,14 @@ Keeping #alerts readable (one root cause → one message):
|
||||
`ended` line is ERROR (≥10s blocked with a request affected), so one freeze is
|
||||
one pattern. The `backend/event_loop_stalls` metric keys on that line too.
|
||||
- **Known noise** (`known_issues.py`) can be service-scoped, e.g.
|
||||
github-runner-manager's scale-up "no available instance" aborts.
|
||||
github-runner-manager's scale-up "no available instance" aborts, and scoped to a
|
||||
request path too (decide-search's `/api/search/health` aborts).
|
||||
- **Unhandled exceptions**: `Exception in ASGI application` and `request_audit_error`
|
||||
become one `Unhandled exception: <Type: message>` pattern per cause, read from the
|
||||
entry's `exception` field; the path goes in the sample only.
|
||||
- **One alert per job failure**: `ops_alerts` sends it from the status write, so
|
||||
`Job <id> failed` and the download-failure lines before it are not ERROR patterns.
|
||||
YouTube video ids are normalized out of signatures.
|
||||
|
||||
Frontend crashes are captured via `window.onerror`, `unhandledrejection`, Next.js
|
||||
`error.tsx`, and a `CrashReportBoundary` around the review UIs. They POST to
|
||||
|
||||
@@ -110,6 +110,13 @@ class SpotifyProvider(BaseLyricsProvider):
|
||||
|
||||
self.logger.debug(f"Making RapidAPI lyrics request for track ID {track_id}")
|
||||
lyrics_response = requests.get(lyrics_url, headers=headers, params=lyrics_params, timeout=10)
|
||||
if lyrics_response.status_code == 403:
|
||||
# The search just succeeded with the same key, so the key is fine: a 401/403
|
||||
# on /search stays the dead-key ERROR. Other providers fill in.
|
||||
self.logger.warning(
|
||||
f"Spotify RapidAPI lyrics step returned 403 for track {track_id}: {lyrics_response.text[:200]}"
|
||||
)
|
||||
return None
|
||||
lyrics_response.raise_for_status()
|
||||
|
||||
lyrics_data = lyrics_response.json()
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.290.0"
|
||||
version = "0.290.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -585,3 +585,111 @@ class TestRunnerManagerNoAvailableInstance:
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("github-runner-manager", "Failed to create runner VM: quota exceeded") is None
|
||||
|
||||
|
||||
class TestAudioSeparationExperimentTracebacks:
|
||||
"""Ad-hoc `python -c` experiments on prod job images (2026-10-31, 11-01) are not
|
||||
job failures: two consecutive <string> frames."""
|
||||
|
||||
EXPERIMENT = (
|
||||
"Traceback (most recent call last):\n"
|
||||
' File "<string>", line 1, in <module>\n'
|
||||
' File "<string>", line 250, in <module>\n'
|
||||
' File "<string>", line 228, in forced_align\n'
|
||||
"OverflowError: Python integer 1162 out of bounds for int8"
|
||||
)
|
||||
SPAWN = (
|
||||
"Traceback (most recent call last):\n"
|
||||
' File "<string>", line 1, in <module>\n'
|
||||
' File "/usr/local/lib/python3.11/multiprocessing/spawn.py", line 122, in spawn_main\n'
|
||||
" exitcode = _main(fd, parent_sentinel)\n"
|
||||
"RuntimeError: a real worker crash"
|
||||
)
|
||||
|
||||
def test_experiment_ignored_on_the_job(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("audio-separation-job", self.EXPERIMENT)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "audio_separation_experiment_traceback"
|
||||
|
||||
def test_multiprocessing_spawn_traceback_still_alerts(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("audio-separation-job", self.SPAWN) is None
|
||||
|
||||
def test_same_traceback_on_other_services_still_alerts(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("karaoke-backend", self.EXPERIMENT) is None
|
||||
|
||||
def test_extract_message_does_not_truncate_before_should_ignore(self):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
from backend.services.error_monitor.monitor import _extract_message
|
||||
|
||||
entry = MagicMock()
|
||||
entry.payload = "x" * 5000 + "\n" + self.EXPERIMENT
|
||||
assert should_ignore("audio-separation-job", _extract_message(entry)) is not None
|
||||
|
||||
|
||||
class TestUrlDownloadFailedDuplicate:
|
||||
"""fail_job follows this line, and ops_alerts' job-failed alert is the one alert."""
|
||||
|
||||
MSG = "URL download failed: Remote download failed: ERROR: [youtube] abcdefghijk: Sign in to confirm"
|
||||
|
||||
def test_ignored_on_the_backend(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", self.MSG)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "url_download_failed_duplicate"
|
||||
|
||||
def test_other_download_errors_still_alert(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("karaoke-backend", "GCS upload failed: URL download failed: x") is None
|
||||
assert should_ignore("karaoke-backend", "Download failed: something") is None
|
||||
|
||||
|
||||
class TestDecideSearchHealthAborts:
|
||||
MSG = (
|
||||
"The request was aborted after waiting too long for an available instance. "
|
||||
"Additional troubleshooting documentation can be found at: "
|
||||
"https://cloud.google.com/run/docs/troubleshooting#abort-request"
|
||||
)
|
||||
HEALTH = "https://karaoke-decide-search-x.a.run.app/api/search/health"
|
||||
|
||||
def test_health_probe_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-decide-search", self.MSG, self.HEALTH)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "decide_search_health_abort"
|
||||
|
||||
def test_health_probe_with_query_string_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("karaoke-decide-search", self.MSG, self.HEALTH + "?ping=1") is not None
|
||||
|
||||
def test_real_search_paths_still_alert(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
for url in (
|
||||
"https://karaoke-decide-search-x.a.run.app/api/search?q=hello",
|
||||
"https://karaoke-decide-search-x.a.run.app/api/search/health/deep",
|
||||
"https://karaoke-decide-search-x.a.run.app/api/search/songs?q=/api/search/health",
|
||||
):
|
||||
assert should_ignore("karaoke-decide-search", self.MSG, url) is None
|
||||
|
||||
def test_no_url_still_alerts(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("karaoke-decide-search", self.MSG) is None
|
||||
|
||||
def test_other_services_and_messages_still_alert(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("karaoke-decide", self.MSG, self.HEALTH) is None
|
||||
assert should_ignore("karaoke-decide-search", "boom", self.HEALTH) is None
|
||||
@@ -178,6 +178,243 @@ class TestExtractMessage:
|
||||
assert result is None or result == ""
|
||||
|
||||
|
||||
# Invented paths/ids; the shapes are copied from real prod entries (2026-11-03).
|
||||
_PLAIN_TRACEBACK = (
|
||||
"Traceback (most recent call last):\n"
|
||||
' File "/app/backend/middleware/audit_logging.py", line 63, in dispatch\n'
|
||||
" response = await call_next(request)\n"
|
||||
" ^^^^^^^^^^^^^^^^^^^^^^^^\n"
|
||||
' File "/app/backend/services/job_manager.py", line 657, in validate_state_transition\n'
|
||||
" raise InvalidStateTransitionError(\n"
|
||||
"backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: "
|
||||
"complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']\n"
|
||||
)
|
||||
_GROUP_TRACEBACK = (
|
||||
" + Exception Group Traceback (most recent call last):\n"
|
||||
' | File "/usr/local/lib/python3.11/site-packages/starlette/_utils.py", line 104, in create_collapsing_task_group\n'
|
||||
" | async with anyio.create_task_group() as tg:\n"
|
||||
' | File "/usr/local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 847, in __aexit__\n'
|
||||
" | raise BaseExceptionGroup(\n"
|
||||
" | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)\n"
|
||||
" +-+---------------- 1 ----------------\n"
|
||||
" | Traceback (most recent call last):\n"
|
||||
' | File "/app/backend/services/job_manager.py", line 657, in validate_state_transition\n'
|
||||
" | raise InvalidStateTransitionError(\n"
|
||||
" | backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: "
|
||||
"complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']\n"
|
||||
" +------------------------------------\n"
|
||||
)
|
||||
_EXPECTED_CAUSE = (
|
||||
"Unhandled exception: backend.exceptions.InvalidStateTransitionError: Invalid state transition "
|
||||
"for job 7f89f884: complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']"
|
||||
)
|
||||
|
||||
|
||||
class TestUnhandledExceptionMessages:
|
||||
"""Every unhandled exception logs one of two fixed messages; the pattern must
|
||||
come from the exception, or all crashes share one (already 'new') pattern and
|
||||
a second, different crash never alerts."""
|
||||
|
||||
def _entry(self, message, exception, **extra):
|
||||
entry = MagicMock()
|
||||
entry.payload = {"message": message, "exception": exception, **extra}
|
||||
return entry
|
||||
|
||||
def test_audit_error_plain_traceback(self):
|
||||
m = _import_monitor_module()
|
||||
entry = self._entry("request_audit_error", _PLAIN_TRACEBACK, path="/api/jobs/x/start-review", method="POST")
|
||||
assert m._extract_message(entry) == _EXPECTED_CAUSE
|
||||
|
||||
def test_asgi_exception_group_traceback_with_pipe_prefixes(self):
|
||||
m = _import_monitor_module()
|
||||
entry = self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK)
|
||||
assert m._extract_message(entry) == _EXPECTED_CAUSE
|
||||
|
||||
def test_nested_exception_group(self):
|
||||
m = _import_monitor_module()
|
||||
nested = (
|
||||
" + Exception Group Traceback (most recent call last):\n"
|
||||
" | ExceptionGroup: outer (1 sub-exception)\n"
|
||||
" +-+---------------- 1 ----------------\n"
|
||||
" | Exception Group Traceback (most recent call last):\n"
|
||||
" | ExceptionGroup: inner (1 sub-exception)\n"
|
||||
" +-+---------------- 1 ----------------\n"
|
||||
" | Traceback (most recent call last):\n"
|
||||
' | File "/app/a.py", line 1, in f\n'
|
||||
" | boom()\n"
|
||||
" | ValueError: the real cause\n"
|
||||
" +------------------------------------\n"
|
||||
" +------------------------------------\n"
|
||||
)
|
||||
assert m._extract_message(self._entry("request_audit_error", nested)) == (
|
||||
"Unhandled exception: ValueError: the real cause"
|
||||
)
|
||||
|
||||
def test_multiline_exception_message_uses_its_first_line(self):
|
||||
# pydantic's last traceback line is a docs link, the same for every failure.
|
||||
m = _import_monitor_module()
|
||||
exc = (
|
||||
"Traceback (most recent call last):\n"
|
||||
' File "/app/a.py", line 1, in f\n'
|
||||
" boom()\n"
|
||||
"pydantic_core._pydantic_core.ValidationError: 1 validation error for Job\n"
|
||||
"status\n"
|
||||
" Input should be a valid string\n"
|
||||
" For further information visit https://errors.pydantic.dev/2/v/string_type\n"
|
||||
)
|
||||
assert m._extract_message(self._entry("request_audit_error", exc)) == (
|
||||
"Unhandled exception: pydantic_core._pydantic_core.ValidationError: 1 validation error for Job"
|
||||
)
|
||||
|
||||
def test_chained_exception_uses_the_final_one(self):
|
||||
m = _import_monitor_module()
|
||||
exc = (
|
||||
"Traceback (most recent call last):\n"
|
||||
' File "/app/a.py", line 1, in f\n'
|
||||
" boom()\n"
|
||||
"KeyError: 'first'\n\n"
|
||||
"During handling of the above exception, another exception occurred:\n\n"
|
||||
"Traceback (most recent call last):\n"
|
||||
' File "/app/b.py", line 2, in g\n'
|
||||
" bang()\n"
|
||||
"RuntimeError: second\n"
|
||||
)
|
||||
assert m._extract_message(self._entry("request_audit_error", exc)) == (
|
||||
"Unhandled exception: RuntimeError: second"
|
||||
)
|
||||
|
||||
def test_no_exception_field_keeps_the_message(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = {"message": "request_audit_error"}
|
||||
assert m._extract_message(entry) == "request_audit_error"
|
||||
|
||||
def test_unparseable_exception_keeps_last_line(self):
|
||||
m = _import_monitor_module()
|
||||
assert m._extract_message(self._entry("request_audit_error", "something odd\nno frames here\n")) == (
|
||||
"Unhandled exception: no frames here"
|
||||
)
|
||||
|
||||
def test_other_messages_with_exception_field_are_untouched(self):
|
||||
m = _import_monitor_module()
|
||||
entry = self._entry("Job 1 could not save", _PLAIN_TRACEBACK)
|
||||
assert m._extract_message(entry) == "Job 1 could not save"
|
||||
|
||||
def test_both_log_lines_of_one_crash_collapse_to_one_pattern(self):
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message
|
||||
|
||||
m = _import_monitor_module()
|
||||
audit = m._extract_message(self._entry("request_audit_error", _PLAIN_TRACEBACK))
|
||||
asgi = m._extract_message(self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK))
|
||||
assert compute_pattern_hash("s", normalize_message(audit)) == compute_pattern_hash("s", normalize_message(asgi))
|
||||
|
||||
def test_two_different_crashes_are_two_patterns(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
m = _import_monitor_module()
|
||||
other = _PLAIN_TRACEBACK.replace(
|
||||
"backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: "
|
||||
"complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']",
|
||||
"ZeroDivisionError: division by zero",
|
||||
)
|
||||
first = m._extract_message(self._entry("request_audit_error", _PLAIN_TRACEBACK))
|
||||
second = m._extract_message(self._entry("request_audit_error", other))
|
||||
assert normalize_message(first) != normalize_message(second)
|
||||
|
||||
def test_sample_keeps_the_path_but_the_pattern_does_not(self):
|
||||
m = _import_monitor_module()
|
||||
entry = self._entry(
|
||||
"request_audit_error", _PLAIN_TRACEBACK, path="/api/jobs/7f89f884/start-review", method="POST"
|
||||
)
|
||||
message = m._extract_message(entry)
|
||||
sample = m._extract_sample(entry, message)
|
||||
assert sample.startswith("POST /api/jobs/7f89f884/start-review\n")
|
||||
assert sample.endswith(_EXPECTED_CAUSE)
|
||||
assert "start-review" not in message
|
||||
|
||||
def test_sample_is_the_message_without_a_path(self):
|
||||
m = _import_monitor_module()
|
||||
entry = self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK)
|
||||
message = m._extract_message(entry)
|
||||
assert m._extract_sample(entry, message) == message
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_cycle_groups_a_crash_into_one_alert_with_the_path_in_its_sample(self, _digest, _llm):
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
m = _import_monitor_module()
|
||||
audit = _make_entry(
|
||||
payload={
|
||||
"message": "request_audit_error",
|
||||
"exception": _PLAIN_TRACEBACK,
|
||||
"path": "/api/jobs/7f89f884/start-review",
|
||||
"method": "POST",
|
||||
}
|
||||
)
|
||||
asgi = _make_entry(payload={"message": "Exception in ASGI application\n", "exception": _GROUP_TRACEBACK})
|
||||
audit.http_request = asgi.http_request = None
|
||||
monitor = _make_monitor_with_mocks()
|
||||
monitor.logging_client.list_entries.side_effect = [iter([audit, asgi])] + [iter([])] * (
|
||||
len(m.RESOURCE_TYPE_FILTERS) - 1
|
||||
)
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(pattern_id="p", is_new=True)
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.firestore_adapter.get_unalerted_new_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
assert monitor.firestore_adapter.upsert_pattern.call_count == 1
|
||||
data = monitor.firestore_adapter.upsert_pattern.call_args[0][0]
|
||||
assert data.count == 2
|
||||
assert "/api/jobs/7f89f884/start-review" in data.sample_message
|
||||
assert "start-review" not in data.normalized_message
|
||||
assert "InvalidStateTransitionError" in data.normalized_message
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
|
||||
|
||||
class TestExtractRequestUrl:
|
||||
def test_reads_the_cloud_run_request_url(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.http_request = {"requestUrl": "https://x.a.run.app/api/search/health"}
|
||||
assert m._extract_request_url(entry) == "https://x.a.run.app/api/search/health"
|
||||
|
||||
def test_none_without_http_request(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.http_request = None
|
||||
assert m._extract_request_url(entry) is None
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_cycle_drops_health_aborts_but_alerts_real_search_paths(self, _digest, _llm):
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
m = _import_monitor_module()
|
||||
msg = "The request was aborted after waiting too long for an available instance."
|
||||
health = _make_entry(payload=msg, service_name="karaoke-decide-search")
|
||||
health.http_request = {"requestUrl": "https://x.a.run.app/api/search/health"}
|
||||
real = _make_entry(payload=msg, service_name="karaoke-decide-search")
|
||||
real.http_request = {"requestUrl": "https://x.a.run.app/api/search?q=hello"}
|
||||
monitor = _make_monitor_with_mocks()
|
||||
monitor.logging_client.list_entries.side_effect = [iter([health, real])] + [iter([])] * (
|
||||
len(m.RESOURCE_TYPE_FILTERS) - 1
|
||||
)
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(pattern_id="p", is_new=True)
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.firestore_adapter.get_unalerted_new_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
assert monitor.firestore_adapter.upsert_pattern.call_count == 1
|
||||
assert monitor.firestore_adapter.upsert_pattern.call_args[0][0].count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _extract_service_name
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -452,3 +452,46 @@ def test_loop_stall_request_counts_collapse_to_one_pattern():
|
||||
b = normalize_message("EVENT_LOOP_STALL ended duration_ms=81183 culprit=x.py:539 in f requests_affected=3")
|
||||
assert a == b
|
||||
assert "requests_affected=<N>" in a
|
||||
|
||||
|
||||
class TestYouTubeVideoIds:
|
||||
"""A video id in the text made every bot block a "first time" signature."""
|
||||
|
||||
def _bot_block(self, video_id):
|
||||
return (
|
||||
f"Download failed: ERROR: [youtube] {video_id}: Sign in to confirm you\u2019re not a bot. "
|
||||
"Use --cookies-from-browser or --cookies for the authentication."
|
||||
)
|
||||
|
||||
def test_same_error_on_different_videos_is_one_signature(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
assert normalize_message(self._bot_block("mBJivIsN0n0")) == normalize_message(self._bot_block("a_b-cDeFgHi"))
|
||||
|
||||
def test_id_replaced_with_a_placeholder(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
result = normalize_message(self._bot_block("mBJivIsN0n0"))
|
||||
assert "[youtube] <VIDEO_ID>: Sign in" in result
|
||||
assert "mBJivIsN0n0" not in result
|
||||
|
||||
def test_tab_extractor_variant(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
assert "[youtube:tab] <VIDEO_ID>:" in normalize_message("ERROR: [youtube:tab] mBJivIsN0n0: gone")
|
||||
|
||||
def test_bare_watch_url_id(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
result = normalize_message("Could not fetch youtube.com/watch?v=mBJivIsN0n0 today")
|
||||
assert "mBJivIsN0n0" not in result
|
||||
|
||||
def test_youtube_words_without_an_id_are_kept(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
assert normalize_message("[youtube] Downloading webpage") == "[youtube] Downloading webpage"
|
||||
|
||||
def test_ops_alert_signature_ignores_the_video_id(self):
|
||||
from backend.services.ops_alerts import _signature
|
||||
|
||||
assert _signature(self._bot_block("mBJivIsN0n0")) == _signature(self._bot_block("a_b-cDeFgHi"))
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Log levels on the Spotify RapidAPI provider: ERROR pages a human, so a 403 on the
|
||||
/lyrics step (the search just succeeded with the same key, other providers fill in)
|
||||
is a WARNING, while 401/403 on /search stays the dead-key ERROR."""
|
||||
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from karaoke_gen.lyrics_transcriber.lyrics.base_lyrics_provider import LyricsProviderConfig
|
||||
from karaoke_gen.lyrics_transcriber.lyrics.spotify import SpotifyProvider
|
||||
|
||||
SEARCH_OK = {"status": True, "errorId": "Success", "id": "track1"}
|
||||
|
||||
|
||||
def _response(status, body=None, text=""):
|
||||
resp = MagicMock()
|
||||
resp.status_code = status
|
||||
resp.text = text
|
||||
resp.json.return_value = body or {}
|
||||
if status >= 400:
|
||||
err = requests.exceptions.HTTPError(f"{status} Client Error", response=resp)
|
||||
resp.raise_for_status.side_effect = err
|
||||
return resp
|
||||
|
||||
|
||||
def _provider():
|
||||
return SpotifyProvider(LyricsProviderConfig(rapidapi_key="test-key"), logger=logging.getLogger("spotify-test"))
|
||||
|
||||
|
||||
def _levels(caplog):
|
||||
return [r.levelno for r in caplog.records if r.name == "spotify-test"]
|
||||
|
||||
|
||||
def test_lyrics_403_is_a_warning_with_the_body(caplog):
|
||||
long_body = "not subscribed " + "x" * 500
|
||||
with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(403, text=long_body)]):
|
||||
with caplog.at_level(logging.INFO, logger="spotify-test"):
|
||||
assert _provider()._fetch_from_rapidapi("A", "T") is None
|
||||
assert logging.ERROR not in _levels(caplog)
|
||||
warning = next(r.message for r in caplog.records if r.levelno == logging.WARNING)
|
||||
assert "not subscribed" in warning
|
||||
assert "x" * 201 not in warning
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [401, 403])
|
||||
def test_search_auth_failure_stays_an_error(caplog, status):
|
||||
with patch("requests.get", side_effect=[_response(status)]):
|
||||
with caplog.at_level(logging.INFO, logger="spotify-test"):
|
||||
assert _provider()._fetch_from_rapidapi("A", "T") is None
|
||||
assert logging.ERROR in _levels(caplog)
|
||||
|
||||
|
||||
def test_lyrics_401_stays_an_error(caplog):
|
||||
with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(401)]):
|
||||
with caplog.at_level(logging.INFO, logger="spotify-test"):
|
||||
assert _provider()._fetch_from_rapidapi("A", "T") is None
|
||||
assert logging.ERROR in _levels(caplog)
|
||||
|
||||
|
||||
def test_lyrics_success_still_returns_data():
|
||||
with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(200, {"lyrics": []})]):
|
||||
result = _provider()._fetch_from_rapidapi("A", "T")
|
||||
assert result["_rapidapi_source"] is True
|
||||
Reference in new issue
Block a user