fix(alerts): start-review 409 on finished jobs, per-cause crash patterns, one alert per job failure (v0.290.1)

From the 2026-11-03 #alerts triage.

- start-review returns 409 for a job not in review (and when losing the
  auto-approver race) instead of a 500; correction-data's same race too.
- The error monitor builds unhandled-exception patterns from the exception's
  cause, so a second crash on another endpoint is no longer swallowed by an
  already-open "Exception in ASGI application" pattern.
- YouTube video ids are normalized out of signatures; one job failure is one
  alert (fail_job and the known download-failure lines log WARNING).
- Suppressions: Zalo's injected script, python -c experiments on job images,
  decide-search health-probe aborts. Demotions: Spotify 403 on /lyrics, Dropbox
  token probe (ERROR moves to the refresh failing).
- Cold start: import the anyio backend and warm the middleware stack off-loop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
This commit is contained in:
Andrew BeveridgeandClaude Opus 5.5 committed 2026-11-03 17:53:57 -05:00
1 parent ea69386e50
commit f006246bd6
25 files changed
+942 -26

No files matched your search

+10
View File
@@ -19,6 +19,7 @@ from backend.services.error_monitor.frontend_ingestion import (
RateLimiter,
build_pattern_data,
is_bot_user_agent,
is_host_app_injection_error,
is_opaque_script_error,
sanitize_url,
)
@@ -90,6 +91,15 @@ def report_client_error(payload: ClientErrorPayload, request: Request) -> Client
)
return ClientErrorResponse(pattern_id="opaque-script-error-ignored", is_new=False)
# A script the in-app browser's host app injected (Zalo), not our code.
if is_host_app_injection_error(payload.message, payload.user_agent):
logger.info(
"frontend_crash_ignored_host_app_injection url=%s source=%s",
sanitize_url(payload.url),
payload.source,
)
return ClientErrorResponse(pattern_id="host-app-injection-ignored", is_new=False)
report = FrontendErrorReport(
message=payload.message,
stack=payload.stack,
+22 -3
View File
@@ -1085,6 +1085,14 @@ def get_review_data(
}
def _job_not_in_review_conflict(locale: str, status: Any) -> HTTPException:
"""409 for a review request on a job that isn't in review."""
return HTTPException(
status_code=409,
detail=t(locale, "jobs.jobNotInReviewState", status=getattr(status, "value", status)),
)
@router.post("/{job_id}/start-review")
def start_review(
job_id: str,
@@ -1109,10 +1117,21 @@ def start_review(
# Idempotent: concurrent/duplicate opens (React double-mount, double-tap, two
# tabs) must not fail with "in_review -> in_review". Already in review == success,
# and the one-time side effects (record_review_started) are not repeated.
# A job past review (e.g. a stale "review lyrics" push tapped on a finished song)
# is the client's to handle: 409, no transition attempted, nothing logged at ERROR.
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
raise _job_not_in_review_conflict(locale, job.status)
was_awaiting = job.status == JobStatus.AWAITING_REVIEW
result = ensure_in_review(
job_manager, job_id, job.status, "User started reviewing lyrics"
)
try:
result = ensure_in_review(
job_manager, job_id, job.status, "User started reviewing lyrics"
)
except InvalidStateTransitionError:
# Lost the race: the auto-approver (or another worker) moved the job on
# between our read and the transition.
current = job_manager.get_job(job_id)
raise _job_not_in_review_conflict(locale, current.status if current else job.status)
if result == "refused":
raise HTTPException(status_code=400, detail=t(locale, "jobs.reviewCannotStart"))
if result == "started" and was_awaiting:
+20 -7
View File
@@ -32,6 +32,7 @@ from google.cloud.firestore_v1.field_path import FieldPath
from backend.models.job import JobStatus
from backend.models.review_session import ReviewSession, ReviewSessionSummary
from backend.services.job_manager import JobManager
from backend.exceptions import InvalidStateTransitionError
from backend.services.review_state import ensure_in_review
from backend.services.firestore_service import FirestoreService
from backend.services.storage_service import StorageService
@@ -932,13 +933,23 @@ async def get_correction_data(
if not replay and job.status == JobStatus.AWAITING_REVIEW:
# Idempotent: iOS/in-app browsers fire two concurrent loads; the loser
# must not 500 on "in_review -> in_review" nor repeat one-time side effects.
started = "started" == await asyncio.to_thread(
ensure_in_review,
job_manager,
job_id,
job.status,
"User opened combined review interface",
)
try:
started = "started" == await asyncio.to_thread(
ensure_in_review,
job_manager,
job_id,
job.status,
"User opened combined review interface",
)
except InvalidStateTransitionError:
# Lost the race: the auto-approver moved the job on after the status
# gate above. Not a load failure, so 409 rather than the 500 below.
current = await asyncio.to_thread(job_manager.get_job, job_id)
status = current.status if current else job.status
raise HTTPException(
status_code=409,
detail=t("en", "review.reviewNotReady", status=getattr(status, "value", status)),
)
if started:
# Set by require_review_auth (absent for review-token links).
auth_state = getattr(request, "state", None)
@@ -978,6 +989,8 @@ async def get_correction_data(
logger.info(f"Job {job_id}: Serving correction data (replay={replay}) for combined review")
return corrections_data
except HTTPException:
raise
except Exception as e:
logger.error(f"Job {job_id}: Error loading corrections: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=t("en", "review.correctionsLoadFailed", error=str(e)))
+37
View File
@@ -9,6 +9,7 @@ import logging
import threading
import time
from contextlib import asynccontextmanager
from typing import Any
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
@@ -95,6 +96,9 @@ _LAZY_ROUTE_IMPORTS = (
"flacfetch.core.manager",
"flacfetch.downloaders.youtube",
"yt_dlp",
# starlette/middleware/base.py picks the anyio backend on the first request:
# 17.3s on the loop with 18 requests in flight (2026-10-30).
"anyio._backends._asyncio",
)
@@ -137,6 +141,29 @@ def _warm_route_contexts(application: FastAPI) -> int:
return count
def _warm_middleware_stack(application: Any) -> None:
"""Send one request through the app, so the first user's request isn't what builds
the middleware stack and runs its first-use imports on the loop.
Blocking: runs its own event loop, so call it from a worker thread. /api/health is
exempt from the edge lock, skips the audit log and touches no storage. Best-effort.
"""
import httpx
async def _request() -> None:
transport = httpx.ASGITransport(app=application)
async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client:
await client.get("/api/health")
start = time.time()
try:
asyncio.run(_request())
except Exception as e: # noqa: BLE001 - warmup must never block startup
logger.warning(f"Middleware warm request failed (stack builds on first request): {e}")
return
logger.info(f"Warmed middleware stack in {time.time() - start:.2f}s")
def _warm_route_contexts_safely(application: FastAPI) -> None:
start = time.time()
try:
@@ -243,6 +270,16 @@ async def lifespan(app: FastAPI):
f"Route-context warmup exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; continuing startup"
)
# Same reason, for what only the first request through the middleware touches.
try:
await asyncio.wait_for(
asyncio.to_thread(_warm_middleware_stack, app), timeout=LAZY_ROUTE_IMPORT_TIMEOUT_S
)
except asyncio.TimeoutError:
logger.warning(
f"Middleware warmup exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; continuing startup"
)
# NLP model / credential warmup runs in the background so it doesn't gate
# readiness — Cloud Run holds all routed requests until lifespan startup
# returns, and these preloads were ~7.5s of the ~16s cold start.
+8 -3
View File
@@ -222,7 +222,9 @@ class CredentialManager:
message="Dropbox credentials refreshed successfully",
last_checked=datetime.utcnow()
)
else:
logger.error("Dropbox access token rejected and no refresh credentials are stored")
return CredentialCheckResult(
service="dropbox",
status=CredentialStatus.INVALID,
@@ -410,7 +412,9 @@ class CredentialManager:
return True
except Exception as e:
logger.error(f"Dropbox API test failed: {e}")
# Short-lived tokens expire constantly and the caller refreshes them, so
# the probe failing isn't page-worthy; the refresh failing is.
logger.warning(f"Dropbox API test failed: {e}")
return False
def _refresh_dropbox_token(self, creds: dict) -> bool:
@@ -436,7 +440,8 @@ class CredentialManager:
# Update in Secret Manager
self._update_dropbox_credentials(creds)
return True
logger.error(f"Dropbox token refresh rejected ({response.status_code}): {response.text[:200]}")
return False
except Exception as e:
@@ -74,6 +74,20 @@ def is_opaque_script_error(message: str | None, source: str | None = None, extra
return not extra.get("filename") and not extra.get("lineno")
# Zalo's in-app browser injects scripts into every page it opens; one references a
# global it hasn't defined yet. Not our code, and nothing the user sees. Needs both
# the host app's UA and its global, so a real error inside Zalo still reports.
_HOST_APP_INJECTION_RE = re.compile(r"\bzaloJSV\d*\b is not defined")
_ZALO_UA_RE = re.compile(r"\bZalo\b", re.IGNORECASE)
def is_host_app_injection_error(message: str | None, user_agent: str | None) -> bool:
"""True for an error from a script the in-app browser's host app injected."""
if not message or not user_agent:
return False
return bool(_HOST_APP_INJECTION_RE.search(message) and _ZALO_UA_RE.search(user_agent))
# A chunk-load failure is a tab running an old build after a deploy: the client
# auto-reloads (frontend/lib/version-check.ts), so one report is not actionable.
# Mirrors isChunkLoadError() there. Reports collapse into one pattern per build
+46 -2
View File
@@ -15,6 +15,7 @@ Usage:
import re
from dataclasses import dataclass
from urllib.parse import urlsplit
@dataclass
@@ -129,18 +130,55 @@ _SERVICE_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str]] = [
"while instances start, GitHub redelivers and the runners still dispatch "
"(seen 2026-09-20, 09-23, 09-27, 10-08)",
),
(
"audio_separation_experiment_traceback",
frozenset({"audio-separation-job"}),
# Two consecutive <string> frames: `python -c` running exec'd code, i.e. an
# experiment on a prod job image. A multiprocessing-spawn traceback also
# starts with a <string> frame, but its second frame is a real file.
re.compile(r'File "<string>", line 1, in <module>\s+File "<string>", line \d+'),
"Ad-hoc experiment run on a prod audio-separation-job image, not a job failure "
"(seen 2026-10-31, 11-01)",
),
(
"url_download_failed_duplicate",
frozenset({"karaoke-backend"}),
# file_upload.py logs this, then fail_job: ops_alerts' job-failed alert is the
# one alert for the failure.
re.compile(r"^URL download failed: "),
"Same failure as the job-failed alert that fail_job sends right after this line",
),
]
def should_ignore(service: str, message: str) -> IgnoreReason | None:
# Noise only for one service AND request URL path (Cloud Run request logs carry the
# URL outside the message). (pattern_name, services, message regex, exact URL path, reason)
_SERVICE_URL_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str, str]] = [
(
"decide_search_health_abort",
frozenset({"karaoke-decide-search"}),
re.compile(
r"request was aborted after waiting too long for an available instance",
re.IGNORECASE,
),
"/api/search/health",
"The keep-warm ping waiting on a min-0 cold start (seen 2026-10-30). The same "
"abort on a real search path drops a user's request and still alerts.",
),
]
def should_ignore(service: str, message: str, request_url: str | None = None) -> IgnoreReason | None:
"""Check whether a log entry matches a known infrastructure noise pattern.
``_IGNORE_PATTERNS`` match on the message alone; ``_SERVICE_IGNORE_PATTERNS``
also require ``service`` to be one of the listed services.
also require ``service`` to be one of the listed services, and
``_SERVICE_URL_IGNORE_PATTERNS`` also require ``request_url``'s path.
Args:
service: The originating service name (e.g. ``"karaoke-backend"``).
message: The raw log message string to evaluate.
request_url: The entry's request URL, if it is a request log.
Returns:
An :class:`IgnoreReason` if the message matches a known noise pattern,
@@ -157,4 +195,10 @@ def should_ignore(service: str, message: str) -> IgnoreReason | None:
if service in services and pattern.search(message):
return IgnoreReason(pattern_name=pattern_name, reason=reason)
if request_url:
url_path = urlsplit(request_url).path
for pattern_name, services, pattern, path, reason in _SERVICE_URL_IGNORE_PATTERNS:
if service in services and url_path == path and pattern.search(message):
return IgnoreReason(pattern_name=pattern_name, reason=reason)
return None
+67 -3
View File
@@ -15,6 +15,7 @@ from __future__ import annotations
import logging
import os
import re
from datetime import datetime, timedelta, timezone
from typing import Any
@@ -115,6 +116,39 @@ def _build_log_filter(
)
#: Fixed messages uvicorn and the audit middleware log for every unhandled exception.
#: The cause is only in the entry's ``exception`` field.
_UNHANDLED_EXCEPTION_MESSAGES = frozenset({"Exception in ASGI application", "request_audit_error"})
#: The margin of an ExceptionGroup traceback (" | ", " | | ", " + ").
_TRACEBACK_MARGIN_RE = re.compile(r"^\s*(?:[|+] )+")
#: Its separators (" +-+---- 1 ----", " +-------").
_TRACEBACK_SEPARATOR_RE = re.compile(r"^\s*(?:\| )*\+[-+]")
_TRACEBACK_FRAME_RE = re.compile(r'^\s*File ".*", line \d+')
def _exception_summary(exception_text: str) -> str | None:
"""The ``Type: message`` line a traceback ends on, without its group margin.
Looks after the last frame, not at the last line: a multi-line exception message
(pydantic's ends in a docs link shared by every failure) would otherwise replace
the cause. Frame source and caret lines are indented; the exception line isn't.
"""
lines = [
_TRACEBACK_MARGIN_RE.sub("", line)
for line in exception_text.splitlines()
if line.strip() and not _TRACEBACK_SEPARATOR_RE.match(line)
]
if not lines:
return None
last_frame = max((i for i, line in enumerate(lines) if _TRACEBACK_FRAME_RE.match(line)), default=None)
if last_frame is not None:
for line in lines[last_frame + 1 :]:
if not line[0].isspace():
return line
return lines[-1].strip()
def _extract_message(entry: Any) -> str | None:
"""Extract the log message text from a Cloud Logging entry.
@@ -125,6 +159,9 @@ def _extract_message(entry: Any) -> str | None:
``"msg"`` keys in that order.
* ``None`` payload — returns ``None``.
An unhandled-exception message becomes ``Unhandled exception: <cause>``, so
each cause is its own pattern and both log lines of one crash share it.
Args:
entry: A Cloud Logging ``LogEntry`` object.
@@ -140,13 +177,40 @@ def _extract_message(entry: Any) -> str | None:
for key in ("message", "textPayload", "error", "msg"):
value = payload.get(key)
if value:
return str(value)
text = str(value)
if text.strip() in _UNHANDLED_EXCEPTION_MESSAGES:
summary = _exception_summary(str(payload.get("exception") or ""))
if summary:
return f"Unhandled exception: {summary}"
return text
return None
# Fallback: convert to string representation
text = str(payload)
return text if text else None
def _extract_sample(entry: Any, message: str) -> str:
"""The alert's sample: the message, led by the request path when the entry has one.
The path stays out of the pattern (one cause on many jobs or endpoints is one
alert) but the alert needs it to say where to look.
"""
payload = entry.payload
path = payload.get("path") if isinstance(payload, dict) else None
if not path:
return message
method = payload.get("method")
return f"{method} {path}\n{message}" if method else f"{path}\n{message}"
def _extract_request_url(entry: Any) -> str | None:
"""The request URL on a Cloud Run request-log entry, if any."""
http_request = getattr(entry, "http_request", None)
if not isinstance(http_request, dict):
return None
return http_request.get("requestUrl") or None
def _extract_service_name(entry: Any, resource_type: str) -> str:
"""Extract the service/job/function/instance name from a log entry.
@@ -370,7 +434,7 @@ class ErrorMonitor:
if not message:
continue
service = _extract_service_name(entry, resource_type)
ignore_reason = should_ignore(service, message)
ignore_reason = should_ignore(service, message, _extract_request_url(entry))
if ignore_reason:
logger.debug(
"Ignoring entry from %s (%s): %s",
@@ -387,7 +451,7 @@ class ErrorMonitor:
"service": service,
"resource_type": _classify_resource_type(resource_type),
"normalized": normalized,
"message": message,
"message": _extract_sample(entry, message),
}
)
except Exception as exc: # noqa: BLE001
@@ -74,6 +74,17 @@ _NORMALIZERS: list[tuple[re.Pattern, str]] = [
re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"),
r"https://\1",
),
# 8b. YouTube video IDs: yt-dlp's "[youtube] <id>: <error>" and bare watch links.
# Otherwise the same error on each video is a new pattern and a "first time"
# job-failure signature (ops_alerts._signature uses this normalizer).
(
re.compile(r"(\[youtube(?::[\w-]+)?\]) [A-Za-z0-9_-]{11}(?=:)"),
r"\1 <VIDEO_ID>",
),
(
re.compile(r"(youtube\.com/watch\?v=|youtu\.be/)[A-Za-z0-9_-]{11}(?![A-Za-z0-9_-])"),
r"\1<VIDEO_ID>",
),
# 9. Job IDs in paths (/jobs/<alphanumeric>)
(
re.compile(r"/jobs/[A-Za-z0-9]+"),
+4 -2
View File
@@ -1346,7 +1346,8 @@ class JobManager:
message=error_message
)
logger.error(f"Job {job_id} failed: {error_message}")
# WARNING: the status write above sends the one job-failed alert (ops_alerts).
logger.warning(f"Job {job_id} failed: {error_message}")
# Auto-refund credit on failure
job = self.get_job(job_id)
@@ -1693,7 +1694,8 @@ class JobManager:
message=f"Failed: {error_message}",
**updates
)
logger.error(f"Job {job_id} failed: {error_message}")
# WARNING: the status write above sends the one job-failed alert (ops_alerts).
logger.warning(f"Job {job_id} failed: {error_message}")
if not already_notified:
self._trigger_state_notifications(job_id, JobStatus.FAILED)
+29
View File
@@ -142,3 +142,32 @@ def test_strips_sensitive_query_string(client):
(pd,), _ = client.fake_adapter.upsert_pattern.call_args
assert "SECRET" not in pd.sample_message
assert "token=" not in pd.sample_message
ZALO_UA = (
"Mozilla/5.0 (Linux; Android 16; X Build/Y;) AppleWebKit/537.36 (KHTML, like Gecko) "
"Version/4.0 Chrome/156.0.8078.47 Mobile Safari/537.36 Zalo android/261002903 ZaloTheme/light"
)
ZALO_MESSAGE = "ReferenceError: zaloJSV2 is not defined"
def test_zalo_host_app_injection_is_accepted_but_not_persisted(client):
resp = client.post(
"/api/client-errors",
json=_payload(message=ZALO_MESSAGE, stack=f"{ZALO_MESSAGE}\n at <anonymous>:1:2", user_agent=ZALO_UA),
)
assert resp.status_code == 202
assert resp.json()["is_new"] is False
client.fake_adapter.upsert_pattern.assert_not_called()
def test_real_errors_inside_zalo_are_persisted(client):
resp = client.post("/api/client-errors", json=_payload(user_agent=ZALO_UA))
assert resp.status_code == 202
client.fake_adapter.upsert_pattern.assert_called_once()
def test_zalo_message_from_another_browser_is_persisted(client):
resp = client.post("/api/client-errors", json=_payload(message=ZALO_MESSAGE))
assert resp.status_code == 202
client.fake_adapter.upsert_pattern.assert_called_once()
@@ -1085,6 +1085,17 @@ class TestTorrentStallAwaitsUser:
assert details["stall_minutes"] == 60
assert details["keep_trying"] is True
@pytest.mark.asyncio
async def test_known_download_failure_logs_warning_not_error(self, caplog):
"""fail_job's job-failed alert is the one alert for this failure."""
import logging
flacfetch = _mock_flacfetch(FlacfetchServiceError("Download failed: tracker 500"))
with caplog.at_level(logging.INFO, logger="backend.workers.audio_download_worker"):
await self._run(_red_job(), flacfetch, attempt="0")
records = [r for r in caplog.records if "Download failed: Download failed" in r.getMessage()]
assert [r.levelno for r in records] == [logging.WARNING]
@pytest.mark.asyncio
async def test_non_stall_failure_still_marks_retry_pending(self):
flacfetch = _mock_flacfetch(FlacfetchServiceError("Download failed: tracker 500"))
+60
View File
@@ -1,6 +1,7 @@
"""
Tests for credential manager and auth endpoints.
"""
import logging
import pytest
import json
from unittest.mock import MagicMock, patch
@@ -194,6 +195,65 @@ class TestCredentialManager:
assert result is False
def test_expired_dropbox_token_probe_logs_warning_not_error(self, caplog):
"""Short-lived tokens expire constantly and get refreshed; that isn't page-worthy."""
manager = CredentialManager()
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")):
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
assert manager._test_dropbox_api({"access_token": "test"}) is False
levels = [r.levelno for r in caplog.records if "Dropbox" in r.message]
assert levels == [logging.WARNING]
def _dropbox_manager(self, creds):
manager = CredentialManager()
manager.settings = MagicMock()
manager.settings.get_secret.return_value = json.dumps(creds)
return manager
FULL_DROPBOX_CREDS = {"access_token": "a", "refresh_token": "r", "app_key": "k", "app_secret": "s"}
def test_dropbox_refresh_success_logs_no_error(self, caplog):
manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS)
ok = MagicMock(status_code=200)
ok.json.return_value = {"access_token": "new"}
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \
patch('requests.post', return_value=ok), \
patch.object(manager, "_update_dropbox_credentials", return_value=True):
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
result = manager.check_dropbox_credentials()
assert result.status == CredentialStatus.VALID
assert not [r for r in caplog.records if r.levelno >= logging.ERROR]
def test_dropbox_refresh_rejected_logs_error(self, caplog):
manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS)
bad = MagicMock(status_code=400, text='{"error": "invalid_grant"}')
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \
patch('requests.post', return_value=bad):
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
result = manager.check_dropbox_credentials()
assert result.status == CredentialStatus.INVALID
errors = [r.message for r in caplog.records if r.levelno >= logging.ERROR]
assert len(errors) == 1
assert "400" in errors[0]
assert "invalid_grant" in errors[0]
def test_dropbox_refresh_raising_logs_one_error(self, caplog):
manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS)
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \
patch('requests.post', side_effect=RuntimeError("network down")):
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
result = manager.check_dropbox_credentials()
assert result.status == CredentialStatus.INVALID
assert len([r for r in caplog.records if r.levelno >= logging.ERROR]) == 1
def test_dropbox_invalid_with_no_refresh_token_logs_error(self, caplog):
manager = self._dropbox_manager({"access_token": "a"})
with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")):
with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"):
result = manager.check_dropbox_credentials()
assert result.status == CredentialStatus.INVALID
assert len([r for r in caplog.records if r.levelno >= logging.ERROR]) == 1
def test_send_credential_alert_no_webhook(self):
"""Test alert without webhook URL."""
manager = CredentialManager()
+18
View File
@@ -290,6 +290,24 @@ class TestJobStatusTransitions:
class TestJobFailure:
"""Test job failure handling."""
def test_fail_job_logs_warning_not_error(self, job_manager, mock_firestore_service, caplog):
"""ops_alerts' job-failed alert is the one alert; an ERROR here made a second one."""
import logging
mock_firestore_service.get_job.return_value = None
with caplog.at_level(logging.INFO, logger="backend.services.job_manager"):
job_manager.fail_job("job-1", "Download failed: boom")
records = [r for r in caplog.records if "job-1 failed" in r.getMessage()]
assert [r.levelno for r in records] == [logging.WARNING]
def test_mark_job_failed_logs_warning_not_error(self, job_manager, mock_firestore_service, caplog):
import logging
with caplog.at_level(logging.INFO, logger="backend.services.job_manager"):
job_manager.mark_job_failed("job-2", "boom")
records = [r for r in caplog.records if "job-2 failed" in r.getMessage()]
assert [r.levelno for r in records] == [logging.WARNING]
def test_mark_job_as_failed(self, job_manager, mock_firestore_service):
"""Test marking a job as failed."""
error_message = "Audio separation failed"
+21 -3
View File
@@ -440,12 +440,30 @@ class TestStartReview:
assert response.json() == {"status": "success", "job_status": "in_review"}
mock_job_manager.record_review_started.assert_not_called()
def test_start_review_invalid_transition_otherwise_still_errors(self, client, mock_job_manager, mock_job, auth_headers):
def test_start_review_invalid_transition_race_returns_409(self, client, mock_job_manager, mock_job, auth_headers):
"""The auto-approver moved the job past review between our read and the transition."""
from backend.exceptions import InvalidStateTransitionError
mock_job.status = JobStatus.AWAITING_REVIEW
moved_on = mock_job.model_copy(update={"status": JobStatus.REVIEW_COMPLETE})
mock_job_manager.get_job.side_effect = [mock_job, moved_on, moved_on]
mock_job_manager.transition_to_state.side_effect = InvalidStateTransitionError(message="nope")
with pytest.raises(InvalidStateTransitionError):
client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers)
response = client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers)
assert response.status_code == 409
assert "review_complete" in response.json()["detail"]
mock_job_manager.record_review_started.assert_not_called()
@pytest.mark.parametrize("status", [JobStatus.COMPLETE, JobStatus.FAILED, JobStatus.PENDING])
def test_start_review_outside_review_returns_409_without_transition(
self, client, mock_job_manager, mock_job, auth_headers, status
):
"""A stale "review lyrics" push on a finished job: no 500, no transition, no ERROR log."""
mock_job.status = status
response = client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers)
assert response.status_code == 409
assert status.value in response.json()["detail"]
assert "JobStatus." not in response.json()["detail"]
mock_job_manager.transition_to_state.assert_not_called()
mock_job_manager.record_review_started.assert_not_called()
def test_start_review_from_awaiting_records_once(self, client, mock_job_manager, mock_job, auth_headers):
mock_job.status = JobStatus.AWAITING_REVIEW
+64
View File
@@ -181,3 +181,67 @@ def test_lifespan_warms_route_contexts_off_loop_before_readiness():
assert seen[0][0] is main_module.app
assert seen[0][1].startswith("asyncio_"), seen
def test_anyio_backend_is_a_lazy_route_import():
"""starlette/middleware/base.py imports it on the first request: 17.3s on the loop
with 18 requests in flight, 2026-10-30."""
assert "anyio._backends._asyncio" in main_module._LAZY_ROUTE_IMPORTS
def test_warm_request_runs_through_the_whole_middleware_stack_off_loop():
from starlette.applications import Starlette
from starlette.middleware import Middleware
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.responses import PlainTextResponse
from starlette.routing import Route
seen = {}
class Spy(BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
seen["thread"] = threading.current_thread().name
seen["path"] = request.url.path
return await call_next(request)
async def health(_request):
return PlainTextResponse("ok")
app = Starlette(routes=[Route("/api/health", health)], middleware=[Middleware(Spy)])
assert app.middleware_stack is None
worker = threading.Thread(target=main_module._warm_middleware_stack, args=(app,), name="warm-test")
worker.start()
worker.join(timeout=10)
assert seen == {"thread": "warm-test", "path": "/api/health"}
assert app.middleware_stack is not None
def test_warm_request_failure_never_blocks_startup():
async def broken_app(scope, receive, send):
raise RuntimeError("boom")
main_module._warm_middleware_stack(broken_app) # must not raise
def test_lifespan_warms_the_middleware_stack_before_readiness():
calls = []
def fake_warm(application):
calls.append(application)
with patch.object(main_module, "_run_background_warmup"), \
patch.object(main_module, "_warm_middleware_stack", fake_warm):
with TestClient(main_module.app):
assert calls == [main_module.app]
def test_warm_request_on_the_real_app_returns_healthy_and_skips_the_audit_log(caplog):
import logging
with caplog.at_level(logging.INFO):
main_module._warm_middleware_stack(main_module.app)
assert main_module.app.middleware_stack is not None
assert not [r for r in caplog.records if r.name == "audit"]
assert not [r for r in caplog.records if "warm request failed" in r.message.lower()]
+29
View File
@@ -1738,6 +1738,35 @@ class TestReplayCorrectionData:
finally:
self._teardown()
def _awaiting_job(self):
from backend.models.job import JobStatus
job = self._completed_job()
job.status = JobStatus.AWAITING_REVIEW
job.state_data = {}
return job
def test_auto_approver_race_returns_409_not_500(self):
"""Job read as awaiting_review, then auto-approved before our transition."""
from backend.exceptions import InvalidStateTransitionError
from backend.models.job import JobStatus
client = self._client_with_auth("full")
try:
with patch("backend.api.routes.review.JobManager") as jm, \
patch("backend.api.routes.review.StorageService") as storage_cls:
moved_on = self._awaiting_job()
moved_on.status = JobStatus.REVIEW_COMPLETE
jm.return_value.get_job.side_effect = [self._awaiting_job(), moved_on, moved_on]
jm.return_value.transition_to_state.side_effect = InvalidStateTransitionError(message="nope")
storage_cls.return_value.file_exists.return_value = True
storage_cls.return_value.download_json.return_value = {"corrected_segments": []}
resp = client.get("/api/review/job1/correction-data")
assert resp.status_code == 409
assert "review_complete" in resp.json()["detail"]
jm.return_value.record_review_started.assert_not_called()
finally:
self._teardown()
def test_replay_requires_full_auth(self):
client = self._client_with_auth("review_token")
try:
+2 -1
View File
@@ -399,7 +399,8 @@ async def process_audio_download(job_id: str) -> bool:
if not isinstance(e, (DownloadError, FlacfetchServiceError)):
logger.error(f"[job:{job_id}] Download failed: {e}", exc_info=True)
else:
logger.error(f"[job:{job_id}] Download failed: {e}")
# Expected failure type; fail_job below sends the one job-failed alert.
logger.warning(f"[job:{job_id}] Download failed: {e}")
_mark_retry_pending_if_attempts_remain(job_manager, job_id)
job_manager.fail_job(job_id, f"Audio download failed: {e}",
error_details={'stage': 'audio_download'})
+8 -1
View File
@@ -117,7 +117,14 @@ Keeping #alerts readable (one root cause → one message):
`ended` line is ERROR (≥10s blocked with a request affected), so one freeze is
one pattern. The `backend/event_loop_stalls` metric keys on that line too.
- **Known noise** (`known_issues.py`) can be service-scoped, e.g.
github-runner-manager's scale-up "no available instance" aborts.
github-runner-manager's scale-up "no available instance" aborts, and scoped to a
request path too (decide-search's `/api/search/health` aborts).
- **Unhandled exceptions**: `Exception in ASGI application` and `request_audit_error`
become one `Unhandled exception: <Type: message>` pattern per cause, read from the
entry's `exception` field; the path goes in the sample only.
- **One alert per job failure**: `ops_alerts` sends it from the status write, so
`Job <id> failed` and the download-failure lines before it are not ERROR patterns.
YouTube video ids are normalized out of signatures.
Frontend crashes are captured via `window.onerror`, `unhandledrejection`, Next.js
`error.tsx`, and a `CrashReportBoundary` around the review UIs. They POST to
@@ -110,6 +110,13 @@ class SpotifyProvider(BaseLyricsProvider):
self.logger.debug(f"Making RapidAPI lyrics request for track ID {track_id}")
lyrics_response = requests.get(lyrics_url, headers=headers, params=lyrics_params, timeout=10)
if lyrics_response.status_code == 403:
# The search just succeeded with the same key, so the key is fine: a 401/403
# on /search stays the dead-key ERROR. Other providers fill in.
self.logger.warning(
f"Spotify RapidAPI lyrics step returned 403 for track {track_id}: {lyrics_response.text[:200]}"
)
return None
lyrics_response.raise_for_status()
lyrics_data = lyrics_response.json()
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.290.0"
version = "0.290.1"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
@@ -585,3 +585,111 @@ class TestRunnerManagerNoAvailableInstance:
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("github-runner-manager", "Failed to create runner VM: quota exceeded") is None
class TestAudioSeparationExperimentTracebacks:
"""Ad-hoc `python -c` experiments on prod job images (2026-10-31, 11-01) are not
job failures: two consecutive <string> frames."""
EXPERIMENT = (
"Traceback (most recent call last):\n"
' File "<string>", line 1, in <module>\n'
' File "<string>", line 250, in <module>\n'
' File "<string>", line 228, in forced_align\n'
"OverflowError: Python integer 1162 out of bounds for int8"
)
SPAWN = (
"Traceback (most recent call last):\n"
' File "<string>", line 1, in <module>\n'
' File "/usr/local/lib/python3.11/multiprocessing/spawn.py", line 122, in spawn_main\n'
" exitcode = _main(fd, parent_sentinel)\n"
"RuntimeError: a real worker crash"
)
def test_experiment_ignored_on_the_job(self):
from backend.services.error_monitor.known_issues import should_ignore
result = should_ignore("audio-separation-job", self.EXPERIMENT)
assert result is not None
assert result.pattern_name == "audio_separation_experiment_traceback"
def test_multiprocessing_spawn_traceback_still_alerts(self):
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("audio-separation-job", self.SPAWN) is None
def test_same_traceback_on_other_services_still_alerts(self):
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("karaoke-backend", self.EXPERIMENT) is None
def test_extract_message_does_not_truncate_before_should_ignore(self):
from unittest.mock import MagicMock
from backend.services.error_monitor.known_issues import should_ignore
from backend.services.error_monitor.monitor import _extract_message
entry = MagicMock()
entry.payload = "x" * 5000 + "\n" + self.EXPERIMENT
assert should_ignore("audio-separation-job", _extract_message(entry)) is not None
class TestUrlDownloadFailedDuplicate:
"""fail_job follows this line, and ops_alerts' job-failed alert is the one alert."""
MSG = "URL download failed: Remote download failed: ERROR: [youtube] abcdefghijk: Sign in to confirm"
def test_ignored_on_the_backend(self):
from backend.services.error_monitor.known_issues import should_ignore
result = should_ignore("karaoke-backend", self.MSG)
assert result is not None
assert result.pattern_name == "url_download_failed_duplicate"
def test_other_download_errors_still_alert(self):
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("karaoke-backend", "GCS upload failed: URL download failed: x") is None
assert should_ignore("karaoke-backend", "Download failed: something") is None
class TestDecideSearchHealthAborts:
MSG = (
"The request was aborted after waiting too long for an available instance. "
"Additional troubleshooting documentation can be found at: "
"https://cloud.google.com/run/docs/troubleshooting#abort-request"
)
HEALTH = "https://karaoke-decide-search-x.a.run.app/api/search/health"
def test_health_probe_ignored(self):
from backend.services.error_monitor.known_issues import should_ignore
result = should_ignore("karaoke-decide-search", self.MSG, self.HEALTH)
assert result is not None
assert result.pattern_name == "decide_search_health_abort"
def test_health_probe_with_query_string_ignored(self):
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("karaoke-decide-search", self.MSG, self.HEALTH + "?ping=1") is not None
def test_real_search_paths_still_alert(self):
from backend.services.error_monitor.known_issues import should_ignore
for url in (
"https://karaoke-decide-search-x.a.run.app/api/search?q=hello",
"https://karaoke-decide-search-x.a.run.app/api/search/health/deep",
"https://karaoke-decide-search-x.a.run.app/api/search/songs?q=/api/search/health",
):
assert should_ignore("karaoke-decide-search", self.MSG, url) is None
def test_no_url_still_alerts(self):
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("karaoke-decide-search", self.MSG) is None
def test_other_services_and_messages_still_alert(self):
from backend.services.error_monitor.known_issues import should_ignore
assert should_ignore("karaoke-decide", self.MSG, self.HEALTH) is None
assert should_ignore("karaoke-decide-search", "boom", self.HEALTH) is None
@@ -178,6 +178,243 @@ class TestExtractMessage:
assert result is None or result == ""
# Invented paths/ids; the shapes are copied from real prod entries (2026-11-03).
_PLAIN_TRACEBACK = (
"Traceback (most recent call last):\n"
' File "/app/backend/middleware/audit_logging.py", line 63, in dispatch\n'
" response = await call_next(request)\n"
" ^^^^^^^^^^^^^^^^^^^^^^^^\n"
' File "/app/backend/services/job_manager.py", line 657, in validate_state_transition\n'
" raise InvalidStateTransitionError(\n"
"backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: "
"complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']\n"
)
_GROUP_TRACEBACK = (
" + Exception Group Traceback (most recent call last):\n"
' | File "/usr/local/lib/python3.11/site-packages/starlette/_utils.py", line 104, in create_collapsing_task_group\n'
" | async with anyio.create_task_group() as tg:\n"
' | File "/usr/local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 847, in __aexit__\n'
" | raise BaseExceptionGroup(\n"
" | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)\n"
" +-+---------------- 1 ----------------\n"
" | Traceback (most recent call last):\n"
' | File "/app/backend/services/job_manager.py", line 657, in validate_state_transition\n'
" | raise InvalidStateTransitionError(\n"
" | backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: "
"complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']\n"
" +------------------------------------\n"
)
_EXPECTED_CAUSE = (
"Unhandled exception: backend.exceptions.InvalidStateTransitionError: Invalid state transition "
"for job 7f89f884: complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']"
)
class TestUnhandledExceptionMessages:
"""Every unhandled exception logs one of two fixed messages; the pattern must
come from the exception, or all crashes share one (already 'new') pattern and
a second, different crash never alerts."""
def _entry(self, message, exception, **extra):
entry = MagicMock()
entry.payload = {"message": message, "exception": exception, **extra}
return entry
def test_audit_error_plain_traceback(self):
m = _import_monitor_module()
entry = self._entry("request_audit_error", _PLAIN_TRACEBACK, path="/api/jobs/x/start-review", method="POST")
assert m._extract_message(entry) == _EXPECTED_CAUSE
def test_asgi_exception_group_traceback_with_pipe_prefixes(self):
m = _import_monitor_module()
entry = self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK)
assert m._extract_message(entry) == _EXPECTED_CAUSE
def test_nested_exception_group(self):
m = _import_monitor_module()
nested = (
" + Exception Group Traceback (most recent call last):\n"
" | ExceptionGroup: outer (1 sub-exception)\n"
" +-+---------------- 1 ----------------\n"
" | Exception Group Traceback (most recent call last):\n"
" | ExceptionGroup: inner (1 sub-exception)\n"
" +-+---------------- 1 ----------------\n"
" | Traceback (most recent call last):\n"
' | File "/app/a.py", line 1, in f\n'
" | boom()\n"
" | ValueError: the real cause\n"
" +------------------------------------\n"
" +------------------------------------\n"
)
assert m._extract_message(self._entry("request_audit_error", nested)) == (
"Unhandled exception: ValueError: the real cause"
)
def test_multiline_exception_message_uses_its_first_line(self):
# pydantic's last traceback line is a docs link, the same for every failure.
m = _import_monitor_module()
exc = (
"Traceback (most recent call last):\n"
' File "/app/a.py", line 1, in f\n'
" boom()\n"
"pydantic_core._pydantic_core.ValidationError: 1 validation error for Job\n"
"status\n"
" Input should be a valid string\n"
" For further information visit https://errors.pydantic.dev/2/v/string_type\n"
)
assert m._extract_message(self._entry("request_audit_error", exc)) == (
"Unhandled exception: pydantic_core._pydantic_core.ValidationError: 1 validation error for Job"
)
def test_chained_exception_uses_the_final_one(self):
m = _import_monitor_module()
exc = (
"Traceback (most recent call last):\n"
' File "/app/a.py", line 1, in f\n'
" boom()\n"
"KeyError: 'first'\n\n"
"During handling of the above exception, another exception occurred:\n\n"
"Traceback (most recent call last):\n"
' File "/app/b.py", line 2, in g\n'
" bang()\n"
"RuntimeError: second\n"
)
assert m._extract_message(self._entry("request_audit_error", exc)) == (
"Unhandled exception: RuntimeError: second"
)
def test_no_exception_field_keeps_the_message(self):
m = _import_monitor_module()
entry = MagicMock()
entry.payload = {"message": "request_audit_error"}
assert m._extract_message(entry) == "request_audit_error"
def test_unparseable_exception_keeps_last_line(self):
m = _import_monitor_module()
assert m._extract_message(self._entry("request_audit_error", "something odd\nno frames here\n")) == (
"Unhandled exception: no frames here"
)
def test_other_messages_with_exception_field_are_untouched(self):
m = _import_monitor_module()
entry = self._entry("Job 1 could not save", _PLAIN_TRACEBACK)
assert m._extract_message(entry) == "Job 1 could not save"
def test_both_log_lines_of_one_crash_collapse_to_one_pattern(self):
from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message
m = _import_monitor_module()
audit = m._extract_message(self._entry("request_audit_error", _PLAIN_TRACEBACK))
asgi = m._extract_message(self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK))
assert compute_pattern_hash("s", normalize_message(audit)) == compute_pattern_hash("s", normalize_message(asgi))
def test_two_different_crashes_are_two_patterns(self):
from backend.services.error_monitor.normalizer import normalize_message
m = _import_monitor_module()
other = _PLAIN_TRACEBACK.replace(
"backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: "
"complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']",
"ZeroDivisionError: division by zero",
)
first = m._extract_message(self._entry("request_audit_error", _PLAIN_TRACEBACK))
second = m._extract_message(self._entry("request_audit_error", other))
assert normalize_message(first) != normalize_message(second)
def test_sample_keeps_the_path_but_the_pattern_does_not(self):
m = _import_monitor_module()
entry = self._entry(
"request_audit_error", _PLAIN_TRACEBACK, path="/api/jobs/7f89f884/start-review", method="POST"
)
message = m._extract_message(entry)
sample = m._extract_sample(entry, message)
assert sample.startswith("POST /api/jobs/7f89f884/start-review\n")
assert sample.endswith(_EXPECTED_CAUSE)
assert "start-review" not in message
def test_sample_is_the_message_without_a_path(self):
m = _import_monitor_module()
entry = self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK)
message = m._extract_message(entry)
assert m._extract_sample(entry, message) == message
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
def test_cycle_groups_a_crash_into_one_alert_with_the_path_in_its_sample(self, _digest, _llm):
from backend.services.error_monitor.firestore_adapter import UpsertResult
m = _import_monitor_module()
audit = _make_entry(
payload={
"message": "request_audit_error",
"exception": _PLAIN_TRACEBACK,
"path": "/api/jobs/7f89f884/start-review",
"method": "POST",
}
)
asgi = _make_entry(payload={"message": "Exception in ASGI application\n", "exception": _GROUP_TRACEBACK})
audit.http_request = asgi.http_request = None
monitor = _make_monitor_with_mocks()
monitor.logging_client.list_entries.side_effect = [iter([audit, asgi])] + [iter([])] * (
len(m.RESOURCE_TYPE_FILTERS) - 1
)
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(pattern_id="p", is_new=True)
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
monitor.firestore_adapter.get_active_patterns.return_value = []
monitor.firestore_adapter.get_unalerted_new_patterns.return_value = []
monitor.discord.send_message.return_value = True
monitor._run_monitor_cycle()
assert monitor.firestore_adapter.upsert_pattern.call_count == 1
data = monitor.firestore_adapter.upsert_pattern.call_args[0][0]
assert data.count == 2
assert "/api/jobs/7f89f884/start-review" in data.sample_message
assert "start-review" not in data.normalized_message
assert "InvalidStateTransitionError" in data.normalized_message
assert monitor.discord.send_message.call_count == 1
class TestExtractRequestUrl:
def test_reads_the_cloud_run_request_url(self):
m = _import_monitor_module()
entry = MagicMock()
entry.http_request = {"requestUrl": "https://x.a.run.app/api/search/health"}
assert m._extract_request_url(entry) == "https://x.a.run.app/api/search/health"
def test_none_without_http_request(self):
m = _import_monitor_module()
entry = MagicMock()
entry.http_request = None
assert m._extract_request_url(entry) is None
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
def test_cycle_drops_health_aborts_but_alerts_real_search_paths(self, _digest, _llm):
from backend.services.error_monitor.firestore_adapter import UpsertResult
m = _import_monitor_module()
msg = "The request was aborted after waiting too long for an available instance."
health = _make_entry(payload=msg, service_name="karaoke-decide-search")
health.http_request = {"requestUrl": "https://x.a.run.app/api/search/health"}
real = _make_entry(payload=msg, service_name="karaoke-decide-search")
real.http_request = {"requestUrl": "https://x.a.run.app/api/search?q=hello"}
monitor = _make_monitor_with_mocks()
monitor.logging_client.list_entries.side_effect = [iter([health, real])] + [iter([])] * (
len(m.RESOURCE_TYPE_FILTERS) - 1
)
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(pattern_id="p", is_new=True)
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
monitor.firestore_adapter.get_active_patterns.return_value = []
monitor.firestore_adapter.get_unalerted_new_patterns.return_value = []
monitor.discord.send_message.return_value = True
monitor._run_monitor_cycle()
assert monitor.firestore_adapter.upsert_pattern.call_count == 1
assert monitor.firestore_adapter.upsert_pattern.call_args[0][0].count == 1
# ---------------------------------------------------------------------------
# Tests: _extract_service_name
# ---------------------------------------------------------------------------
@@ -452,3 +452,46 @@ def test_loop_stall_request_counts_collapse_to_one_pattern():
b = normalize_message("EVENT_LOOP_STALL ended duration_ms=81183 culprit=x.py:539 in f requests_affected=3")
assert a == b
assert "requests_affected=<N>" in a
class TestYouTubeVideoIds:
"""A video id in the text made every bot block a "first time" signature."""
def _bot_block(self, video_id):
return (
f"Download failed: ERROR: [youtube] {video_id}: Sign in to confirm you\u2019re not a bot. "
"Use --cookies-from-browser or --cookies for the authentication."
)
def test_same_error_on_different_videos_is_one_signature(self):
from backend.services.error_monitor.normalizer import normalize_message
assert normalize_message(self._bot_block("mBJivIsN0n0")) == normalize_message(self._bot_block("a_b-cDeFgHi"))
def test_id_replaced_with_a_placeholder(self):
from backend.services.error_monitor.normalizer import normalize_message
result = normalize_message(self._bot_block("mBJivIsN0n0"))
assert "[youtube] <VIDEO_ID>: Sign in" in result
assert "mBJivIsN0n0" not in result
def test_tab_extractor_variant(self):
from backend.services.error_monitor.normalizer import normalize_message
assert "[youtube:tab] <VIDEO_ID>:" in normalize_message("ERROR: [youtube:tab] mBJivIsN0n0: gone")
def test_bare_watch_url_id(self):
from backend.services.error_monitor.normalizer import normalize_message
result = normalize_message("Could not fetch youtube.com/watch?v=mBJivIsN0n0 today")
assert "mBJivIsN0n0" not in result
def test_youtube_words_without_an_id_are_kept(self):
from backend.services.error_monitor.normalizer import normalize_message
assert normalize_message("[youtube] Downloading webpage") == "[youtube] Downloading webpage"
def test_ops_alert_signature_ignores_the_video_id(self):
from backend.services.ops_alerts import _signature
assert _signature(self._bot_block("mBJivIsN0n0")) == _signature(self._bot_block("a_b-cDeFgHi"))
@@ -0,0 +1,65 @@
"""Log levels on the Spotify RapidAPI provider: ERROR pages a human, so a 403 on the
/lyrics step (the search just succeeded with the same key, other providers fill in)
is a WARNING, while 401/403 on /search stays the dead-key ERROR."""
import logging
from unittest.mock import MagicMock, patch
import pytest
import requests
from karaoke_gen.lyrics_transcriber.lyrics.base_lyrics_provider import LyricsProviderConfig
from karaoke_gen.lyrics_transcriber.lyrics.spotify import SpotifyProvider
SEARCH_OK = {"status": True, "errorId": "Success", "id": "track1"}
def _response(status, body=None, text=""):
resp = MagicMock()
resp.status_code = status
resp.text = text
resp.json.return_value = body or {}
if status >= 400:
err = requests.exceptions.HTTPError(f"{status} Client Error", response=resp)
resp.raise_for_status.side_effect = err
return resp
def _provider():
return SpotifyProvider(LyricsProviderConfig(rapidapi_key="test-key"), logger=logging.getLogger("spotify-test"))
def _levels(caplog):
return [r.levelno for r in caplog.records if r.name == "spotify-test"]
def test_lyrics_403_is_a_warning_with_the_body(caplog):
long_body = "not subscribed " + "x" * 500
with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(403, text=long_body)]):
with caplog.at_level(logging.INFO, logger="spotify-test"):
assert _provider()._fetch_from_rapidapi("A", "T") is None
assert logging.ERROR not in _levels(caplog)
warning = next(r.message for r in caplog.records if r.levelno == logging.WARNING)
assert "not subscribed" in warning
assert "x" * 201 not in warning
@pytest.mark.parametrize("status", [401, 403])
def test_search_auth_failure_stays_an_error(caplog, status):
with patch("requests.get", side_effect=[_response(status)]):
with caplog.at_level(logging.INFO, logger="spotify-test"):
assert _provider()._fetch_from_rapidapi("A", "T") is None
assert logging.ERROR in _levels(caplog)
def test_lyrics_401_stays_an_error(caplog):
with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(401)]):
with caplog.at_level(logging.INFO, logger="spotify-test"):
assert _provider()._fetch_from_rapidapi("A", "T") is None
assert logging.ERROR in _levels(caplog)
def test_lyrics_success_still_returns_data():
with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(200, {"lyrics": []})]):
result = _provider()._fetch_from_rapidapi("A", "T")
assert result["_rapidapi_source"] is True