diff --git a/backend/api/routes/client_errors.py b/backend/api/routes/client_errors.py index 6ca0fa1c..af61e8f1 100644 --- a/backend/api/routes/client_errors.py +++ b/backend/api/routes/client_errors.py @@ -19,6 +19,7 @@ from backend.services.error_monitor.frontend_ingestion import ( RateLimiter, build_pattern_data, is_bot_user_agent, + is_host_app_injection_error, is_opaque_script_error, sanitize_url, ) @@ -90,6 +91,15 @@ def report_client_error(payload: ClientErrorPayload, request: Request) -> Client ) return ClientErrorResponse(pattern_id="opaque-script-error-ignored", is_new=False) + # A script the in-app browser's host app injected (Zalo), not our code. + if is_host_app_injection_error(payload.message, payload.user_agent): + logger.info( + "frontend_crash_ignored_host_app_injection url=%s source=%s", + sanitize_url(payload.url), + payload.source, + ) + return ClientErrorResponse(pattern_id="host-app-injection-ignored", is_new=False) + report = FrontendErrorReport( message=payload.message, stack=payload.stack, diff --git a/backend/api/routes/jobs.py b/backend/api/routes/jobs.py index 27484880..02d091c2 100644 --- a/backend/api/routes/jobs.py +++ b/backend/api/routes/jobs.py @@ -1085,6 +1085,14 @@ def get_review_data( } +def _job_not_in_review_conflict(locale: str, status: Any) -> HTTPException: + """409 for a review request on a job that isn't in review.""" + return HTTPException( + status_code=409, + detail=t(locale, "jobs.jobNotInReviewState", status=getattr(status, "value", status)), + ) + + @router.post("/{job_id}/start-review") def start_review( job_id: str, @@ -1109,10 +1117,21 @@ def start_review( # Idempotent: concurrent/duplicate opens (React double-mount, double-tap, two # tabs) must not fail with "in_review -> in_review". Already in review == success, # and the one-time side effects (record_review_started) are not repeated. + # A job past review (e.g. a stale "review lyrics" push tapped on a finished song) + # is the client's to handle: 409, no transition attempted, nothing logged at ERROR. + if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]: + raise _job_not_in_review_conflict(locale, job.status) + was_awaiting = job.status == JobStatus.AWAITING_REVIEW - result = ensure_in_review( - job_manager, job_id, job.status, "User started reviewing lyrics" - ) + try: + result = ensure_in_review( + job_manager, job_id, job.status, "User started reviewing lyrics" + ) + except InvalidStateTransitionError: + # Lost the race: the auto-approver (or another worker) moved the job on + # between our read and the transition. + current = job_manager.get_job(job_id) + raise _job_not_in_review_conflict(locale, current.status if current else job.status) if result == "refused": raise HTTPException(status_code=400, detail=t(locale, "jobs.reviewCannotStart")) if result == "started" and was_awaiting: diff --git a/backend/api/routes/review.py b/backend/api/routes/review.py index 199704e0..58956ed1 100644 --- a/backend/api/routes/review.py +++ b/backend/api/routes/review.py @@ -32,6 +32,7 @@ from google.cloud.firestore_v1.field_path import FieldPath from backend.models.job import JobStatus from backend.models.review_session import ReviewSession, ReviewSessionSummary from backend.services.job_manager import JobManager +from backend.exceptions import InvalidStateTransitionError from backend.services.review_state import ensure_in_review from backend.services.firestore_service import FirestoreService from backend.services.storage_service import StorageService @@ -932,13 +933,23 @@ async def get_correction_data( if not replay and job.status == JobStatus.AWAITING_REVIEW: # Idempotent: iOS/in-app browsers fire two concurrent loads; the loser # must not 500 on "in_review -> in_review" nor repeat one-time side effects. - started = "started" == await asyncio.to_thread( - ensure_in_review, - job_manager, - job_id, - job.status, - "User opened combined review interface", - ) + try: + started = "started" == await asyncio.to_thread( + ensure_in_review, + job_manager, + job_id, + job.status, + "User opened combined review interface", + ) + except InvalidStateTransitionError: + # Lost the race: the auto-approver moved the job on after the status + # gate above. Not a load failure, so 409 rather than the 500 below. + current = await asyncio.to_thread(job_manager.get_job, job_id) + status = current.status if current else job.status + raise HTTPException( + status_code=409, + detail=t("en", "review.reviewNotReady", status=getattr(status, "value", status)), + ) if started: # Set by require_review_auth (absent for review-token links). auth_state = getattr(request, "state", None) @@ -978,6 +989,8 @@ async def get_correction_data( logger.info(f"Job {job_id}: Serving correction data (replay={replay}) for combined review") return corrections_data + except HTTPException: + raise except Exception as e: logger.error(f"Job {job_id}: Error loading corrections: {e}", exc_info=True) raise HTTPException(status_code=500, detail=t("en", "review.correctionsLoadFailed", error=str(e))) diff --git a/backend/main.py b/backend/main.py index 3349c153..4400e6d0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -9,6 +9,7 @@ import logging import threading import time from contextlib import asynccontextmanager +from typing import Any from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -95,6 +96,9 @@ _LAZY_ROUTE_IMPORTS = ( "flacfetch.core.manager", "flacfetch.downloaders.youtube", "yt_dlp", + # starlette/middleware/base.py picks the anyio backend on the first request: + # 17.3s on the loop with 18 requests in flight (2026-10-30). + "anyio._backends._asyncio", ) @@ -137,6 +141,29 @@ def _warm_route_contexts(application: FastAPI) -> int: return count +def _warm_middleware_stack(application: Any) -> None: + """Send one request through the app, so the first user's request isn't what builds + the middleware stack and runs its first-use imports on the loop. + + Blocking: runs its own event loop, so call it from a worker thread. /api/health is + exempt from the edge lock, skips the audit log and touches no storage. Best-effort. + """ + import httpx + + async def _request() -> None: + transport = httpx.ASGITransport(app=application) + async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client: + await client.get("/api/health") + + start = time.time() + try: + asyncio.run(_request()) + except Exception as e: # noqa: BLE001 - warmup must never block startup + logger.warning(f"Middleware warm request failed (stack builds on first request): {e}") + return + logger.info(f"Warmed middleware stack in {time.time() - start:.2f}s") + + def _warm_route_contexts_safely(application: FastAPI) -> None: start = time.time() try: @@ -243,6 +270,16 @@ async def lifespan(app: FastAPI): f"Route-context warmup exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; continuing startup" ) + # Same reason, for what only the first request through the middleware touches. + try: + await asyncio.wait_for( + asyncio.to_thread(_warm_middleware_stack, app), timeout=LAZY_ROUTE_IMPORT_TIMEOUT_S + ) + except asyncio.TimeoutError: + logger.warning( + f"Middleware warmup exceeded {LAZY_ROUTE_IMPORT_TIMEOUT_S}s; continuing startup" + ) + # NLP model / credential warmup runs in the background so it doesn't gate # readiness — Cloud Run holds all routed requests until lifespan startup # returns, and these preloads were ~7.5s of the ~16s cold start. diff --git a/backend/services/credential_manager.py b/backend/services/credential_manager.py index 50673bf1..598963f8 100644 --- a/backend/services/credential_manager.py +++ b/backend/services/credential_manager.py @@ -222,7 +222,9 @@ class CredentialManager: message="Dropbox credentials refreshed successfully", last_checked=datetime.utcnow() ) - + else: + logger.error("Dropbox access token rejected and no refresh credentials are stored") + return CredentialCheckResult( service="dropbox", status=CredentialStatus.INVALID, @@ -410,7 +412,9 @@ class CredentialManager: return True except Exception as e: - logger.error(f"Dropbox API test failed: {e}") + # Short-lived tokens expire constantly and the caller refreshes them, so + # the probe failing isn't page-worthy; the refresh failing is. + logger.warning(f"Dropbox API test failed: {e}") return False def _refresh_dropbox_token(self, creds: dict) -> bool: @@ -436,7 +440,8 @@ class CredentialManager: # Update in Secret Manager self._update_dropbox_credentials(creds) return True - + + logger.error(f"Dropbox token refresh rejected ({response.status_code}): {response.text[:200]}") return False except Exception as e: diff --git a/backend/services/error_monitor/frontend_ingestion.py b/backend/services/error_monitor/frontend_ingestion.py index 325f3255..a4c9462a 100644 --- a/backend/services/error_monitor/frontend_ingestion.py +++ b/backend/services/error_monitor/frontend_ingestion.py @@ -74,6 +74,20 @@ def is_opaque_script_error(message: str | None, source: str | None = None, extra return not extra.get("filename") and not extra.get("lineno") +# Zalo's in-app browser injects scripts into every page it opens; one references a +# global it hasn't defined yet. Not our code, and nothing the user sees. Needs both +# the host app's UA and its global, so a real error inside Zalo still reports. +_HOST_APP_INJECTION_RE = re.compile(r"\bzaloJSV\d*\b is not defined") +_ZALO_UA_RE = re.compile(r"\bZalo\b", re.IGNORECASE) + + +def is_host_app_injection_error(message: str | None, user_agent: str | None) -> bool: + """True for an error from a script the in-app browser's host app injected.""" + if not message or not user_agent: + return False + return bool(_HOST_APP_INJECTION_RE.search(message) and _ZALO_UA_RE.search(user_agent)) + + # A chunk-load failure is a tab running an old build after a deploy: the client # auto-reloads (frontend/lib/version-check.ts), so one report is not actionable. # Mirrors isChunkLoadError() there. Reports collapse into one pattern per build diff --git a/backend/services/error_monitor/known_issues.py b/backend/services/error_monitor/known_issues.py index 67126580..7fd6c2b9 100644 --- a/backend/services/error_monitor/known_issues.py +++ b/backend/services/error_monitor/known_issues.py @@ -15,6 +15,7 @@ Usage: import re from dataclasses import dataclass +from urllib.parse import urlsplit @dataclass @@ -129,18 +130,55 @@ _SERVICE_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str]] = [ "while instances start, GitHub redelivers and the runners still dispatch " "(seen 2026-09-20, 09-23, 09-27, 10-08)", ), + ( + "audio_separation_experiment_traceback", + frozenset({"audio-separation-job"}), + # Two consecutive frames: `python -c` running exec'd code, i.e. an + # experiment on a prod job image. A multiprocessing-spawn traceback also + # starts with a frame, but its second frame is a real file. + re.compile(r'File "", line 1, in \s+File "", line \d+'), + "Ad-hoc experiment run on a prod audio-separation-job image, not a job failure " + "(seen 2026-10-31, 11-01)", + ), + ( + "url_download_failed_duplicate", + frozenset({"karaoke-backend"}), + # file_upload.py logs this, then fail_job: ops_alerts' job-failed alert is the + # one alert for the failure. + re.compile(r"^URL download failed: "), + "Same failure as the job-failed alert that fail_job sends right after this line", + ), ] -def should_ignore(service: str, message: str) -> IgnoreReason | None: +# Noise only for one service AND request URL path (Cloud Run request logs carry the +# URL outside the message). (pattern_name, services, message regex, exact URL path, reason) +_SERVICE_URL_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str, str]] = [ + ( + "decide_search_health_abort", + frozenset({"karaoke-decide-search"}), + re.compile( + r"request was aborted after waiting too long for an available instance", + re.IGNORECASE, + ), + "/api/search/health", + "The keep-warm ping waiting on a min-0 cold start (seen 2026-10-30). The same " + "abort on a real search path drops a user's request and still alerts.", + ), +] + + +def should_ignore(service: str, message: str, request_url: str | None = None) -> IgnoreReason | None: """Check whether a log entry matches a known infrastructure noise pattern. ``_IGNORE_PATTERNS`` match on the message alone; ``_SERVICE_IGNORE_PATTERNS`` - also require ``service`` to be one of the listed services. + also require ``service`` to be one of the listed services, and + ``_SERVICE_URL_IGNORE_PATTERNS`` also require ``request_url``'s path. Args: service: The originating service name (e.g. ``"karaoke-backend"``). message: The raw log message string to evaluate. + request_url: The entry's request URL, if it is a request log. Returns: An :class:`IgnoreReason` if the message matches a known noise pattern, @@ -157,4 +195,10 @@ def should_ignore(service: str, message: str) -> IgnoreReason | None: if service in services and pattern.search(message): return IgnoreReason(pattern_name=pattern_name, reason=reason) + if request_url: + url_path = urlsplit(request_url).path + for pattern_name, services, pattern, path, reason in _SERVICE_URL_IGNORE_PATTERNS: + if service in services and url_path == path and pattern.search(message): + return IgnoreReason(pattern_name=pattern_name, reason=reason) + return None diff --git a/backend/services/error_monitor/monitor.py b/backend/services/error_monitor/monitor.py index 2421044d..50ba1057 100644 --- a/backend/services/error_monitor/monitor.py +++ b/backend/services/error_monitor/monitor.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging import os +import re from datetime import datetime, timedelta, timezone from typing import Any @@ -115,6 +116,39 @@ def _build_log_filter( ) +#: Fixed messages uvicorn and the audit middleware log for every unhandled exception. +#: The cause is only in the entry's ``exception`` field. +_UNHANDLED_EXCEPTION_MESSAGES = frozenset({"Exception in ASGI application", "request_audit_error"}) + +#: The margin of an ExceptionGroup traceback (" | ", " | | ", " + "). +_TRACEBACK_MARGIN_RE = re.compile(r"^\s*(?:[|+] )+") +#: Its separators (" +-+---- 1 ----", " +-------"). +_TRACEBACK_SEPARATOR_RE = re.compile(r"^\s*(?:\| )*\+[-+]") +_TRACEBACK_FRAME_RE = re.compile(r'^\s*File ".*", line \d+') + + +def _exception_summary(exception_text: str) -> str | None: + """The ``Type: message`` line a traceback ends on, without its group margin. + + Looks after the last frame, not at the last line: a multi-line exception message + (pydantic's ends in a docs link shared by every failure) would otherwise replace + the cause. Frame source and caret lines are indented; the exception line isn't. + """ + lines = [ + _TRACEBACK_MARGIN_RE.sub("", line) + for line in exception_text.splitlines() + if line.strip() and not _TRACEBACK_SEPARATOR_RE.match(line) + ] + if not lines: + return None + last_frame = max((i for i, line in enumerate(lines) if _TRACEBACK_FRAME_RE.match(line)), default=None) + if last_frame is not None: + for line in lines[last_frame + 1 :]: + if not line[0].isspace(): + return line + return lines[-1].strip() + + def _extract_message(entry: Any) -> str | None: """Extract the log message text from a Cloud Logging entry. @@ -125,6 +159,9 @@ def _extract_message(entry: Any) -> str | None: ``"msg"`` keys in that order. * ``None`` payload — returns ``None``. + An unhandled-exception message becomes ``Unhandled exception: ``, so + each cause is its own pattern and both log lines of one crash share it. + Args: entry: A Cloud Logging ``LogEntry`` object. @@ -140,13 +177,40 @@ def _extract_message(entry: Any) -> str | None: for key in ("message", "textPayload", "error", "msg"): value = payload.get(key) if value: - return str(value) + text = str(value) + if text.strip() in _UNHANDLED_EXCEPTION_MESSAGES: + summary = _exception_summary(str(payload.get("exception") or "")) + if summary: + return f"Unhandled exception: {summary}" + return text return None # Fallback: convert to string representation text = str(payload) return text if text else None +def _extract_sample(entry: Any, message: str) -> str: + """The alert's sample: the message, led by the request path when the entry has one. + + The path stays out of the pattern (one cause on many jobs or endpoints is one + alert) but the alert needs it to say where to look. + """ + payload = entry.payload + path = payload.get("path") if isinstance(payload, dict) else None + if not path: + return message + method = payload.get("method") + return f"{method} {path}\n{message}" if method else f"{path}\n{message}" + + +def _extract_request_url(entry: Any) -> str | None: + """The request URL on a Cloud Run request-log entry, if any.""" + http_request = getattr(entry, "http_request", None) + if not isinstance(http_request, dict): + return None + return http_request.get("requestUrl") or None + + def _extract_service_name(entry: Any, resource_type: str) -> str: """Extract the service/job/function/instance name from a log entry. @@ -370,7 +434,7 @@ class ErrorMonitor: if not message: continue service = _extract_service_name(entry, resource_type) - ignore_reason = should_ignore(service, message) + ignore_reason = should_ignore(service, message, _extract_request_url(entry)) if ignore_reason: logger.debug( "Ignoring entry from %s (%s): %s", @@ -387,7 +451,7 @@ class ErrorMonitor: "service": service, "resource_type": _classify_resource_type(resource_type), "normalized": normalized, - "message": message, + "message": _extract_sample(entry, message), } ) except Exception as exc: # noqa: BLE001 diff --git a/backend/services/error_monitor/normalizer.py b/backend/services/error_monitor/normalizer.py index d3f26851..4d9b7d67 100644 --- a/backend/services/error_monitor/normalizer.py +++ b/backend/services/error_monitor/normalizer.py @@ -74,6 +74,17 @@ _NORMALIZERS: list[tuple[re.Pattern, str]] = [ re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"), r"https://\1", ), + # 8b. YouTube video IDs: yt-dlp's "[youtube] : " and bare watch links. + # Otherwise the same error on each video is a new pattern and a "first time" + # job-failure signature (ops_alerts._signature uses this normalizer). + ( + re.compile(r"(\[youtube(?::[\w-]+)?\]) [A-Za-z0-9_-]{11}(?=:)"), + r"\1 ", + ), + ( + re.compile(r"(youtube\.com/watch\?v=|youtu\.be/)[A-Za-z0-9_-]{11}(?![A-Za-z0-9_-])"), + r"\1", + ), # 9. Job IDs in paths (/jobs/) ( re.compile(r"/jobs/[A-Za-z0-9]+"), diff --git a/backend/services/job_manager.py b/backend/services/job_manager.py index 8811b613..fc8d66ff 100644 --- a/backend/services/job_manager.py +++ b/backend/services/job_manager.py @@ -1346,7 +1346,8 @@ class JobManager: message=error_message ) - logger.error(f"Job {job_id} failed: {error_message}") + # WARNING: the status write above sends the one job-failed alert (ops_alerts). + logger.warning(f"Job {job_id} failed: {error_message}") # Auto-refund credit on failure job = self.get_job(job_id) @@ -1693,7 +1694,8 @@ class JobManager: message=f"Failed: {error_message}", **updates ) - logger.error(f"Job {job_id} failed: {error_message}") + # WARNING: the status write above sends the one job-failed alert (ops_alerts). + logger.warning(f"Job {job_id} failed: {error_message}") if not already_notified: self._trigger_state_notifications(job_id, JobStatus.FAILED) diff --git a/backend/tests/api/test_client_errors.py b/backend/tests/api/test_client_errors.py index 9e704131..d17a7261 100644 --- a/backend/tests/api/test_client_errors.py +++ b/backend/tests/api/test_client_errors.py @@ -142,3 +142,32 @@ def test_strips_sensitive_query_string(client): (pd,), _ = client.fake_adapter.upsert_pattern.call_args assert "SECRET" not in pd.sample_message assert "token=" not in pd.sample_message + + +ZALO_UA = ( + "Mozilla/5.0 (Linux; Android 16; X Build/Y;) AppleWebKit/537.36 (KHTML, like Gecko) " + "Version/4.0 Chrome/156.0.8078.47 Mobile Safari/537.36 Zalo android/261002903 ZaloTheme/light" +) +ZALO_MESSAGE = "ReferenceError: zaloJSV2 is not defined" + + +def test_zalo_host_app_injection_is_accepted_but_not_persisted(client): + resp = client.post( + "/api/client-errors", + json=_payload(message=ZALO_MESSAGE, stack=f"{ZALO_MESSAGE}\n at :1:2", user_agent=ZALO_UA), + ) + assert resp.status_code == 202 + assert resp.json()["is_new"] is False + client.fake_adapter.upsert_pattern.assert_not_called() + + +def test_real_errors_inside_zalo_are_persisted(client): + resp = client.post("/api/client-errors", json=_payload(user_agent=ZALO_UA)) + assert resp.status_code == 202 + client.fake_adapter.upsert_pattern.assert_called_once() + + +def test_zalo_message_from_another_browser_is_persisted(client): + resp = client.post("/api/client-errors", json=_payload(message=ZALO_MESSAGE)) + assert resp.status_code == 202 + client.fake_adapter.upsert_pattern.assert_called_once() diff --git a/backend/tests/test_audio_download_worker.py b/backend/tests/test_audio_download_worker.py index cd70b136..11a0d2db 100644 --- a/backend/tests/test_audio_download_worker.py +++ b/backend/tests/test_audio_download_worker.py @@ -1085,6 +1085,17 @@ class TestTorrentStallAwaitsUser: assert details["stall_minutes"] == 60 assert details["keep_trying"] is True + @pytest.mark.asyncio + async def test_known_download_failure_logs_warning_not_error(self, caplog): + """fail_job's job-failed alert is the one alert for this failure.""" + import logging + + flacfetch = _mock_flacfetch(FlacfetchServiceError("Download failed: tracker 500")) + with caplog.at_level(logging.INFO, logger="backend.workers.audio_download_worker"): + await self._run(_red_job(), flacfetch, attempt="0") + records = [r for r in caplog.records if "Download failed: Download failed" in r.getMessage()] + assert [r.levelno for r in records] == [logging.WARNING] + @pytest.mark.asyncio async def test_non_stall_failure_still_marks_retry_pending(self): flacfetch = _mock_flacfetch(FlacfetchServiceError("Download failed: tracker 500")) diff --git a/backend/tests/test_credential_manager.py b/backend/tests/test_credential_manager.py index 16c1baf9..a557fc28 100644 --- a/backend/tests/test_credential_manager.py +++ b/backend/tests/test_credential_manager.py @@ -1,6 +1,7 @@ """ Tests for credential manager and auth endpoints. """ +import logging import pytest import json from unittest.mock import MagicMock, patch @@ -194,6 +195,65 @@ class TestCredentialManager: assert result is False + def test_expired_dropbox_token_probe_logs_warning_not_error(self, caplog): + """Short-lived tokens expire constantly and get refreshed; that isn't page-worthy.""" + manager = CredentialManager() + with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")): + with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"): + assert manager._test_dropbox_api({"access_token": "test"}) is False + levels = [r.levelno for r in caplog.records if "Dropbox" in r.message] + assert levels == [logging.WARNING] + + def _dropbox_manager(self, creds): + manager = CredentialManager() + manager.settings = MagicMock() + manager.settings.get_secret.return_value = json.dumps(creds) + return manager + + FULL_DROPBOX_CREDS = {"access_token": "a", "refresh_token": "r", "app_key": "k", "app_secret": "s"} + + def test_dropbox_refresh_success_logs_no_error(self, caplog): + manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS) + ok = MagicMock(status_code=200) + ok.json.return_value = {"access_token": "new"} + with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \ + patch('requests.post', return_value=ok), \ + patch.object(manager, "_update_dropbox_credentials", return_value=True): + with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"): + result = manager.check_dropbox_credentials() + assert result.status == CredentialStatus.VALID + assert not [r for r in caplog.records if r.levelno >= logging.ERROR] + + def test_dropbox_refresh_rejected_logs_error(self, caplog): + manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS) + bad = MagicMock(status_code=400, text='{"error": "invalid_grant"}') + with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \ + patch('requests.post', return_value=bad): + with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"): + result = manager.check_dropbox_credentials() + assert result.status == CredentialStatus.INVALID + errors = [r.message for r in caplog.records if r.levelno >= logging.ERROR] + assert len(errors) == 1 + assert "400" in errors[0] + assert "invalid_grant" in errors[0] + + def test_dropbox_refresh_raising_logs_one_error(self, caplog): + manager = self._dropbox_manager(self.FULL_DROPBOX_CREDS) + with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")), \ + patch('requests.post', side_effect=RuntimeError("network down")): + with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"): + result = manager.check_dropbox_credentials() + assert result.status == CredentialStatus.INVALID + assert len([r for r in caplog.records if r.levelno >= logging.ERROR]) == 1 + + def test_dropbox_invalid_with_no_refresh_token_logs_error(self, caplog): + manager = self._dropbox_manager({"access_token": "a"}) + with patch('dropbox.Dropbox', side_effect=Exception("expired_access_token")): + with caplog.at_level(logging.INFO, logger="backend.services.credential_manager"): + result = manager.check_dropbox_credentials() + assert result.status == CredentialStatus.INVALID + assert len([r for r in caplog.records if r.levelno >= logging.ERROR]) == 1 + def test_send_credential_alert_no_webhook(self): """Test alert without webhook URL.""" manager = CredentialManager() diff --git a/backend/tests/test_job_manager.py b/backend/tests/test_job_manager.py index 8336b373..e4cf852e 100644 --- a/backend/tests/test_job_manager.py +++ b/backend/tests/test_job_manager.py @@ -290,6 +290,24 @@ class TestJobStatusTransitions: class TestJobFailure: """Test job failure handling.""" + def test_fail_job_logs_warning_not_error(self, job_manager, mock_firestore_service, caplog): + """ops_alerts' job-failed alert is the one alert; an ERROR here made a second one.""" + import logging + + mock_firestore_service.get_job.return_value = None + with caplog.at_level(logging.INFO, logger="backend.services.job_manager"): + job_manager.fail_job("job-1", "Download failed: boom") + records = [r for r in caplog.records if "job-1 failed" in r.getMessage()] + assert [r.levelno for r in records] == [logging.WARNING] + + def test_mark_job_failed_logs_warning_not_error(self, job_manager, mock_firestore_service, caplog): + import logging + + with caplog.at_level(logging.INFO, logger="backend.services.job_manager"): + job_manager.mark_job_failed("job-2", "boom") + records = [r for r in caplog.records if "job-2 failed" in r.getMessage()] + assert [r.levelno for r in records] == [logging.WARNING] + def test_mark_job_as_failed(self, job_manager, mock_firestore_service): """Test marking a job as failed.""" error_message = "Audio separation failed" diff --git a/backend/tests/test_jobs_api.py b/backend/tests/test_jobs_api.py index 1c9c85ef..bf119927 100644 --- a/backend/tests/test_jobs_api.py +++ b/backend/tests/test_jobs_api.py @@ -440,12 +440,30 @@ class TestStartReview: assert response.json() == {"status": "success", "job_status": "in_review"} mock_job_manager.record_review_started.assert_not_called() - def test_start_review_invalid_transition_otherwise_still_errors(self, client, mock_job_manager, mock_job, auth_headers): + def test_start_review_invalid_transition_race_returns_409(self, client, mock_job_manager, mock_job, auth_headers): + """The auto-approver moved the job past review between our read and the transition.""" from backend.exceptions import InvalidStateTransitionError mock_job.status = JobStatus.AWAITING_REVIEW + moved_on = mock_job.model_copy(update={"status": JobStatus.REVIEW_COMPLETE}) + mock_job_manager.get_job.side_effect = [mock_job, moved_on, moved_on] mock_job_manager.transition_to_state.side_effect = InvalidStateTransitionError(message="nope") - with pytest.raises(InvalidStateTransitionError): - client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers) + response = client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers) + assert response.status_code == 409 + assert "review_complete" in response.json()["detail"] + mock_job_manager.record_review_started.assert_not_called() + + @pytest.mark.parametrize("status", [JobStatus.COMPLETE, JobStatus.FAILED, JobStatus.PENDING]) + def test_start_review_outside_review_returns_409_without_transition( + self, client, mock_job_manager, mock_job, auth_headers, status + ): + """A stale "review lyrics" push on a finished job: no 500, no transition, no ERROR log.""" + mock_job.status = status + response = client.post("/api/jobs/test123/start-review", json={}, headers=auth_headers) + assert response.status_code == 409 + assert status.value in response.json()["detail"] + assert "JobStatus." not in response.json()["detail"] + mock_job_manager.transition_to_state.assert_not_called() + mock_job_manager.record_review_started.assert_not_called() def test_start_review_from_awaiting_records_once(self, client, mock_job_manager, mock_job, auth_headers): mock_job.status = JobStatus.AWAITING_REVIEW diff --git a/backend/tests/test_main_startup_warmup.py b/backend/tests/test_main_startup_warmup.py index 7c11e523..3675d830 100644 --- a/backend/tests/test_main_startup_warmup.py +++ b/backend/tests/test_main_startup_warmup.py @@ -181,3 +181,67 @@ def test_lifespan_warms_route_contexts_off_loop_before_readiness(): assert seen[0][0] is main_module.app assert seen[0][1].startswith("asyncio_"), seen + + +def test_anyio_backend_is_a_lazy_route_import(): + """starlette/middleware/base.py imports it on the first request: 17.3s on the loop + with 18 requests in flight, 2026-10-30.""" + assert "anyio._backends._asyncio" in main_module._LAZY_ROUTE_IMPORTS + + +def test_warm_request_runs_through_the_whole_middleware_stack_off_loop(): + from starlette.applications import Starlette + from starlette.middleware import Middleware + from starlette.middleware.base import BaseHTTPMiddleware + from starlette.responses import PlainTextResponse + from starlette.routing import Route + + seen = {} + + class Spy(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + seen["thread"] = threading.current_thread().name + seen["path"] = request.url.path + return await call_next(request) + + async def health(_request): + return PlainTextResponse("ok") + + app = Starlette(routes=[Route("/api/health", health)], middleware=[Middleware(Spy)]) + assert app.middleware_stack is None + + worker = threading.Thread(target=main_module._warm_middleware_stack, args=(app,), name="warm-test") + worker.start() + worker.join(timeout=10) + + assert seen == {"thread": "warm-test", "path": "/api/health"} + assert app.middleware_stack is not None + + +def test_warm_request_failure_never_blocks_startup(): + async def broken_app(scope, receive, send): + raise RuntimeError("boom") + + main_module._warm_middleware_stack(broken_app) # must not raise + + +def test_lifespan_warms_the_middleware_stack_before_readiness(): + calls = [] + + def fake_warm(application): + calls.append(application) + + with patch.object(main_module, "_run_background_warmup"), \ + patch.object(main_module, "_warm_middleware_stack", fake_warm): + with TestClient(main_module.app): + assert calls == [main_module.app] + + +def test_warm_request_on_the_real_app_returns_healthy_and_skips_the_audit_log(caplog): + import logging + + with caplog.at_level(logging.INFO): + main_module._warm_middleware_stack(main_module.app) + assert main_module.app.middleware_stack is not None + assert not [r for r in caplog.records if r.name == "audit"] + assert not [r for r in caplog.records if "warm request failed" in r.message.lower()] diff --git a/backend/tests/test_routes_review.py b/backend/tests/test_routes_review.py index 2dffc842..f6344498 100644 --- a/backend/tests/test_routes_review.py +++ b/backend/tests/test_routes_review.py @@ -1738,6 +1738,35 @@ class TestReplayCorrectionData: finally: self._teardown() + def _awaiting_job(self): + from backend.models.job import JobStatus + job = self._completed_job() + job.status = JobStatus.AWAITING_REVIEW + job.state_data = {} + return job + + def test_auto_approver_race_returns_409_not_500(self): + """Job read as awaiting_review, then auto-approved before our transition.""" + from backend.exceptions import InvalidStateTransitionError + from backend.models.job import JobStatus + + client = self._client_with_auth("full") + try: + with patch("backend.api.routes.review.JobManager") as jm, \ + patch("backend.api.routes.review.StorageService") as storage_cls: + moved_on = self._awaiting_job() + moved_on.status = JobStatus.REVIEW_COMPLETE + jm.return_value.get_job.side_effect = [self._awaiting_job(), moved_on, moved_on] + jm.return_value.transition_to_state.side_effect = InvalidStateTransitionError(message="nope") + storage_cls.return_value.file_exists.return_value = True + storage_cls.return_value.download_json.return_value = {"corrected_segments": []} + resp = client.get("/api/review/job1/correction-data") + assert resp.status_code == 409 + assert "review_complete" in resp.json()["detail"] + jm.return_value.record_review_started.assert_not_called() + finally: + self._teardown() + def test_replay_requires_full_auth(self): client = self._client_with_auth("review_token") try: diff --git a/backend/workers/audio_download_worker.py b/backend/workers/audio_download_worker.py index ead3c081..daaf5b2f 100644 --- a/backend/workers/audio_download_worker.py +++ b/backend/workers/audio_download_worker.py @@ -399,7 +399,8 @@ async def process_audio_download(job_id: str) -> bool: if not isinstance(e, (DownloadError, FlacfetchServiceError)): logger.error(f"[job:{job_id}] Download failed: {e}", exc_info=True) else: - logger.error(f"[job:{job_id}] Download failed: {e}") + # Expected failure type; fail_job below sends the one job-failed alert. + logger.warning(f"[job:{job_id}] Download failed: {e}") _mark_retry_pending_if_attempts_remain(job_manager, job_id) job_manager.fail_job(job_id, f"Audio download failed: {e}", error_details={'stage': 'audio_download'}) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b0349d75..0f2f506c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -117,7 +117,14 @@ Keeping #alerts readable (one root cause → one message): `ended` line is ERROR (≥10s blocked with a request affected), so one freeze is one pattern. The `backend/event_loop_stalls` metric keys on that line too. - **Known noise** (`known_issues.py`) can be service-scoped, e.g. - github-runner-manager's scale-up "no available instance" aborts. + github-runner-manager's scale-up "no available instance" aborts, and scoped to a + request path too (decide-search's `/api/search/health` aborts). +- **Unhandled exceptions**: `Exception in ASGI application` and `request_audit_error` + become one `Unhandled exception: ` pattern per cause, read from the + entry's `exception` field; the path goes in the sample only. +- **One alert per job failure**: `ops_alerts` sends it from the status write, so + `Job failed` and the download-failure lines before it are not ERROR patterns. + YouTube video ids are normalized out of signatures. Frontend crashes are captured via `window.onerror`, `unhandledrejection`, Next.js `error.tsx`, and a `CrashReportBoundary` around the review UIs. They POST to diff --git a/karaoke_gen/lyrics_transcriber/lyrics/spotify.py b/karaoke_gen/lyrics_transcriber/lyrics/spotify.py index 873ae9e9..15076cb1 100644 --- a/karaoke_gen/lyrics_transcriber/lyrics/spotify.py +++ b/karaoke_gen/lyrics_transcriber/lyrics/spotify.py @@ -110,6 +110,13 @@ class SpotifyProvider(BaseLyricsProvider): self.logger.debug(f"Making RapidAPI lyrics request for track ID {track_id}") lyrics_response = requests.get(lyrics_url, headers=headers, params=lyrics_params, timeout=10) + if lyrics_response.status_code == 403: + # The search just succeeded with the same key, so the key is fine: a 401/403 + # on /search stays the dead-key ERROR. Other providers fill in. + self.logger.warning( + f"Spotify RapidAPI lyrics step returned 403 for track {track_id}: {lyrics_response.text[:200]}" + ) + return None lyrics_response.raise_for_status() lyrics_data = lyrics_response.json() diff --git a/pyproject.toml b/pyproject.toml index 1bd0ac93..9b797b6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.290.0" +version = "0.290.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/services/error_monitor/test_known_issues.py b/tests/unit/services/error_monitor/test_known_issues.py index 9792ddad..75ef7a26 100644 --- a/tests/unit/services/error_monitor/test_known_issues.py +++ b/tests/unit/services/error_monitor/test_known_issues.py @@ -585,3 +585,111 @@ class TestRunnerManagerNoAvailableInstance: from backend.services.error_monitor.known_issues import should_ignore assert should_ignore("github-runner-manager", "Failed to create runner VM: quota exceeded") is None + + +class TestAudioSeparationExperimentTracebacks: + """Ad-hoc `python -c` experiments on prod job images (2026-10-31, 11-01) are not + job failures: two consecutive frames.""" + + EXPERIMENT = ( + "Traceback (most recent call last):\n" + ' File "", line 1, in \n' + ' File "", line 250, in \n' + ' File "", line 228, in forced_align\n' + "OverflowError: Python integer 1162 out of bounds for int8" + ) + SPAWN = ( + "Traceback (most recent call last):\n" + ' File "", line 1, in \n' + ' File "/usr/local/lib/python3.11/multiprocessing/spawn.py", line 122, in spawn_main\n' + " exitcode = _main(fd, parent_sentinel)\n" + "RuntimeError: a real worker crash" + ) + + def test_experiment_ignored_on_the_job(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("audio-separation-job", self.EXPERIMENT) + assert result is not None + assert result.pattern_name == "audio_separation_experiment_traceback" + + def test_multiprocessing_spawn_traceback_still_alerts(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("audio-separation-job", self.SPAWN) is None + + def test_same_traceback_on_other_services_still_alerts(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("karaoke-backend", self.EXPERIMENT) is None + + def test_extract_message_does_not_truncate_before_should_ignore(self): + from unittest.mock import MagicMock + + from backend.services.error_monitor.known_issues import should_ignore + from backend.services.error_monitor.monitor import _extract_message + + entry = MagicMock() + entry.payload = "x" * 5000 + "\n" + self.EXPERIMENT + assert should_ignore("audio-separation-job", _extract_message(entry)) is not None + + +class TestUrlDownloadFailedDuplicate: + """fail_job follows this line, and ops_alerts' job-failed alert is the one alert.""" + + MSG = "URL download failed: Remote download failed: ERROR: [youtube] abcdefghijk: Sign in to confirm" + + def test_ignored_on_the_backend(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", self.MSG) + assert result is not None + assert result.pattern_name == "url_download_failed_duplicate" + + def test_other_download_errors_still_alert(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("karaoke-backend", "GCS upload failed: URL download failed: x") is None + assert should_ignore("karaoke-backend", "Download failed: something") is None + + +class TestDecideSearchHealthAborts: + MSG = ( + "The request was aborted after waiting too long for an available instance. " + "Additional troubleshooting documentation can be found at: " + "https://cloud.google.com/run/docs/troubleshooting#abort-request" + ) + HEALTH = "https://karaoke-decide-search-x.a.run.app/api/search/health" + + def test_health_probe_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-decide-search", self.MSG, self.HEALTH) + assert result is not None + assert result.pattern_name == "decide_search_health_abort" + + def test_health_probe_with_query_string_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("karaoke-decide-search", self.MSG, self.HEALTH + "?ping=1") is not None + + def test_real_search_paths_still_alert(self): + from backend.services.error_monitor.known_issues import should_ignore + + for url in ( + "https://karaoke-decide-search-x.a.run.app/api/search?q=hello", + "https://karaoke-decide-search-x.a.run.app/api/search/health/deep", + "https://karaoke-decide-search-x.a.run.app/api/search/songs?q=/api/search/health", + ): + assert should_ignore("karaoke-decide-search", self.MSG, url) is None + + def test_no_url_still_alerts(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("karaoke-decide-search", self.MSG) is None + + def test_other_services_and_messages_still_alert(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("karaoke-decide", self.MSG, self.HEALTH) is None + assert should_ignore("karaoke-decide-search", "boom", self.HEALTH) is None diff --git a/tests/unit/services/error_monitor/test_monitor.py b/tests/unit/services/error_monitor/test_monitor.py index 7bac0fad..8efc3e37 100644 --- a/tests/unit/services/error_monitor/test_monitor.py +++ b/tests/unit/services/error_monitor/test_monitor.py @@ -178,6 +178,243 @@ class TestExtractMessage: assert result is None or result == "" +# Invented paths/ids; the shapes are copied from real prod entries (2026-11-03). +_PLAIN_TRACEBACK = ( + "Traceback (most recent call last):\n" + ' File "/app/backend/middleware/audit_logging.py", line 63, in dispatch\n' + " response = await call_next(request)\n" + " ^^^^^^^^^^^^^^^^^^^^^^^^\n" + ' File "/app/backend/services/job_manager.py", line 657, in validate_state_transition\n' + " raise InvalidStateTransitionError(\n" + "backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: " + "complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']\n" +) +_GROUP_TRACEBACK = ( + " + Exception Group Traceback (most recent call last):\n" + ' | File "/usr/local/lib/python3.11/site-packages/starlette/_utils.py", line 104, in create_collapsing_task_group\n' + " | async with anyio.create_task_group() as tg:\n" + ' | File "/usr/local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 847, in __aexit__\n' + " | raise BaseExceptionGroup(\n" + " | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)\n" + " +-+---------------- 1 ----------------\n" + " | Traceback (most recent call last):\n" + ' | File "/app/backend/services/job_manager.py", line 657, in validate_state_transition\n' + " | raise InvalidStateTransitionError(\n" + " | backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: " + "complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']\n" + " +------------------------------------\n" +) +_EXPECTED_CAUSE = ( + "Unhandled exception: backend.exceptions.InvalidStateTransitionError: Invalid state transition " + "for job 7f89f884: complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']" +) + + +class TestUnhandledExceptionMessages: + """Every unhandled exception logs one of two fixed messages; the pattern must + come from the exception, or all crashes share one (already 'new') pattern and + a second, different crash never alerts.""" + + def _entry(self, message, exception, **extra): + entry = MagicMock() + entry.payload = {"message": message, "exception": exception, **extra} + return entry + + def test_audit_error_plain_traceback(self): + m = _import_monitor_module() + entry = self._entry("request_audit_error", _PLAIN_TRACEBACK, path="/api/jobs/x/start-review", method="POST") + assert m._extract_message(entry) == _EXPECTED_CAUSE + + def test_asgi_exception_group_traceback_with_pipe_prefixes(self): + m = _import_monitor_module() + entry = self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK) + assert m._extract_message(entry) == _EXPECTED_CAUSE + + def test_nested_exception_group(self): + m = _import_monitor_module() + nested = ( + " + Exception Group Traceback (most recent call last):\n" + " | ExceptionGroup: outer (1 sub-exception)\n" + " +-+---------------- 1 ----------------\n" + " | Exception Group Traceback (most recent call last):\n" + " | ExceptionGroup: inner (1 sub-exception)\n" + " +-+---------------- 1 ----------------\n" + " | Traceback (most recent call last):\n" + ' | File "/app/a.py", line 1, in f\n' + " | boom()\n" + " | ValueError: the real cause\n" + " +------------------------------------\n" + " +------------------------------------\n" + ) + assert m._extract_message(self._entry("request_audit_error", nested)) == ( + "Unhandled exception: ValueError: the real cause" + ) + + def test_multiline_exception_message_uses_its_first_line(self): + # pydantic's last traceback line is a docs link, the same for every failure. + m = _import_monitor_module() + exc = ( + "Traceback (most recent call last):\n" + ' File "/app/a.py", line 1, in f\n' + " boom()\n" + "pydantic_core._pydantic_core.ValidationError: 1 validation error for Job\n" + "status\n" + " Input should be a valid string\n" + " For further information visit https://errors.pydantic.dev/2/v/string_type\n" + ) + assert m._extract_message(self._entry("request_audit_error", exc)) == ( + "Unhandled exception: pydantic_core._pydantic_core.ValidationError: 1 validation error for Job" + ) + + def test_chained_exception_uses_the_final_one(self): + m = _import_monitor_module() + exc = ( + "Traceback (most recent call last):\n" + ' File "/app/a.py", line 1, in f\n' + " boom()\n" + "KeyError: 'first'\n\n" + "During handling of the above exception, another exception occurred:\n\n" + "Traceback (most recent call last):\n" + ' File "/app/b.py", line 2, in g\n' + " bang()\n" + "RuntimeError: second\n" + ) + assert m._extract_message(self._entry("request_audit_error", exc)) == ( + "Unhandled exception: RuntimeError: second" + ) + + def test_no_exception_field_keeps_the_message(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = {"message": "request_audit_error"} + assert m._extract_message(entry) == "request_audit_error" + + def test_unparseable_exception_keeps_last_line(self): + m = _import_monitor_module() + assert m._extract_message(self._entry("request_audit_error", "something odd\nno frames here\n")) == ( + "Unhandled exception: no frames here" + ) + + def test_other_messages_with_exception_field_are_untouched(self): + m = _import_monitor_module() + entry = self._entry("Job 1 could not save", _PLAIN_TRACEBACK) + assert m._extract_message(entry) == "Job 1 could not save" + + def test_both_log_lines_of_one_crash_collapse_to_one_pattern(self): + from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message + + m = _import_monitor_module() + audit = m._extract_message(self._entry("request_audit_error", _PLAIN_TRACEBACK)) + asgi = m._extract_message(self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK)) + assert compute_pattern_hash("s", normalize_message(audit)) == compute_pattern_hash("s", normalize_message(asgi)) + + def test_two_different_crashes_are_two_patterns(self): + from backend.services.error_monitor.normalizer import normalize_message + + m = _import_monitor_module() + other = _PLAIN_TRACEBACK.replace( + "backend.exceptions.InvalidStateTransitionError: Invalid state transition for job 7f89f884: " + "complete -> in_review. Valid transitions: ['awaiting_review', 'lyrics_complete']", + "ZeroDivisionError: division by zero", + ) + first = m._extract_message(self._entry("request_audit_error", _PLAIN_TRACEBACK)) + second = m._extract_message(self._entry("request_audit_error", other)) + assert normalize_message(first) != normalize_message(second) + + def test_sample_keeps_the_path_but_the_pattern_does_not(self): + m = _import_monitor_module() + entry = self._entry( + "request_audit_error", _PLAIN_TRACEBACK, path="/api/jobs/7f89f884/start-review", method="POST" + ) + message = m._extract_message(entry) + sample = m._extract_sample(entry, message) + assert sample.startswith("POST /api/jobs/7f89f884/start-review\n") + assert sample.endswith(_EXPECTED_CAUSE) + assert "start-review" not in message + + def test_sample_is_the_message_without_a_path(self): + m = _import_monitor_module() + entry = self._entry("Exception in ASGI application\n", _GROUP_TRACEBACK) + message = m._extract_message(entry) + assert m._extract_sample(entry, message) == message + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_cycle_groups_a_crash_into_one_alert_with_the_path_in_its_sample(self, _digest, _llm): + from backend.services.error_monitor.firestore_adapter import UpsertResult + + m = _import_monitor_module() + audit = _make_entry( + payload={ + "message": "request_audit_error", + "exception": _PLAIN_TRACEBACK, + "path": "/api/jobs/7f89f884/start-review", + "method": "POST", + } + ) + asgi = _make_entry(payload={"message": "Exception in ASGI application\n", "exception": _GROUP_TRACEBACK}) + audit.http_request = asgi.http_request = None + monitor = _make_monitor_with_mocks() + monitor.logging_client.list_entries.side_effect = [iter([audit, asgi])] + [iter([])] * ( + len(m.RESOURCE_TYPE_FILTERS) - 1 + ) + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(pattern_id="p", is_new=True) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.firestore_adapter.get_unalerted_new_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + assert monitor.firestore_adapter.upsert_pattern.call_count == 1 + data = monitor.firestore_adapter.upsert_pattern.call_args[0][0] + assert data.count == 2 + assert "/api/jobs/7f89f884/start-review" in data.sample_message + assert "start-review" not in data.normalized_message + assert "InvalidStateTransitionError" in data.normalized_message + assert monitor.discord.send_message.call_count == 1 + + +class TestExtractRequestUrl: + def test_reads_the_cloud_run_request_url(self): + m = _import_monitor_module() + entry = MagicMock() + entry.http_request = {"requestUrl": "https://x.a.run.app/api/search/health"} + assert m._extract_request_url(entry) == "https://x.a.run.app/api/search/health" + + def test_none_without_http_request(self): + m = _import_monitor_module() + entry = MagicMock() + entry.http_request = None + assert m._extract_request_url(entry) is None + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_cycle_drops_health_aborts_but_alerts_real_search_paths(self, _digest, _llm): + from backend.services.error_monitor.firestore_adapter import UpsertResult + + m = _import_monitor_module() + msg = "The request was aborted after waiting too long for an available instance." + health = _make_entry(payload=msg, service_name="karaoke-decide-search") + health.http_request = {"requestUrl": "https://x.a.run.app/api/search/health"} + real = _make_entry(payload=msg, service_name="karaoke-decide-search") + real.http_request = {"requestUrl": "https://x.a.run.app/api/search?q=hello"} + monitor = _make_monitor_with_mocks() + monitor.logging_client.list_entries.side_effect = [iter([health, real])] + [iter([])] * ( + len(m.RESOURCE_TYPE_FILTERS) - 1 + ) + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(pattern_id="p", is_new=True) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.firestore_adapter.get_unalerted_new_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + assert monitor.firestore_adapter.upsert_pattern.call_count == 1 + assert monitor.firestore_adapter.upsert_pattern.call_args[0][0].count == 1 + + # --------------------------------------------------------------------------- # Tests: _extract_service_name # --------------------------------------------------------------------------- diff --git a/tests/unit/services/error_monitor/test_normalizer.py b/tests/unit/services/error_monitor/test_normalizer.py index 6378fd7d..fd08a772 100644 --- a/tests/unit/services/error_monitor/test_normalizer.py +++ b/tests/unit/services/error_monitor/test_normalizer.py @@ -452,3 +452,46 @@ def test_loop_stall_request_counts_collapse_to_one_pattern(): b = normalize_message("EVENT_LOOP_STALL ended duration_ms=81183 culprit=x.py:539 in f requests_affected=3") assert a == b assert "requests_affected=" in a + + +class TestYouTubeVideoIds: + """A video id in the text made every bot block a "first time" signature.""" + + def _bot_block(self, video_id): + return ( + f"Download failed: ERROR: [youtube] {video_id}: Sign in to confirm you\u2019re not a bot. " + "Use --cookies-from-browser or --cookies for the authentication." + ) + + def test_same_error_on_different_videos_is_one_signature(self): + from backend.services.error_monitor.normalizer import normalize_message + + assert normalize_message(self._bot_block("mBJivIsN0n0")) == normalize_message(self._bot_block("a_b-cDeFgHi")) + + def test_id_replaced_with_a_placeholder(self): + from backend.services.error_monitor.normalizer import normalize_message + + result = normalize_message(self._bot_block("mBJivIsN0n0")) + assert "[youtube] : Sign in" in result + assert "mBJivIsN0n0" not in result + + def test_tab_extractor_variant(self): + from backend.services.error_monitor.normalizer import normalize_message + + assert "[youtube:tab] :" in normalize_message("ERROR: [youtube:tab] mBJivIsN0n0: gone") + + def test_bare_watch_url_id(self): + from backend.services.error_monitor.normalizer import normalize_message + + result = normalize_message("Could not fetch youtube.com/watch?v=mBJivIsN0n0 today") + assert "mBJivIsN0n0" not in result + + def test_youtube_words_without_an_id_are_kept(self): + from backend.services.error_monitor.normalizer import normalize_message + + assert normalize_message("[youtube] Downloading webpage") == "[youtube] Downloading webpage" + + def test_ops_alert_signature_ignores_the_video_id(self): + from backend.services.ops_alerts import _signature + + assert _signature(self._bot_block("mBJivIsN0n0")) == _signature(self._bot_block("a_b-cDeFgHi")) diff --git a/tests/unit/test_spotify_provider_log_levels.py b/tests/unit/test_spotify_provider_log_levels.py new file mode 100644 index 00000000..db4857fd --- /dev/null +++ b/tests/unit/test_spotify_provider_log_levels.py @@ -0,0 +1,65 @@ +"""Log levels on the Spotify RapidAPI provider: ERROR pages a human, so a 403 on the +/lyrics step (the search just succeeded with the same key, other providers fill in) +is a WARNING, while 401/403 on /search stays the dead-key ERROR.""" + +import logging +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from karaoke_gen.lyrics_transcriber.lyrics.base_lyrics_provider import LyricsProviderConfig +from karaoke_gen.lyrics_transcriber.lyrics.spotify import SpotifyProvider + +SEARCH_OK = {"status": True, "errorId": "Success", "id": "track1"} + + +def _response(status, body=None, text=""): + resp = MagicMock() + resp.status_code = status + resp.text = text + resp.json.return_value = body or {} + if status >= 400: + err = requests.exceptions.HTTPError(f"{status} Client Error", response=resp) + resp.raise_for_status.side_effect = err + return resp + + +def _provider(): + return SpotifyProvider(LyricsProviderConfig(rapidapi_key="test-key"), logger=logging.getLogger("spotify-test")) + + +def _levels(caplog): + return [r.levelno for r in caplog.records if r.name == "spotify-test"] + + +def test_lyrics_403_is_a_warning_with_the_body(caplog): + long_body = "not subscribed " + "x" * 500 + with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(403, text=long_body)]): + with caplog.at_level(logging.INFO, logger="spotify-test"): + assert _provider()._fetch_from_rapidapi("A", "T") is None + assert logging.ERROR not in _levels(caplog) + warning = next(r.message for r in caplog.records if r.levelno == logging.WARNING) + assert "not subscribed" in warning + assert "x" * 201 not in warning + + +@pytest.mark.parametrize("status", [401, 403]) +def test_search_auth_failure_stays_an_error(caplog, status): + with patch("requests.get", side_effect=[_response(status)]): + with caplog.at_level(logging.INFO, logger="spotify-test"): + assert _provider()._fetch_from_rapidapi("A", "T") is None + assert logging.ERROR in _levels(caplog) + + +def test_lyrics_401_stays_an_error(caplog): + with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(401)]): + with caplog.at_level(logging.INFO, logger="spotify-test"): + assert _provider()._fetch_from_rapidapi("A", "T") is None + assert logging.ERROR in _levels(caplog) + + +def test_lyrics_success_still_returns_data(): + with patch("requests.get", side_effect=[_response(200, SEARCH_OK), _response(200, {"lyrics": []})]): + result = _provider()._fetch_from_rapidapi("A", "T") + assert result["_rapidapi_source"] is True