From 70b2f4279ab2edab5e42322649036dc0d83d4c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Mon, 19 Oct 2026 01:13:13 -0400 Subject: [PATCH] feat(alerts): collapse error bursts, suppress link previews, de-noise #alerts (v0.278.9) (#1188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Andrew's goal: "I'd like to make sure that our alerts channel only has alerts in it that are actionable and important, or are just signal and not noise." This follows #1186 (#noise routing). ## Changes 1. **Burst collapse** (`error_monitor/monitor.py`, `discord.py`, `config.py`). When a run has more than 3 new patterns for one service, including leftover un-alerted ones, the monitor sends ONE #alerts message. It has the count, total occurrences, the top 5 by count, first/last seen, and a `scripts/query-error-patterns.py --service X --status new` pointer. All of those patterns are marked alerted. The grouping is deterministic, since LLM grouping is off in prod. The 10-messages-per-run cap still applies, and unsent patterns are picked up again next run. Evidence: on 2026-10-07/08, one mb-refresh run produced 73 separate alerts. 2. **No link previews.** Every error-monitor post (`_post`) and every `ops_alerts` post now sets `flags: 4` (SUPPRESS_EMBEDS), through a new `suppress_embeds` kwarg on `DiscordNotificationService.post_message`. The YouTube "New upload" post keeps its embed. 3. **Loop watchdog.** `EVENT_LOOP_STALL in progress` is now always WARNING. The `ended` line keeps the existing ERROR rule. In the last two weeks there were 24 "in progress" pattern alerts on top of 15 "ended" ones. I checked `infrastructure/modules/monitoring.py` `EVENT_LOOP_STALL_LOG_FILTER`: it keys on `"EVENT_LOOP_STALL ended"` + `severity>=ERROR`, so this needs no Pulumi change. 4. **Frontend chunk-load errors** (`frontend_ingestion.py`, `firestore_adapter.py`). Reports matching `isChunkLoadError` collapse into one pattern per build with `alert_min_hourly_count=10`. The monitor holds that pattern un-alerted until more than 10 arrive within an hour, which means a broken deploy. Separately, Safari/Firefox stacks now get the message prepended before hashing (a pattern was named `@https://gen.nomadkaraoke.com`). Chrome hashes are unchanged. Existing Safari/Firefox patterns will re-key once, and burst collapse groups them. 5. **known_issues.** Added a service-scoped ignore for `github-runner-manager` "no available instance" aborts during scale-up (seen 09-20, 09-23, 09-27, 10-08). The same message from other services still alerts. 6. **divebar-mirror schedulers.** Added `attempt_deadline="600s"` on both `divebar-mirror-daily` and `divebar-mirror-refresh`; the function timeout is 540s. With the 180s default, Cloud Scheduler retried syncs that were still running, and the overlapping runs hit OOM (4x in 30 days). ## ⚠️ Pulumi `infrastructure/modules/divebar_mirror.py` changed, so **this needs `pulumi up` by Andrew before merge**. Auto-merge is NOT enabled. ## Testing - New and updated unit tests in `tests/unit/services/error_monitor/{test_monitor,test_discord,test_frontend_ingestion,test_firestore_adapter,test_known_issues}.py`, `backend/tests/{test_loop_watchdog,test_discord_service,test_ops_alerts}.py`, and `tests/unit/test_divebar_mirror_scheduler.py` (an AST guard: deadline > function timeout). - `pytest backend/tests --ignore=backend/tests/emulator`: 6421 passed, 36 skipped. - `tests/unit/services/error_monitor`: all pass. - Local CodeRabbit review (`--committed`) completed with 0 findings. @coderabbitai ignore πŸ€– Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_019pmG2XAgtUguEJWLFpkKHN --------- Co-authored-by: Claude Opus 5.5 --- backend/services/discord_service.py | 9 ++ backend/services/error_monitor/config.py | 12 ++ backend/services/error_monitor/discord.py | 49 +++++- .../error_monitor/firestore_adapter.py | 69 +++++--- .../error_monitor/frontend_ingestion.py | 37 ++++- .../services/error_monitor/known_issues.py | 25 ++- backend/services/error_monitor/monitor.py | 91 ++++++++--- backend/services/loop_watchdog.py | 21 +-- backend/services/ops_alerts.py | 4 +- backend/tests/test_discord_service.py | 21 +++ backend/tests/test_loop_watchdog.py | 11 +- backend/tests/test_ops_alerts.py | 18 +++ docs/ARCHITECTURE.md | 27 +++- infrastructure/modules/divebar_mirror.py | 13 ++ pyproject.toml | 2 +- .../services/error_monitor/test_discord.py | 75 ++++++++- .../error_monitor/test_firestore_adapter.py | 54 +++++++ .../error_monitor/test_frontend_ingestion.py | 98 +++++++++++ .../error_monitor/test_known_issues.py | 30 ++++ .../services/error_monitor/test_monitor.py | 152 ++++++++++++++++++ tests/unit/test_divebar_mirror_scheduler.py | 57 +++++++ 21 files changed, 807 insertions(+), 68 deletions(-) create mode 100644 tests/unit/test_divebar_mirror_scheduler.py diff --git a/backend/services/discord_service.py b/backend/services/discord_service.py index 3a369a71..85900dc7 100644 --- a/backend/services/discord_service.py +++ b/backend/services/discord_service.py @@ -19,6 +19,10 @@ import requests logger = logging.getLogger(__name__) +# Discord message flag SUPPRESS_EMBEDS (1 << 2): no link-preview cards. +SUPPRESS_EMBEDS_FLAG = 4 + + class DiscordNotificationService: """ Service for sending Discord webhook notifications. @@ -75,6 +79,7 @@ class DiscordNotificationService: self, message: str, webhook_url: Optional[str] = None, + suppress_embeds: bool = False, ) -> bool: """ Post a message to a Discord channel via webhook. @@ -82,6 +87,8 @@ class DiscordNotificationService: Args: message: The message content to post webhook_url: Optional webhook URL (uses instance URL if not provided) + suppress_embeds: Send Discord's SUPPRESS_EMBEDS flag so URLs in the + text don't unfurl into link-preview cards (alerts) Returns: True if message was posted successfully, False otherwise @@ -109,6 +116,8 @@ class DiscordNotificationService: # trigger a Discord ping. (CWE-74 β€” output injection into a downstream # component.) data = {"content": message, "allowed_mentions": {"parse": []}} + if suppress_embeds: + data["flags"] = SUPPRESS_EMBEDS_FLAG response = requests.post(url, json=data, timeout=30) response.raise_for_status() self.logger.info("Message posted to Discord successfully") diff --git a/backend/services/error_monitor/config.py b/backend/services/error_monitor/config.py index 16cbdc97..782a288a 100644 --- a/backend/services/error_monitor/config.py +++ b/backend/services/error_monitor/config.py @@ -150,6 +150,18 @@ MAX_DISCORD_MESSAGES_PER_RUN: int = 10 #: Discord API hard limit on message length. DISCORD_MAX_MESSAGE_LENGTH: int = 2000 +#: Discord message flag SUPPRESS_EMBEDS (1 << 2). Set on every error-monitor +#: post so quoted URLs (GCP docs links) don't unfurl into large preview cards. +DISCORD_SUPPRESS_EMBEDS_FLAG: int = 4 + +#: When one run has MORE than this many new patterns for one service, send a +#: single burst summary instead of one message each (2026-10-07/08: one bad +#: mb-refresh run produced 73 "New Error Pattern" messages, 10 an hour). +BURST_COLLAPSE_MIN_PATTERNS: int = 3 + +#: Number of patterns listed in a burst summary. +BURST_TOP_PATTERNS: int = 5 + # --------------------------------------------------------------------------- # Pattern tracking settings # --------------------------------------------------------------------------- diff --git a/backend/services/error_monitor/discord.py b/backend/services/error_monitor/discord.py index 547deaa5..0928d0f9 100644 --- a/backend/services/error_monitor/discord.py +++ b/backend/services/error_monitor/discord.py @@ -29,7 +29,9 @@ from zoneinfo import ZoneInfo import requests from backend.services.error_monitor.config import ( + BURST_TOP_PATTERNS, DISCORD_MAX_MESSAGE_LENGTH, + DISCORD_SUPPRESS_EMBEDS_FLAG, MAX_DISCORD_MESSAGES_PER_RUN, ) @@ -109,6 +111,47 @@ def format_new_pattern_alert( return _truncate("\n".join(lines)) +def format_pattern_burst_alert(service: str, patterns: list[dict]) -> str: + """Format ONE Discord alert for a burst of new patterns from one service. + + One root cause (a bad run, a schema change) often produces dozens of + distinct messages, e.g. one per table. Listing them one message each buried + #alerts, so they collapse into a summary with the top patterns by count and + a pointer to the full list. + + Args: + service: The service all the patterns came from. + patterns: Dicts with keys normalized_message, count, first_seen, last_seen. + + Returns: + Formatted Discord message string, truncated to 2000 chars. + """ + ranked = sorted(patterns, key=lambda p: p.get("count") or 0, reverse=True) + total = sum(int(p.get("count") or 0) for p in patterns) + first = min((p["first_seen"] for p in patterns if p.get("first_seen")), default="?") + last = max((p["last_seen"] for p in patterns if p.get("last_seen")), default="?") + lines = [ + f"πŸ”΄ **{len(patterns)} new error patterns from {service}** (one burst, {total} occurrences)", + "", + f"**Service:** {service}", + f"**First seen:** {first}", + f"**Last seen:** {last}", + f"**Alerted at:** {_now_eastern()}", + "", + f"**Top {min(BURST_TOP_PATTERNS, len(ranked))} by count:**", + ] + for p in ranked[:BURST_TOP_PATTERNS]: + msg = (p.get("normalized_message") or "").splitlines()[0][:150] if p.get("normalized_message") else "" + lines.append(f"β€’ `{msg}` Γ—{p.get('count', 0)}") + if len(ranked) > BURST_TOP_PATTERNS: + lines.append(f"…and {len(ranked) - BURST_TOP_PATTERNS} more") + lines += [ + "", + f"All patterns: `python scripts/query-error-patterns.py --service {service} --status new`", + ] + return _truncate("\n".join(lines)) + + def format_incident_alert( title: str, severity: str, @@ -305,7 +348,11 @@ class ErrorMonitorDiscord: @staticmethod def _post(url: str, content: str) -> bool: try: - response = requests.post(url, json={"content": content}, timeout=30) + response = requests.post( + url, + json={"content": content, "flags": DISCORD_SUPPRESS_EMBEDS_FLAG}, + timeout=30, + ) response.raise_for_status() logger.debug("Discord message sent successfully.") return True diff --git a/backend/services/error_monitor/firestore_adapter.py b/backend/services/error_monitor/firestore_adapter.py index 88a14504..1180ab05 100644 --- a/backend/services/error_monitor/firestore_adapter.py +++ b/backend/services/error_monitor/firestore_adapter.py @@ -38,6 +38,9 @@ class PatternData: sample_message: str count: int timestamp: datetime + # Hold the alert until more than this many occurrences land within one hour + # (see is_below_alert_threshold). None = alert on first sight, as usual. + alert_min_hourly_count: int | None = None @dataclass @@ -72,6 +75,33 @@ def _utcnow() -> datetime: return datetime.now(tz=timezone.utc) +def is_below_alert_threshold(pattern: dict, now: datetime | None = None) -> bool: + """True if a gated pattern hasn't (yet) earned an alert. + + A pattern with ``alert_min_hourly_count`` (frontend chunk-load failures) + alerts only once MORE than that many occurrences land within the last hour, + counted from ``rolling_counts``. Until then it stays ``new`` with + ``alerted_at=None`` so a later burst can still fire. Ungated patterns are + never held. + """ + threshold = pattern.get("alert_min_hourly_count") + if threshold is None: + return False + now = now or _utcnow() + cutoff = now - timedelta(hours=1) + recent = 0 + for entry in pattern.get("rolling_counts") or []: + try: + ts = datetime.fromisoformat(entry["ts"]) + except (KeyError, TypeError, ValueError): + continue + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + if ts >= cutoff: + recent += int(entry.get("count") or 0) + return recent <= threshold + + def _iso(dt: datetime) -> str: """Serialise a datetime to ISO-8601 string.""" return dt.isoformat() @@ -181,24 +211,25 @@ class ErrorPatternsAdapter: if not snap.exists: # ── brand new pattern ────────────────────────────────────────── - doc_ref.set( - { - "pattern_id": data.pattern_id, - "service": data.service, - "resource_type": data.resource_type, - "normalized_message": data.normalized_message, - "sample_message": data.sample_message, - "status": "new", - "severity": "P3", - "total_count": data.count, - "rolling_counts": [new_rolling_entry], - "first_seen": ts_iso, - "last_seen": ts_iso, - "alerted_at": None, - "created_at": ts_iso, - "updated_at": ts_iso, - } - ) + doc = { + "pattern_id": data.pattern_id, + "service": data.service, + "resource_type": data.resource_type, + "normalized_message": data.normalized_message, + "sample_message": data.sample_message, + "status": "new", + "severity": "P3", + "total_count": data.count, + "rolling_counts": [new_rolling_entry], + "first_seen": ts_iso, + "last_seen": ts_iso, + "alerted_at": None, + "created_at": ts_iso, + "updated_at": ts_iso, + } + if data.alert_min_hourly_count is not None: + doc["alert_min_hourly_count"] = data.alert_min_hourly_count + doc_ref.set(doc) return UpsertResult(pattern_id=data.pattern_id, is_new=True) # ── existing pattern ─────────────────────────────────────────────── @@ -216,6 +247,8 @@ class ErrorPatternsAdapter: "sample_message": data.sample_message, "updated_at": ts_iso, } + if data.alert_min_hourly_count is not None: + update["alert_min_hourly_count"] = data.alert_min_hourly_count # Reactivation: pattern was considered resolved/fixed but error recurred is_new = False diff --git a/backend/services/error_monitor/frontend_ingestion.py b/backend/services/error_monitor/frontend_ingestion.py index e0172e89..325f3255 100644 --- a/backend/services/error_monitor/frontend_ingestion.py +++ b/backend/services/error_monitor/frontend_ingestion.py @@ -74,6 +74,23 @@ def is_opaque_script_error(message: str | None, source: str | None = None, extra return not extra.get("filename") and not extra.get("lineno") +# A chunk-load failure is a tab running an old build after a deploy: the client +# auto-reloads (frontend/lib/version-check.ts), so one report is not actionable. +# Mirrors isChunkLoadError() there. Reports collapse into one pattern per build +# that alerts only above CHUNK_LOAD_ALERT_PER_HOUR in an hour (a broken deploy). +_CHUNK_LOAD_RE = re.compile( + r"ChunkLoadError|Loading chunk [\w-]+ failed|Failed to load chunk" + r"|Failed to fetch dynamically imported module|Importing a module script failed", + re.IGNORECASE, +) +CHUNK_LOAD_ALERT_PER_HOUR = 10 + + +def is_chunk_load_error(message: str | None) -> bool: + """True for the bundler's "couldn't load a JS chunk" errors.""" + return bool(message) and bool(_CHUNK_LOAD_RE.search(message)) + + @dataclass class FrontendErrorReport: """In-memory representation of an inbound crash report.""" @@ -118,6 +135,13 @@ def _stack_for_hashing(report: FrontendErrorReport) -> str: Error object, e.g. Firefox "out of memory") hash on message + location. """ if report.stack: + # Safari/Firefox stacks are bare frames ("@https://…:1:5749") with no + # "Name: message" header, so without the message every error thrown from + # one frame shares a pattern named after the URL. Chrome stacks already + # start with the message (their hash is unchanged). + message = (report.message or "").strip() + if message and not report.stack.lstrip().startswith(message): + return f"{message}\n{report.stack}" return report.stack location = _error_location(report.extra) if location: @@ -153,8 +177,16 @@ def build_pattern_data( if now is None: now = datetime.now(tz=timezone.utc) - raw = _stack_for_hashing(report) - normalized = normalize_message(raw) + chunk_load = is_chunk_load_error(report.message) + if chunk_load: + # One pattern per build: chunk names, module ids and stacks vary per + # report but share one cause (that build's chunks are gone or broken). + normalized = ( + f"Failed to load JS chunk on build {report.release or 'unknown'} " + f"(client auto-reloads; alerts above {CHUNK_LOAD_ALERT_PER_HOUR}/hour)" + ) + else: + normalized = normalize_message(_stack_for_hashing(report)) pattern_id = compute_pattern_hash("frontend", normalized) # sample_message is human-readable context. Keep the error message plus a @@ -192,6 +224,7 @@ def build_pattern_data( sample_message=sample_message, count=1, timestamp=now, + alert_min_hourly_count=CHUNK_LOAD_ALERT_PER_HOUR if chunk_load else None, ) diff --git a/backend/services/error_monitor/known_issues.py b/backend/services/error_monitor/known_issues.py index 3b864022..80c5a3c6 100644 --- a/backend/services/error_monitor/known_issues.py +++ b/backend/services/error_monitor/known_issues.py @@ -109,11 +109,28 @@ _IGNORE_PATTERNS: list[tuple[str, re.Pattern, str]] = [ ] +# Patterns that are noise only for specific services: the same message from a +# customer-facing service is a real problem. (pattern_name, services, regex, reason) +_SERVICE_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str]] = [ + ( + "runner_manager_no_available_instance", + frozenset({"github-runner-manager"}), + re.compile( + r"request was aborted because there was no available instance", + re.IGNORECASE, + ), + "github-runner-manager scale-up burst: Cloud Run aborts webhook requests " + "while instances start, GitHub redelivers and the runners still dispatch " + "(seen 2026-09-20, 09-23, 09-27, 10-08)", + ), +] + + def should_ignore(service: str, message: str) -> IgnoreReason | None: """Check whether a log entry matches a known infrastructure noise pattern. - The check is purely message-based; ``service`` is accepted for API symmetry - and future service-specific overrides but does not gate any pattern today. + ``_IGNORE_PATTERNS`` match on the message alone; ``_SERVICE_IGNORE_PATTERNS`` + also require ``service`` to be one of the listed services. Args: service: The originating service name (e.g. ``"karaoke-backend"``). @@ -130,4 +147,8 @@ def should_ignore(service: str, message: str) -> IgnoreReason | None: if pattern.search(message): return IgnoreReason(pattern_name=pattern_name, reason=reason) + for pattern_name, services, pattern, reason in _SERVICE_IGNORE_PATTERNS: + if service in services and pattern.search(message): + return IgnoreReason(pattern_name=pattern_name, reason=reason) + return None diff --git a/backend/services/error_monitor/monitor.py b/backend/services/error_monitor/monitor.py index 4edbc7b3..2421044d 100644 --- a/backend/services/error_monitor/monitor.py +++ b/backend/services/error_monitor/monitor.py @@ -22,6 +22,7 @@ from google.cloud import logging as cloud_logging from google.cloud import secretmanager from backend.services.error_monitor.config import ( + BURST_COLLAPSE_MIN_PATTERNS, GCP_PROJECT, LEGACY_ROLLING_WINDOW_MINUTES, LOOKBACK_MINUTES, @@ -43,12 +44,14 @@ from backend.services.error_monitor.discord import ( format_daily_digest, format_incident_alert, format_new_pattern_alert, + format_pattern_burst_alert, format_spike_alert, ) from backend.services.error_monitor.firestore_adapter import ( ErrorPatternsAdapter, PatternData, UpsertResult, + is_below_alert_threshold, ) from backend.services.error_monitor.known_issues import should_ignore from backend.services.error_monitor.llm_analysis import analyze_patterns, find_duplicate_patterns @@ -446,6 +449,11 @@ class ErrorMonitor: pid = pattern.get("pattern_id") if not pid or pid in groups: continue + if is_below_alert_threshold(pattern, now): + # Gated pattern (e.g. frontend chunk-load failures, which the + # client auto-reloads) that hasn't hit its hourly threshold yet. + # Left unalerted so a later burst still fires. + continue groups[pid] = { "service": pattern.get("service", "unknown"), "resource_type": pattern.get("resource_type", "unknown"), @@ -568,31 +576,25 @@ class ErrorMonitor: incident_pattern_ids.update(incident_pids) - # ── Step 6: Individual alerts for new patterns not in incidents ──── + # ── Step 6: Alerts for new patterns not in incidents ─────────────── + # One root cause often emits many distinct messages (one per table, + # file, step). More than BURST_COLLAPSE_MIN_PATTERNS from one service + # in a run β†’ ONE burst summary; otherwise one alert per pattern. Every + # send still goes through the per-run cap in ErrorMonitorDiscord; + # unsent patterns stay unalerted and are picked up again (and collapse + # with that service's other leftovers) next run. + by_service: dict[str, list[str]] = {} for pattern_id in new_pattern_ids: - if pattern_id in incident_pattern_ids: + if pattern_id in incident_pattern_ids or pattern_id not in groups: continue - if pattern_id not in groups: - continue - group = groups[pattern_id] + by_service.setdefault(groups[pattern_id]["service"], []).append(pattern_id) - # Fetch fresh copy to get first_seen timestamp - existing = self.firestore_adapter.get_pattern(pattern_id) - first_seen = existing.get("first_seen", now_iso) if existing else now_iso - - msg = format_new_pattern_alert( - service=group["service"], - resource_type=group["resource_type"], - normalized_message=group["normalized"], - sample_message=group["sample_message"], - count=group["count"], - first_seen=first_seen, - ) - if self.discord.send_message(msg): - try: - self.firestore_adapter.update_pattern_alerted(pattern_id) - except Exception as exc: # noqa: BLE001 - logger.warning("Could not update alerted_at for %s: %s", pattern_id, exc) + for service, pids in by_service.items(): + if len(pids) > BURST_COLLAPSE_MIN_PATTERNS: + self._send_burst_alert(service, pids, groups, now_iso) + else: + for pattern_id in pids: + self._send_new_pattern_alert(pattern_id, groups[pattern_id], now_iso) # ── Step 7: Spike alerts ─────────────────────────────────────────── for spike in spike_patterns: @@ -607,6 +609,51 @@ class ErrorMonitor: # ── Step 8: Auto-resolve check ───────────────────────────────────── self._check_auto_resolve() + # ------------------------------------------------------------------ + # New-pattern alert helpers + # ------------------------------------------------------------------ + + def _mark_alerted(self, pattern_ids: list[str]) -> None: + for pid in pattern_ids: + try: + self.firestore_adapter.update_pattern_alerted(pid) + except Exception as exc: # noqa: BLE001 + logger.warning("Could not update alerted_at for %s: %s", pid, exc) + + def _send_new_pattern_alert(self, pattern_id: str, group: dict, now_iso: str) -> None: + # Fetch fresh copy to get first_seen timestamp + existing = self.firestore_adapter.get_pattern(pattern_id) + first_seen = existing.get("first_seen", now_iso) if existing else now_iso + + msg = format_new_pattern_alert( + service=group["service"], + resource_type=group["resource_type"], + normalized_message=group["normalized"], + sample_message=group["sample_message"], + count=group["count"], + first_seen=first_seen, + ) + if self.discord.send_message(msg): + self._mark_alerted([pattern_id]) + + def _send_burst_alert( + self, service: str, pattern_ids: list[str], groups: dict, now_iso: str + ) -> None: + patterns = [] + for pid in pattern_ids: + existing = self.firestore_adapter.get_pattern(pid) or {} + patterns.append( + { + "normalized_message": groups[pid]["normalized"], + "count": groups[pid]["count"], + "first_seen": existing.get("first_seen", now_iso), + "last_seen": existing.get("last_seen", now_iso), + } + ) + logger.info("Collapsing %d new %s patterns into one burst alert", len(pattern_ids), service) + if self.discord.send_message(format_pattern_burst_alert(service, patterns)): + self._mark_alerted(pattern_ids) + # ------------------------------------------------------------------ # Auto-resolve helper # ------------------------------------------------------------------ diff --git a/backend/services/loop_watchdog.py b/backend/services/loop_watchdog.py index 1436ff8c..b4b3d9af 100644 --- a/backend/services/loop_watchdog.py +++ b/backend/services/loop_watchdog.py @@ -13,7 +13,8 @@ This module makes such freezes self-diagnosing: * a daemon thread notices when the stamp goes stale (> ``stall_threshold_s``) and captures the *event-loop thread's* Python stack via ``sys._current_frames()`` β€” i.e. the exact line that is blocking β€” resampling - at ``sample_at_s`` so a long freeze shows where it actually spent its time; + at ``sample_at_s`` so a long freeze shows where it actually spent its time + (``EVENT_LOOP_STALL in progress`` lines, always WARNING); * when the loop resumes it logs ``EVENT_LOOP_STALL ended duration_ms=…`` (WARNING, or ERROR at ``error_threshold_s`` β€” the alerting log metric keys on that), and for stalls β‰₯ ``record_threshold_s`` hands a summary to @@ -248,19 +249,11 @@ class LoopWatchdog: if self._samples and self._samples[-1]["stack"] == stack: return # identical to the previous sample β€” nothing new to learn self._samples.append({"at_s": round(age, 1), "culprit": culprit, "stack": stack, "idle": False}) - # Same impact rule as _finish: an idle instance's stall never logs ERROR. - # (Requests still queued in the socket are invisible until the loop - # resumes, so this can under-call mid-stall; the ended line decides.) - no_impact = self.request_tracker is not None and self._requests_affected() == 0 - # Judge severity on blocked time only: frozen (throttled) time is discounted - # from the final duration too, so a post-thaw sample mustn't claim ERROR - # for a block that ends up below the threshold. - blocked = max(0.0, age - self._stall_frozen_s) - level = ( - logging.ERROR - if blocked >= self.error_threshold_s and not no_impact - else logging.WARNING - ) + # Always WARNING. The "ended" line is the one ERROR per freeze (>=10s + # blocked with a request affected): it's what the backend/event_loop_stalls + # metric counts, and the error monitor turns each ERROR line shape into its + # own #alerts pattern, so an ERROR here too meant two alerts per freeze. + level = logging.WARNING logger.log( level, "EVENT_LOOP_STALL in progress age_ms=%d culprit=%s\n%s", diff --git a/backend/services/ops_alerts.py b/backend/services/ops_alerts.py index 7856bc7b..367bba4a 100644 --- a/backend/services/ops_alerts.py +++ b/backend/services/ops_alerts.py @@ -111,7 +111,9 @@ def send_ops_alert(message: str) -> bool: from backend.services.discord_service import get_discord_notification_service service = get_discord_notification_service(webhook_url=webhook) - return service.post_message(message, webhook_url=webhook) + # suppress_embeds: alert text carries error strings with URLs (GCP docs, + # job links) that Discord would otherwise unfurl into big preview cards. + return service.post_message(message, webhook_url=webhook, suppress_embeds=True) except Exception as exc: # noqa: BLE001 - alerting must never raise logger.warning("ops_alerts: failed to send alert: %s", exc) return False diff --git a/backend/tests/test_discord_service.py b/backend/tests/test_discord_service.py index 69f0c70d..cba4b3ce 100644 --- a/backend/tests/test_discord_service.py +++ b/backend/tests/test_discord_service.py @@ -121,6 +121,27 @@ class TestDiscordNotificationServicePostMessage: timeout=30 ) + @patch("backend.services.discord_service.requests.post") + def test_post_message_suppress_embeds_sets_flag(self, mock_post): + """suppress_embeds sends flags=4 (SUPPRESS_EMBEDS) so URLs don't unfurl.""" + mock_post.return_value = MagicMock() + url = "https://discord.com/api/webhooks/123456789/abcdef" + service = DiscordNotificationService(webhook_url=url) + + assert service.post_message("see https://cloud.google.com/run", suppress_embeds=True) is True + payload = mock_post.call_args.kwargs["json"] + assert payload["flags"] == 4 + assert payload["allowed_mentions"] == {"parse": []} + + @patch("backend.services.discord_service.requests.post") + def test_video_notification_keeps_embed(self, mock_post): + """The YouTube 'New upload' post wants its video preview card.""" + mock_post.return_value = MagicMock() + service = DiscordNotificationService(webhook_url="https://discord.com/api/webhooks/1/a") + + service.post_video_notification("https://youtu.be/abc") + assert "flags" not in mock_post.call_args.kwargs["json"] + def test_post_message_no_webhook_raises(self): """Test that posting without webhook URL raises ValueError.""" service = DiscordNotificationService() diff --git a/backend/tests/test_loop_watchdog.py b/backend/tests/test_loop_watchdog.py index 11b762de..add68648 100644 --- a/backend/tests/test_loop_watchdog.py +++ b/backend/tests/test_loop_watchdog.py @@ -427,7 +427,10 @@ class TestRequestAwareSeverity: assert len(progress) == 3 # 1s, 5s, 15s samples assert all(r.levelno == logging.WARNING for r in progress) - def test_in_flight_in_progress_sample_at_15s_is_error(self, caplog): + def test_in_flight_in_progress_samples_stay_warning(self, caplog): + """A freeze used to log ERROR twice (the 15s "in progress" sample AND the + "ended" line), so the error monitor raised two alert patterns per freeze. + "in progress" is now always WARNING; "ended" alone carries the ERROR.""" caplog.set_level(logging.WARNING) wd, clock, tracker, recorded = self._wd() wd._loop_thread_id = 12345 @@ -438,7 +441,11 @@ class TestRequestAwareSeverity: patch("backend.services.loop_watchdog.sys._current_frames", return_value={12345: None}): self._stall(wd, clock, 16.0) progress = [r for r in caplog.records if "in progress" in r.getMessage()] - assert [r.levelno for r in progress] == [logging.WARNING, logging.WARNING, logging.ERROR] + assert [r.levelno for r in progress] == [logging.WARNING, logging.WARNING, logging.WARNING] + ended = self._ended(caplog) + assert len(ended) == 1 and ended[0].levelno == logging.ERROR + errors = [r for r in caplog.records if r.levelno >= logging.ERROR] + assert len(errors) == 1 # one freeze -> one ERROR line -> one alert pattern def test_in_flight_request_makes_it_error(self, caplog): caplog.set_level(logging.WARNING) diff --git a/backend/tests/test_ops_alerts.py b/backend/tests/test_ops_alerts.py index 868c7937..127eb0a2 100644 --- a/backend/tests/test_ops_alerts.py +++ b/backend/tests/test_ops_alerts.py @@ -106,6 +106,24 @@ def test_send_ops_alert_no_webhook_returns_false(monkeypatch): assert ops_alerts.send_ops_alert("hi") is False +def test_send_ops_alert_suppresses_link_embeds(monkeypatch): + """Ops alerts carry error text with URLs; Discord must not unfurl them.""" + import backend.services.discord_service as discord_service + + calls = [] + + class FakeService: + def post_message(self, message, webhook_url=None, suppress_embeds=False): + calls.append((message, webhook_url, suppress_embeds)) + return True + + monkeypatch.setattr(ops_alerts, "_is_enabled", lambda: True) + monkeypatch.setattr(ops_alerts, "_get_webhook_url", lambda: "https://discord.com/api/webhooks/1/a") + monkeypatch.setattr(discord_service, "get_discord_notification_service", lambda **kw: FakeService()) + assert ops_alerts.send_ops_alert("boom https://cloud.google.com/x") is True + assert calls == [("boom https://cloud.google.com/x", "https://discord.com/api/webhooks/1/a", True)] + + # --------------------------------------------------------------------------- # notify_job_failed # --------------------------------------------------------------------------- diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 81d37000..4db77942 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -100,13 +100,38 @@ How the webhooks resolve: to it fails, the message falls back to #alerts. The error monitor rate-limits each channel separately (10/run each), so noise can't consume the alert budget. +Keeping #alerts readable (one root cause β†’ one message): + +- **Burst collapse**: when a run has more than 3 new patterns for one service + (counting leftover un-alerted ones), the monitor sends ONE "N new error patterns + from " message with the top 5 by count, first/last seen and a + `scripts/query-error-patterns.py --service ` pointer, and marks them all + alerted (`BURST_COLLAPSE_MIN_PATTERNS` in `error_monitor/config.py`). It's + deterministic; LLM grouping is off in prod. The 10/run cap still applies. +- **No link previews**: every error-monitor and `ops_alerts` post sets Discord + `flags: 4` (SUPPRESS_EMBEDS), so quoted GCP docs URLs don't unfurl into cards. +- **Gated patterns**: a pattern with `alert_min_hourly_count` stays un-alerted + until more than that many occurrences land within an hour + (`firestore_adapter.is_below_alert_threshold`). Used for frontend chunk-load errors. +- **Watchdog**: `EVENT_LOOP_STALL in progress` samples are WARNING. Only the + `ended` line is ERROR (β‰₯10s blocked with a request affected), so one freeze is + one pattern. The `backend/event_loop_stalls` metric keys on that line too. +- **Known noise** (`known_issues.py`) can be service-scoped, e.g. + github-runner-manager's scale-up "no available instance" aborts. + Frontend crashes are captured via `window.onerror`, `unhandledrejection`, Next.js `error.tsx`, and a `CrashReportBoundary` around the review UIs. They POST to `/api/client-errors`, which normalises and upserts into the same `error_patterns` collection with `service="frontend"` so they flow through the existing Discord-alerting pipeline. `ChunkLoadError` after deploy triggers an automatic hard reload; the `CrashReport` card also checks `/version.json` and shows an -"Update now" CTA when the running bundle is stale. +"Update now" CTA when the running bundle is stale. Because the client recovers, +chunk-load reports (`is_chunk_load_error`, mirrors `isChunkLoadError` in +`frontend/lib/version-check.ts`) collapse into one pattern **per build** that +alerts only above 10 in an hour, which means a broken deploy. Safari/Firefox +stacks have no `Name: message` header, so ingestion prepends the message before +hashing. Otherwise every error from one frame shared a pattern named +`@https://gen.nomadkaraoke.com`. Noise is dropped before alerting: benign messages (media `AbortError`, ResizeObserver loop) and opaque cross-origin `Script error.` events (no error object β€” thrown by diff --git a/infrastructure/modules/divebar_mirror.py b/infrastructure/modules/divebar_mirror.py index 4e460063..dd33ee68 100644 --- a/infrastructure/modules/divebar_mirror.py +++ b/infrastructure/modules/divebar_mirror.py @@ -32,6 +32,11 @@ DIVEBAR_FOLDER_ID = "1zxnSZcE03gzy0YVGOdnTrEIi8It_3Wu8" BIGQUERY_DATASET = "karaoke_decide" +# Cloud Scheduler HTTP attempt deadline for the divebar-mirror jobs. Must exceed +# the function's timeout_seconds (540) or a slow run gets retried mid-sync. +SCHEDULER_ATTEMPT_DEADLINE = "600s" + + def create_divebar_mirror_resources(all_secrets: dict) -> dict: """Create all Divebar mirror infrastructure resources.""" resources = {} @@ -229,6 +234,10 @@ def create_divebar_mirror_resources(all_secrets: dict) -> dict: service_account_email=sa.email, ), ), + # Default deadline is 180s but a run takes 360-525s: the scheduler retried + # still-running syncs, the retries queued behind them (max 1 instance) and + # the third overlapping run OOMed. Outlast the 540s function timeout. + attempt_deadline=SCHEDULER_ATTEMPT_DEADLINE, retry_config=cloudscheduler.JobRetryConfigArgs( retry_count=2, min_backoff_duration="60s", @@ -260,6 +269,10 @@ def create_divebar_mirror_resources(all_secrets: dict) -> dict: service_account_email=sa.email, ), ), + # Default deadline is 180s but a run takes 360-525s: the scheduler retried + # still-running syncs, the retries queued behind them (max 1 instance) and + # the third overlapping run OOMed. Outlast the 540s function timeout. + attempt_deadline=SCHEDULER_ATTEMPT_DEADLINE, retry_config=cloudscheduler.JobRetryConfigArgs( retry_count=2, min_backoff_duration="60s", diff --git a/pyproject.toml b/pyproject.toml index 6df1e78b..1124af99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.278.9" +version = "0.278.10" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/services/error_monitor/test_discord.py b/tests/unit/services/error_monitor/test_discord.py index 69939bec..a21e23bc 100644 --- a/tests/unit/services/error_monitor/test_discord.py +++ b/tests/unit/services/error_monitor/test_discord.py @@ -536,7 +536,7 @@ class TestErrorMonitorDiscordSendMessage: assert result is True mock_post.assert_called_once_with( "https://discord.com/api/webhooks/123/abc", - json={"content": "Hello Discord!"}, + json={"content": "Hello Discord!", "flags": 4}, timeout=30, ) @@ -636,7 +636,7 @@ class TestErrorMonitorDiscordNoiseChannel: client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) with mock.patch("requests.post", return_value=self._ok()) as mock_post: assert client.send_noise_message("fyi") is True - mock_post.assert_called_once_with(self.NOISE, json={"content": "fyi"}, timeout=30) + mock_post.assert_called_once_with(self.NOISE, json={"content": "fyi", "flags": 4}, timeout=30) def test_send_message_still_posts_to_alert_webhook(self): from backend.services.error_monitor.discord import ErrorMonitorDiscord @@ -644,7 +644,7 @@ class TestErrorMonitorDiscordNoiseChannel: client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) with mock.patch("requests.post", return_value=self._ok()) as mock_post: assert client.send_message("problem") is True - mock_post.assert_called_once_with(self.ALERT, json={"content": "problem"}, timeout=30) + mock_post.assert_called_once_with(self.ALERT, json={"content": "problem", "flags": 4}, timeout=30) def test_noise_without_noise_webhook_falls_back_to_alert_webhook(self): from backend.services.error_monitor.discord import ErrorMonitorDiscord @@ -652,7 +652,7 @@ class TestErrorMonitorDiscordNoiseChannel: client = ErrorMonitorDiscord(webhook_url=self.ALERT) with mock.patch("requests.post", return_value=self._ok()) as mock_post: assert client.send_noise_message("fyi") is True - mock_post.assert_called_once_with(self.ALERT, json={"content": "fyi"}, timeout=30) + mock_post.assert_called_once_with(self.ALERT, json={"content": "fyi", "flags": 4}, timeout=30) def test_noise_delivery_failure_retries_on_alert_webhook(self): """A broken noise webhook must never silently drop the message.""" @@ -685,3 +685,70 @@ class TestErrorMonitorDiscordNoiseChannel: with mock.patch("requests.post") as mock_post: assert client.send_noise_message("fyi") is False mock_post.assert_not_called() + + +# --------------------------------------------------------------------------- +# Link-preview suppression +# --------------------------------------------------------------------------- + + +class TestSuppressEmbeds: + """Every error-monitor post sets flags=4 (SUPPRESS_EMBEDS): alerts quote GCP + docs URLs that Discord unfurled into big "AI and Cloud Computing Services" + cards, burying the alert text.""" + + def test_alert_and_noise_posts_suppress_embeds(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord( + webhook_url="https://discord.com/api/webhooks/1/a", + noise_webhook_url="https://discord.com/api/webhooks/2/n", + ) + with mock.patch("requests.post") as mock_post: + client.send_message("see https://cloud.google.com/run/docs") + client.send_noise_message("fyi https://cloud.google.com/run/docs") + assert [c.kwargs["json"]["flags"] for c in mock_post.call_args_list] == [4, 4] + + +# --------------------------------------------------------------------------- +# Burst alert formatting +# --------------------------------------------------------------------------- + + +class TestFormatPatternBurstAlert: + def _patterns(self, n): + return [ + { + "normalized_message": f"MusicBrainz mirror: kept previous musicbrainz.table_{i} (raw table not loaded)", + "count": i + 1, + "first_seen": f"2026-10-07T0{i % 10}:00:00+00:00", + "last_seen": f"2026-10-08T0{i % 10}:30:00+00:00", + } + for i in range(n) + ] + + def test_summarises_burst_in_one_message(self): + from backend.services.error_monitor.discord import format_pattern_burst_alert + + msg = format_pattern_burst_alert("mb-refresh", self._patterns(73)) + assert "73 new error patterns" in msg + assert "mb-refresh" in msg + assert "scripts/query-error-patterns.py --service mb-refresh" in msg + assert "2026-10-07T00:00:00+00:00" in msg # earliest first_seen + assert "2026-10-08T09:30:00+00:00" in msg # latest last_seen + assert len(msg) <= 2000 + + def test_shows_top_five_by_count(self): + from backend.services.error_monitor.discord import format_pattern_burst_alert + + msg = format_pattern_burst_alert("mb-refresh", self._patterns(73)) + shown = [line for line in msg.splitlines() if "table_" in line] + assert len(shown) == 5 + assert "table_72 " in shown[0] # highest count first + assert "…and 68 more" in msg + + def test_total_occurrences(self): + from backend.services.error_monitor.discord import format_pattern_burst_alert + + msg = format_pattern_burst_alert("svc", self._patterns(4)) + assert "10 occurrences" in msg # 1+2+3+4 diff --git a/tests/unit/services/error_monitor/test_firestore_adapter.py b/tests/unit/services/error_monitor/test_firestore_adapter.py index b241a4fd..25479479 100644 --- a/tests/unit/services/error_monitor/test_firestore_adapter.py +++ b/tests/unit/services/error_monitor/test_firestore_adapter.py @@ -1185,3 +1185,57 @@ class TestGetUnalertedNewPatterns: results = adapter.get_unalerted_new_patterns() assert results == [] + + +# --------------------------------------------------------------------------- +# Tests: alert gating (alert_min_hourly_count) +# --------------------------------------------------------------------------- + + +class TestAlertThresholdGate: + NOW = datetime(2026, 10, 18, 22, 0, 0, tzinfo=timezone.utc) + + def _new_doc_for(self, **kw): + db = _make_db() + doc_ref = MagicMock() + doc_ref.get.return_value = _make_doc_snapshot(exists=False) + db.collection.return_value.document.return_value = doc_ref + PatternData = _import_pattern_data() + data = PatternData( + pattern_id="p", service="frontend", resource_type="browser", + normalized_message="m", sample_message="s", count=1, timestamp=self.NOW, **kw, + ) + _import_adapter()(db=db).upsert_pattern(data) + return doc_ref.set.call_args[0][0] + + def test_new_pattern_stores_threshold(self): + assert self._new_doc_for(alert_min_hourly_count=10)["alert_min_hourly_count"] == 10 + + def test_ungated_pattern_has_no_threshold_field(self): + assert "alert_min_hourly_count" not in self._new_doc_for() + + def _pattern(self, threshold, minutes_ago): + return { + "alert_min_hourly_count": threshold, + "rolling_counts": [ + {"ts": (self.NOW - timedelta(minutes=m)).isoformat(), "count": 1} for m in minutes_ago + ], + } + + def test_below_threshold_is_held(self): + from backend.services.error_monitor.firestore_adapter import is_below_alert_threshold + + # 10 in the last hour is "not more than 10"; an old burst doesn't count. + p = self._pattern(10, list(range(0, 50, 5)) + [61, 70, 90]) + assert is_below_alert_threshold(p, now=self.NOW) is True + + def test_burst_over_threshold_alerts(self): + from backend.services.error_monitor.firestore_adapter import is_below_alert_threshold + + p = self._pattern(10, list(range(0, 55, 5))) # 11 within the hour + assert is_below_alert_threshold(p, now=self.NOW) is False + + def test_ungated_pattern_never_held(self): + from backend.services.error_monitor.firestore_adapter import is_below_alert_threshold + + assert is_below_alert_threshold({"rolling_counts": []}, now=self.NOW) is False diff --git a/tests/unit/services/error_monitor/test_frontend_ingestion.py b/tests/unit/services/error_monitor/test_frontend_ingestion.py index e27af2c5..cf2f0383 100644 --- a/tests/unit/services/error_monitor/test_frontend_ingestion.py +++ b/tests/unit/services/error_monitor/test_frontend_ingestion.py @@ -174,3 +174,101 @@ def test_is_opaque_script_error_false_for_other_messages(message): ) def test_is_opaque_script_error_false_when_error_is_located(source, extra): assert is_opaque_script_error("Error: Script error.", source, extra) is False + + +# --------------------------------------------------------------------------- +# Safari/Firefox stacks don't start with the message +# --------------------------------------------------------------------------- + +def _report(message, stack, release="9d6aa3df"): + return FrontendErrorReport( + message=message, stack=stack, url="https://gen.nomadkaraoke.com/", + user_agent="Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) Safari/604.1", + release=release, user_email=None, viewport=None, locale="en", extra=None, + ) + + +SAFARI_STACK = "@https://gen.nomadkaraoke.com/_next/static/chunks/turbopack-7c45166c9b701384.js:1:5749" + + +def test_safari_stack_without_message_gets_message_prepended(): + """2026-10-18: a Safari pattern was named "@https://gen.nomadkaraoke.com" + because its stack has no "Name: message" header, so every Safari error + thrown from that bundle frame collapsed into one meaningless pattern.""" + data = build_pattern_data(_report("TypeError: undefined is not an object", SAFARI_STACK)) + assert data.normalized_message.startswith("TypeError: undefined is not an object") + + +def test_safari_errors_from_same_frame_get_distinct_patterns(): + a = build_pattern_data(_report("TypeError: undefined is not an object", SAFARI_STACK)) + b = build_pattern_data(_report("RangeError: Maximum call stack size exceeded", SAFARI_STACK)) + assert a.pattern_id != b.pattern_id + + +def test_chrome_stack_hash_unchanged(): + """Chrome stacks already start with the message: same hash as before.""" + from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message + + stack = "TypeError: x is null\n at foo (https://gen.nomadkaraoke.com/a.js:1:2)" + data = build_pattern_data(_report("TypeError: x is null", stack)) + assert data.pattern_id == compute_pattern_hash("frontend", normalize_message(stack)) + + +# --------------------------------------------------------------------------- +# Chunk-load errors: alert only on a per-build burst +# --------------------------------------------------------------------------- + +from backend.services.error_monitor.frontend_ingestion import ( # noqa: E402 + CHUNK_LOAD_ALERT_PER_HOUR, + is_chunk_load_error, +) + + +@pytest.mark.parametrize("message", [ + "Error: Failed to load chunk /_next/static/chunks/b87534e20efd1895.js from module 64893", + "ChunkLoadError: Loading chunk 123 failed.", + "Error: Loading chunk app-pages failed", + "TypeError: Failed to fetch dynamically imported module: https://x/y.js", + "TypeError: Importing a module script failed.", +]) +def test_is_chunk_load_error_true(message): + assert is_chunk_load_error(message) is True + + +@pytest.mark.parametrize("message", [ + "TypeError: Cannot read properties of null", + "Error: chunk size too large for upload", + "", + None, +]) +def test_is_chunk_load_error_false(message): + assert is_chunk_load_error(message) is False + + +def test_chunk_errors_collapse_into_one_pattern_per_build(): + a = build_pattern_data(_report( + "Error: Failed to load chunk /_next/static/chunks/b87534e20efd1895.js from module 64893", + "Error: Failed to load chunk …\n at https://gen.nomadkaraoke.com/_next/static/chunks/turbopack-1.js:1:5744", + )) + b = build_pattern_data(_report( + "Error: Failed to load chunk /_next/static/chunks/8226dd5ce61ba9e5.js from module 1234", SAFARI_STACK, + )) + other_build = build_pattern_data(_report( + "Error: Failed to load chunk /_next/static/chunks/8226dd5ce61ba9e5.js from module 1234", + SAFARI_STACK, release="0000abcd", + )) + assert a.pattern_id == b.pattern_id + assert a.pattern_id != other_build.pattern_id + assert "9d6aa3df" in a.normalized_message + # The sample keeps the real error for whoever investigates. + assert "8226dd5ce61ba9e5.js" in b.sample_message + + +def test_chunk_errors_are_gated_by_hourly_threshold(): + data = build_pattern_data(_report("ChunkLoadError: Loading chunk 9 failed.", None)) + assert data.alert_min_hourly_count == CHUNK_LOAD_ALERT_PER_HOUR == 10 + + +def test_ordinary_errors_are_not_gated(): + data = build_pattern_data(_report("TypeError: x is null", "TypeError: x is null\n at f (a.js:1:1)")) + assert data.alert_min_hourly_count is None diff --git a/tests/unit/services/error_monitor/test_known_issues.py b/tests/unit/services/error_monitor/test_known_issues.py index 6263f8eb..0344956d 100644 --- a/tests/unit/services/error_monitor/test_known_issues.py +++ b/tests/unit/services/error_monitor/test_known_issues.py @@ -516,3 +516,33 @@ class TestCaseInsensitiveMatching: result = should_ignore("karaoke-backend", "Container Called EXIT(0)") assert result is not None + + +class TestRunnerManagerNoAvailableInstance: + """github-runner-manager's Cloud Run 'no available instance' abort is a burst + during runner scale-up; the runners still dispatch (seen 09-20, 09-23, 09-27, + 10-08), so it must not page.""" + + MSG = ( + "The request was aborted because there was no available instance. " + "Additional troubleshooting documentation can be found at: " + "https://cloud.google.com/run/docs/troubleshooting#abort-request" + ) + + def test_ignored_for_runner_manager(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("github-runner-manager", self.MSG) + assert result is not None + assert result.pattern_name == "runner_manager_no_available_instance" + + def test_still_alerts_for_other_services(self): + # On the customer-facing backend the same abort means dropped requests. + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("karaoke-backend", self.MSG) is None + + def test_other_runner_manager_errors_still_alert(self): + from backend.services.error_monitor.known_issues import should_ignore + + assert should_ignore("github-runner-manager", "Failed to create runner VM: quota exceeded") is None diff --git a/tests/unit/services/error_monitor/test_monitor.py b/tests/unit/services/error_monitor/test_monitor.py index 07ddd797..7bac0fad 100644 --- a/tests/unit/services/error_monitor/test_monitor.py +++ b/tests/unit/services/error_monitor/test_monitor.py @@ -1059,3 +1059,155 @@ class TestNoiseWebhookResolution: monitor = m.ErrorMonitor() assert monitor.discord.webhook_url == self.ALERT assert monitor.discord.noise_webhook_url == self.NOISE + + +# --------------------------------------------------------------------------- +# Tests: burst collapse + alert gating (2026-10-18 #alerts noise work) +# --------------------------------------------------------------------------- + + +class TestBurstCollapse: + """>3 new patterns from one service in a run -> ONE #alerts message. + + 2026-10-07/08: one bad mb-refresh run produced 73 distinct "New Error + Pattern" messages that dripped into #alerts 10 an hour (the per-run cap).""" + + NOW = "2026-10-08T03:00:00+00:00" + + def _pattern(self, pid, service, i=0): + return { + "pattern_id": pid, + "service": service, + "resource_type": "cloud_run_job", + "normalized_message": f"{service} failure {pid}", + "sample_message": f"sample {pid}", + "total_count": i + 1, + "status": "new", + "alerted_at": None, + "first_seen": "2026-10-07T02:00:00+00:00", + "last_seen": "2026-10-08T02:30:00+00:00", + "rolling_counts": [], + } + + def _monitor(self, patterns, real_discord=False): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + monitor = _make_monitor_with_mocks() + monitor.logging_client.list_entries.return_value = iter([]) + by_id = {p["pattern_id"]: p for p in patterns} + monitor.firestore_adapter.get_unalerted_new_patterns.return_value = patterns + monitor.firestore_adapter.get_pattern.side_effect = lambda pid: by_id.get(pid) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + if real_discord: + monitor.discord = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/1/a") + else: + monitor.discord.send_message.return_value = True + return monitor + + def _alerted_ids(self, monitor): + return [c.args[0] for c in monitor.firestore_adapter.update_pattern_alerted.call_args_list] + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_73_patterns_one_service_send_one_message(self, *_): + patterns = [self._pattern(f"mb{i}", "mb-refresh", i) for i in range(73)] + patterns += [self._pattern("be1", "karaoke-backend"), self._pattern("be2", "karaoke-backend")] + monitor = self._monitor(patterns) + + monitor._run_monitor_cycle() + + msgs = [c.args[0] for c in monitor.discord.send_message.call_args_list] + burst = [m for m in msgs if "mb-refresh" in m] + assert len(burst) == 1 + assert "73 new error patterns" in burst[0] + assert "scripts/query-error-patterns.py --service mb-refresh" in burst[0] + # karaoke-backend had only 2 -> still individual alerts + assert sum("New Error Pattern" in m for m in msgs) == 2 + assert len(msgs) == 3 + assert sorted(self._alerted_ids(monitor)) == sorted(p["pattern_id"] for p in patterns) + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_exactly_three_patterns_stay_individual(self, *_): + patterns = [self._pattern(f"p{i}", "svc", i) for i in range(3)] + monitor = self._monitor(patterns) + + monitor._run_monitor_cycle() + + msgs = [c.args[0] for c in monitor.discord.send_message.call_args_list] + assert len(msgs) == 3 + assert all("New Error Pattern" in m for m in msgs) + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_failed_burst_send_marks_nothing_alerted(self, *_): + patterns = [self._pattern(f"p{i}", "svc", i) for i in range(5)] + monitor = self._monitor(patterns) + monitor.discord.send_message.return_value = False + + monitor._run_monitor_cycle() + + monitor.firestore_adapter.update_pattern_alerted.assert_not_called() + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_per_run_cap_still_holds(self, *_): + """15 services x 1 pattern + a burst: at most 10 posts; unsent stay unalerted.""" + from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN + + patterns = [self._pattern(f"s{i}", f"svc-{i}") for i in range(15)] + patterns += [self._pattern(f"b{i}", "burst-svc", i) for i in range(20)] + monitor = self._monitor(patterns, real_discord=True) + + with patch("requests.post") as mock_post: + monitor._run_monitor_cycle() + + assert mock_post.call_count == MAX_DISCORD_MESSAGES_PER_RUN + assert len(self._alerted_ids(monitor)) < len(patterns) + + +class TestAlertGate: + """Patterns carrying alert_min_hourly_count (frontend chunk-load failures) + alert only once more than that many land within an hour.""" + + def _chunk_pattern(self, recent): + from datetime import datetime, timedelta, timezone + + now = datetime.now(tz=timezone.utc) + return { + "pattern_id": "chunk-build-x", + "service": "frontend", + "resource_type": "browser", + "normalized_message": "Failed to load JS chunk on build x", + "sample_message": "Error: Failed to load chunk …", + "total_count": recent, + "status": "new", + "alerted_at": None, + "first_seen": now.isoformat(), + "last_seen": now.isoformat(), + "alert_min_hourly_count": 10, + "rolling_counts": [ + {"ts": (now - timedelta(minutes=i)).isoformat(), "count": 1} for i in range(recent) + ], + } + + def _run(self, pattern): + monitor = _make_monitor_with_mocks() + monitor.logging_client.list_entries.return_value = iter([]) + monitor.firestore_adapter.get_unalerted_new_patterns.return_value = [pattern] + monitor.firestore_adapter.get_pattern.return_value = pattern + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.discord.send_message.return_value = True + with patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False): + monitor._run_monitor_cycle() + return monitor + + def test_single_chunk_error_does_not_alert(self): + monitor = self._run(self._chunk_pattern(1)) + monitor.discord.send_message.assert_not_called() + monitor.firestore_adapter.update_pattern_alerted.assert_not_called() + + def test_chunk_burst_over_threshold_alerts(self): + monitor = self._run(self._chunk_pattern(11)) + assert monitor.discord.send_message.call_count == 1 + monitor.firestore_adapter.update_pattern_alerted.assert_called_once_with("chunk-build-x") diff --git a/tests/unit/test_divebar_mirror_scheduler.py b/tests/unit/test_divebar_mirror_scheduler.py new file mode 100644 index 00000000..4f03c1a7 --- /dev/null +++ b/tests/unit/test_divebar_mirror_scheduler.py @@ -0,0 +1,57 @@ +"""Guard: divebar-mirror Cloud Scheduler jobs must wait for the function to finish. + +Cloud Scheduler's default HTTP attempt_deadline is 180s, but a divebar-mirror run +takes 360-525s. The scheduler gave up on a still-running run and retried it +(retry_count=2); the retries queued behind it on the max_instance_count=1 +function, and the third concurrent run hit the 512 MiB limit (4 OOM alerts in 30 +days). The deadline has to outlast the function timeout so a slow run is never +retried while it is still working. +""" +from __future__ import annotations + +import ast +from pathlib import Path + +MODULE = Path(__file__).resolve().parents[2] / "infrastructure" / "modules" / "divebar_mirror.py" + + +def _kwargs(call: ast.Call) -> dict[str, ast.expr]: + return {kw.arg: kw.value for kw in call.keywords if kw.arg} + + +def _calls(tree: ast.AST, attr: str) -> list[ast.Call]: + return [ + n for n in ast.walk(tree) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == attr + ] + + +def _function_timeout_s(tree: ast.AST) -> int: + (svc,) = _calls(tree, "FunctionServiceConfigArgs") + return ast.literal_eval(_kwargs(svc)["timeout_seconds"]) + + +def _deadline(tree: ast.Module, node: ast.expr) -> str: + """Literal value, or the module-level constant the job references.""" + if isinstance(node, ast.Name): + for stmt in tree.body: + if isinstance(stmt, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == node.id for t in stmt.targets + ): + return ast.literal_eval(stmt.value) + raise AssertionError(f"constant {node.id} not found") + return ast.literal_eval(node) + + +def test_every_scheduler_job_outlasts_the_function_timeout(): + tree = ast.parse(MODULE.read_text()) + timeout = _function_timeout_s(tree) + jobs = [c for c in _calls(tree, "Job") if "http_target" in _kwargs(c)] + names = [ast.literal_eval(_kwargs(j)["name"]) for j in jobs] + assert names == ["divebar-mirror-daily", "divebar-mirror-refresh"] + for job, name in zip(jobs, names): + kw = _kwargs(job) + assert "attempt_deadline" in kw, f"{name}: no attempt_deadline (default 180s retries a running sync)" + deadline = _deadline(tree, kw["attempt_deadline"]) + assert deadline.endswith("s") + assert int(deadline[:-1]) > timeout, f"{name}: {deadline} <= function timeout {timeout}s"