diff --git a/backend/services/error_monitor/__init__.py b/backend/services/error_monitor/__init__.py new file mode 100644 index 00000000..0957c199 --- /dev/null +++ b/backend/services/error_monitor/__init__.py @@ -0,0 +1 @@ +"""Production error monitoring service for Nomad Karaoke.""" diff --git a/backend/services/error_monitor/config.py b/backend/services/error_monitor/config.py new file mode 100644 index 00000000..36685bcb --- /dev/null +++ b/backend/services/error_monitor/config.py @@ -0,0 +1,175 @@ +"""Configuration for the production error monitoring service. + +All constants, service lists, and environment-driven configuration +functions used across the error monitor modules. +""" + +import os + +# --------------------------------------------------------------------------- +# GCP project settings +# --------------------------------------------------------------------------- + +GCP_PROJECT: str = os.environ.get("GCP_PROJECT", "nomadkaraoke") +GCP_REGION: str = os.environ.get("GCP_REGION", "us-central1") + +# --------------------------------------------------------------------------- +# Log querying settings +# --------------------------------------------------------------------------- + +#: How far back to look for errors on each run (minutes). +LOOKBACK_MINUTES: int = 15 + +#: Maximum number of log entries to fetch per service per run. +MAX_LOG_ENTRIES: int = 500 + +# --------------------------------------------------------------------------- +# Monitored services +# --------------------------------------------------------------------------- + +MONITORED_CLOUD_RUN_SERVICES: list[str] = [ + "karaoke-backend", + "karaoke-decide", + "audio-separator", +] + +MONITORED_CLOUD_RUN_JOBS: list[str] = [ + "video-encoding-job", + "lyrics-transcription-job", + "audio-separation-job", + "audio-download-job", +] + +MONITORED_CLOUD_FUNCTIONS: list[str] = [ + "gdrive-validator", + "runner_manager", + "backup_to_aws", + "divebar_mirror", + "kn_data_sync", + "divebar_lookup", + "encoding_worker_idle", +] + +MONITORED_GCE_INSTANCES: list[str] = [ + "encoding-worker-a", + "encoding-worker-b", + "flacfetch-vm", + "divebar-sync-vm", +] + +# --------------------------------------------------------------------------- +# Service dependency map +# Human-readable description of which services depend on which workers. +# Used as LLM context when generating root-cause analysis. +# --------------------------------------------------------------------------- + +SERVICE_DEPENDENCY_MAP: str = """\ +karaoke-backend (main API) + → video-encoding-job (Cloud Run Job, triggered per order) + → lyrics-transcription-job (Cloud Run Job, triggered after audio-sep) + → audio-separation-job (Cloud Run Job, runs on Cloud Run GPU) + → audio-download-job (Cloud Run Job, uses flacfetch-vm) + → gdrive-validator (Cloud Function, validates output files) + → encoding-worker-a/b (GCE VMs, long-running encoding workers) + +karaoke-decide (catalog / discovery API) + → kn_data_sync (Cloud Function, syncs KaraokeNerds catalog) + → divebar_mirror (Cloud Function, mirrors Divebar catalog) + → divebar_lookup (Cloud Function, single-song lookup) + +audio-separator (Cloud Run service, GPU stem separation) + → audio-separation-job (Cloud Run Job, launched by audio-separator) + +flacfetch-vm (GCE VM) + → audio-download-job (initiated by karaoke-backend) + +divebar-sync-vm (GCE VM) + → divebar_mirror (Cloud Function, reads from this VM) + +Shared infrastructure: + → runner_manager (Cloud Function, manages encoding worker lifecycle) + → encoding_worker_idle (Cloud Function, shuts down idle encoding workers) + → backup_to_aws (Cloud Function, daily GCS → S3 backup) +""" + +# --------------------------------------------------------------------------- +# LLM analysis settings +# --------------------------------------------------------------------------- + +LLM_ANALYSIS_MODEL: str = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-2.0-flash-001") +LLM_VERTEX_LOCATION: str = os.environ.get("LLM_VERTEX_LOCATION", "us-central1") + +#: Minimum number of active error patterns required before triggering LLM analysis. +MIN_PATTERNS_FOR_ANALYSIS: int = 3 + +# --------------------------------------------------------------------------- +# Spike detection +# --------------------------------------------------------------------------- + +#: An error count is a spike if it exceeds SPIKE_MULTIPLIER × baseline average. +SPIKE_MULTIPLIER: float = 5.0 + +#: Minimum absolute error count before spike detection fires (avoids noise). +SPIKE_MIN_COUNT: int = 5 + +# --------------------------------------------------------------------------- +# Auto-resolution settings +# --------------------------------------------------------------------------- + +#: Multiply LOOKBACK_MINUTES by this to get auto-resolve window in minutes. +AUTO_RESOLVE_MULTIPLIER: int = 8 + +#: Minimum hours before a pattern can be auto-resolved. +AUTO_RESOLVE_MIN_HOURS: int = 6 + +#: Maximum hours before a pattern is force-resolved even if still active. +AUTO_RESOLVE_MAX_HOURS: int = 168 # 1 week + +#: Fallback resolve window (hours) if baseline data is unavailable. +AUTO_RESOLVE_FALLBACK_HOURS: int = 48 + +# --------------------------------------------------------------------------- +# Discord settings +# --------------------------------------------------------------------------- + +#: Maximum Discord messages to send per monitor run (rate-limit guard). +MAX_DISCORD_MESSAGES_PER_RUN: int = 10 + +#: Discord API hard limit on message length. +DISCORD_MAX_MESSAGE_LENGTH: int = 2000 + +# --------------------------------------------------------------------------- +# Pattern tracking settings +# --------------------------------------------------------------------------- + +#: Maximum number of active error patterns to track in Firestore. +MAX_ACTIVE_PATTERNS: int = 500 + +#: Rolling window used when computing error baselines. +ROLLING_WINDOW_DAYS: int = 7 + +#: Maximum length of a normalized error message stored in Firestore. +MAX_NORMALIZED_MESSAGE_LENGTH: int = 200 + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- + + +def get_llm_enabled() -> bool: + """Return True if LLM analysis is enabled via the LLM_ANALYSIS_ENABLED env var.""" + return os.environ.get("LLM_ANALYSIS_ENABLED", "false").strip().lower() == "true" + + +def get_discord_webhook_secret_name() -> str: + """Return the Secret Manager secret name for the Discord webhook URL.""" + return os.environ.get("DISCORD_WEBHOOK_SECRET_NAME", "error-monitor-discord-webhook") + + +def get_digest_mode() -> bool: + """Return True if the monitor should run in digest mode (batch alerts instead of immediate). + + In digest mode all alerts are bundled into a single Discord message sent at + the end of the run, rather than sending one message per error pattern. + """ + return os.environ.get("DIGEST_MODE", "false").strip().lower() == "true" diff --git a/backend/services/error_monitor/discord.py b/backend/services/error_monitor/discord.py new file mode 100644 index 00000000..6f5c52f1 --- /dev/null +++ b/backend/services/error_monitor/discord.py @@ -0,0 +1,323 @@ +"""Discord alert client for the production error monitoring service. + +Provides formatting functions for each alert type, and an ErrorMonitorDiscord +webhook client that rate-limits outbound messages to MAX_DISCORD_MESSAGES_PER_RUN +per monitor run. + +Usage: + from backend.services.error_monitor.discord import ( + ErrorMonitorDiscord, + format_new_pattern_alert, + format_incident_alert, + format_auto_resolved_alert, + format_spike_alert, + format_daily_digest, + ) + + discord = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/...") + msg = format_new_pattern_alert(...) + discord.send_message(msg) +""" + +import logging +from datetime import datetime +from zoneinfo import ZoneInfo + +import requests + +from backend.services.error_monitor.config import ( + DISCORD_MAX_MESSAGE_LENGTH, + MAX_DISCORD_MESSAGES_PER_RUN, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +SEVERITY_EMOJI: dict[str, str] = { + "P0": "🚨", + "P1": "🔴", + "P2": "⚠️", + "P3": "🟡", +} + +_EASTERN = ZoneInfo("America/New_York") + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _now_eastern() -> str: + """Return the current Eastern time as 'HH:MM AM/PM ET' (DST-aware).""" + now = datetime.now(tz=_EASTERN) + return now.strftime("%I:%M %p ET").lstrip("0") + + +def _truncate(msg: str) -> str: + """Truncate *msg* to DISCORD_MAX_MESSAGE_LENGTH characters.""" + if len(msg) <= DISCORD_MAX_MESSAGE_LENGTH: + return msg + suffix = "…" + return msg[: DISCORD_MAX_MESSAGE_LENGTH - len(suffix)] + suffix + + +# --------------------------------------------------------------------------- +# Formatting functions +# --------------------------------------------------------------------------- + + +def format_new_pattern_alert( + service: str, + resource_type: str, + normalized_message: str, + sample_message: str, + count: int, + first_seen: str, +) -> str: + """Format a Discord alert for a newly-detected error pattern. + + Args: + service: Name of the service emitting the error. + resource_type: GCP resource type (e.g. 'cloud_run_service', 'gce_instance'). + normalized_message: Normalised error pattern string. + sample_message: A representative raw log message. + count: Number of occurrences seen so far. + first_seen: ISO 8601 timestamp when the pattern was first seen. + + Returns: + Formatted Discord message string, truncated to 2000 chars. + """ + lines = [ + f"🔴 **New Error Pattern Detected**", + f"", + f"**Service:** {service}", + f"**Resource type:** {resource_type}", + f"**Pattern:** {normalized_message}", + f"**Sample:** {sample_message}", + f"**Count:** {count}", + f"**First seen:** {first_seen}", + f"**Alerted at:** {_now_eastern()}", + f"", + f"To investigate: `/prod-investigate {service} \"{normalized_message}\"`", + ] + return _truncate("\n".join(lines)) + + +def format_incident_alert( + title: str, + severity: str, + root_cause: str, + suggested_fix: str, + primary_service: str, + patterns: list[dict], +) -> str: + """Format a Discord alert for an analysed incident (LLM-generated). + + Args: + title: Short incident title. + severity: Severity level — one of P0, P1, P2, P3. + root_cause: Human-readable root-cause analysis. + suggested_fix: Recommended remediation steps. + primary_service: The main service involved in the incident. + patterns: List of dicts, each with keys: service, normalized_message, count. + Only the first 5 are shown; normalized_message is truncated to 80 chars. + + Returns: + Formatted Discord message string, truncated to 2000 chars. + """ + emoji = SEVERITY_EMOJI.get(severity, "⚠️") + lines = [ + f"{emoji} **[{severity}] {title}**", + f"", + f"**Primary service:** {primary_service}", + f"**Root cause:** {root_cause}", + f"**Suggested fix:** {suggested_fix}", + ] + + shown_patterns = patterns[:5] + if shown_patterns: + lines.append("") + lines.append(f"**Patterns ({len(shown_patterns)} of {len(patterns)}):**") + for p in shown_patterns: + svc = p.get("service", "unknown") + msg = p.get("normalized_message", "")[:80] + cnt = p.get("count", 0) + lines.append(f" • `{svc}` — {msg} ({cnt}x)") + + lines.append("") + lines.append(f"**Alerted at:** {_now_eastern()}") + + return _truncate("\n".join(lines)) + + +def format_auto_resolved_alert(resolved_patterns: list[dict]) -> str: + """Format a consolidated auto-resolved alert. + + Args: + resolved_patterns: List of dicts, each with keys: service, message, hours_silent. + Only the first 8 are shown; message is truncated to 60 chars. + + Returns: + Formatted Discord message string, truncated to 2000 chars. + """ + lines = [ + f"✅ **Patterns Auto-Resolved**", + f"", + ] + + shown = resolved_patterns[:8] + if shown: + for p in shown: + svc = p.get("service", "unknown") + msg = p.get("message", "")[:60] + hours = p.get("hours_silent", 0) + lines.append(f" • `{svc}` — {msg} (silent {hours}h)") + else: + lines.append(" *(no patterns)*") + + if len(resolved_patterns) > 8: + lines.append(f" … and {len(resolved_patterns) - 8} more") + + lines.append("") + lines.append(f"**Resolved at:** {_now_eastern()}") + + return _truncate("\n".join(lines)) + + +def format_spike_alert( + service: str, + normalized_message: str, + current_count: int, + rolling_average: float, +) -> str: + """Format a Discord alert for an error rate spike. + + Args: + service: Name of the service. + normalized_message: Normalised error pattern string. + current_count: Error count in the current window. + rolling_average: Baseline rolling average count. + + Returns: + Formatted Discord message string, truncated to 2000 chars. + """ + if rolling_average > 0: + multiplier = current_count / rolling_average + multiplier_str = f"{multiplier:.1f}x" + else: + multiplier_str = "∞x" + + lines = [ + f"⚠️ **Error Rate Spike Detected**", + f"", + f"**Service:** {service}", + f"**Pattern:** {normalized_message}", + f"**Current count:** {current_count}", + f"**Normal (rolling avg):** {rolling_average}", + f"**Multiplier:** {multiplier_str} above normal", + f"", + f"**Alerted at:** {_now_eastern()}", + ] + return _truncate("\n".join(lines)) + + +def format_daily_digest( + total_errors: int, + new_patterns: int, + resolved_patterns: int, + active_patterns: int, + per_service_counts: dict, +) -> str: + """Format the daily 24h error digest. + + Args: + total_errors: Total error count in the last 24 hours. + new_patterns: Number of new error patterns detected. + resolved_patterns: Number of patterns auto-resolved. + active_patterns: Total number of currently active patterns. + per_service_counts: Mapping of service name → error count. + Displayed sorted by count descending. + + Returns: + Formatted Discord message string, truncated to 2000 chars. + """ + lines = [ + f"📊 **Daily Error Digest (24h)**", + f"", + f"**Total errors:** {total_errors}", + f"**New patterns:** {new_patterns}", + f"**Resolved patterns:** {resolved_patterns}", + f"**Active patterns:** {active_patterns}", + ] + + if per_service_counts: + lines.append("") + lines.append("**Per-service breakdown:**") + sorted_services = sorted(per_service_counts.items(), key=lambda kv: kv[1], reverse=True) + for svc, cnt in sorted_services: + lines.append(f" • `{svc}`: {cnt}") + + lines.append("") + lines.append(f"**Generated at:** {_now_eastern()}") + + return _truncate("\n".join(lines)) + + +# --------------------------------------------------------------------------- +# Discord webhook client +# --------------------------------------------------------------------------- + + +class ErrorMonitorDiscord: + """Webhook client for sending error monitor alerts to Discord. + + Enforces a per-run rate limit of MAX_DISCORD_MESSAGES_PER_RUN to avoid + flooding the channel during high-volume incident windows. + + Attributes: + webhook_url: The Discord incoming webhook URL. + messages_sent: Counter of messages attempted this run (success or failure). + """ + + def __init__(self, webhook_url: str) -> None: + self.webhook_url = webhook_url + self.messages_sent: int = 0 + + def send_message(self, content: str) -> bool: + """Send a message to the configured Discord webhook. + + Rate-limits: if messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN, the + message is silently skipped and False is returned without incrementing + the counter. + + Args: + content: The message text to send (should be pre-formatted and + pre-truncated to ≤2000 chars). + + Returns: + True if the message was sent successfully, False otherwise. + """ + if self.messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN: + logger.warning( + "Discord rate limit reached (%d messages sent this run); skipping message.", + self.messages_sent, + ) + return False + + try: + response = requests.post( + self.webhook_url, + json={"content": content}, + timeout=30, + ) + response.raise_for_status() + logger.debug("Discord message sent successfully.") + return True + except Exception as exc: + logger.error("Failed to send Discord message: %s", exc) + return False + finally: + self.messages_sent += 1 diff --git a/backend/services/error_monitor/firestore_adapter.py b/backend/services/error_monitor/firestore_adapter.py new file mode 100644 index 00000000..cddacac3 --- /dev/null +++ b/backend/services/error_monitor/firestore_adapter.py @@ -0,0 +1,489 @@ +"""Firestore adapter for the production error monitoring service. + +Handles all CRUD for three Firestore collections: + - error_patterns: normalized error patterns (pattern_id as doc ID) + - error_incidents: groups of related patterns (incident_id as doc ID) + - discord_alerts: audit trail for Discord messages (alert_id as doc ID) +""" + +from __future__ import annotations + +import secrets +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any + +from backend.services.error_monitor.config import ( + AUTO_RESOLVE_FALLBACK_HOURS, + AUTO_RESOLVE_MAX_HOURS, + AUTO_RESOLVE_MIN_HOURS, + AUTO_RESOLVE_MULTIPLIER, + ROLLING_WINDOW_DAYS, +) + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class PatternData: + """Input data for creating or updating an error pattern.""" + + pattern_id: str + service: str + resource_type: str + normalized_message: str + sample_message: str + count: int + timestamp: datetime + + +@dataclass +class UpsertResult: + """Result of an upsert_pattern operation.""" + + pattern_id: str + is_new: bool + previous_status: str | None = None + + +# --------------------------------------------------------------------------- +# Collection names +# --------------------------------------------------------------------------- + +_COL_PATTERNS = "error_patterns" +_COL_INCIDENTS = "error_incidents" +_COL_ALERTS = "discord_alerts" + +# Statuses that can be queried as "active" +_ACTIVE_STATUSES = ["new", "acknowledged", "known"] + +# Statuses from which a pattern is reactivated when the error recurs +_REACTIVATION_STATUSES = {"auto_resolved", "fixed"} + +# Statuses eligible for auto-resolve checking +_AUTO_RESOLVE_ELIGIBLE = {"new", "acknowledged"} + + +def _utcnow() -> datetime: + """Return current UTC datetime.""" + return datetime.now(tz=timezone.utc) + + +def _iso(dt: datetime) -> str: + """Serialise a datetime to ISO-8601 string.""" + return dt.isoformat() + + +def _generate_id(prefix: str, fmt: str) -> str: + """Generate a time-stamped ID with 8 hex random suffix. + + Args: + prefix: e.g. "inc" or "alert" + fmt: strftime format string applied to utcnow(), e.g. "%Y%m%d_%H%M" + """ + ts = _utcnow().strftime(fmt) + suffix = secrets.token_hex(4) # 8 hex chars + return f"{prefix}_{ts}_{suffix}" + + +# --------------------------------------------------------------------------- +# Adapter +# --------------------------------------------------------------------------- + + +class ErrorPatternsAdapter: + """Firestore CRUD adapter for error monitoring collections. + + Args: + db: Optional Firestore client. If omitted, a default client for the + ``nomadkaraoke`` GCP project is constructed on first use. + """ + + def __init__(self, db=None) -> None: + if db is None: + from google.cloud import firestore # type: ignore[import] + + db = firestore.Client(project="nomadkaraoke") + self._db = db + + # ------------------------------------------------------------------ + # Patterns — read + # ------------------------------------------------------------------ + + def get_pattern(self, pattern_id: str) -> dict | None: + """Return the pattern document as a dict, or None if it doesn't exist. + + Args: + pattern_id: Firestore document ID in the ``error_patterns`` collection. + """ + snap = self._db.collection(_COL_PATTERNS).document(pattern_id).get() + if not snap.exists: + return None + return snap.to_dict() + + def get_active_patterns(self) -> list[dict]: + """Return all patterns with status in [new, acknowledged, known].""" + query = self._db.collection(_COL_PATTERNS).where( + "status", "in", _ACTIVE_STATUSES + ) + return [snap.to_dict() for snap in query.stream()] + + def get_patterns_for_auto_resolve(self) -> list[dict]: + """Return active patterns eligible for auto-resolution (new or acknowledged).""" + query = self._db.collection(_COL_PATTERNS).where( + "status", "in", list(_AUTO_RESOLVE_ELIGIBLE) + ) + return [snap.to_dict() for snap in query.stream()] + + # ------------------------------------------------------------------ + # Patterns — upsert + # ------------------------------------------------------------------ + + def upsert_pattern(self, data: PatternData) -> UpsertResult: + """Create a new pattern or update an existing one. + + New pattern: + - status="new", severity="P3" + - Initialize ``rolling_counts`` with one entry for this run. + + Existing pattern: + - Increment ``total_count`` and append to ``rolling_counts``. + - If status was ``auto_resolved`` or ``fixed`` → reactivate (set status="new"). + - Prune ``rolling_counts`` entries older than ROLLING_WINDOW_DAYS. + + Returns: + UpsertResult with ``is_new=True`` for brand-new patterns *and* for + reactivated patterns (auto_resolved → new, fixed → new). + """ + doc_ref = self._db.collection(_COL_PATTERNS).document(data.pattern_id) + snap = doc_ref.get() + ts_iso = _iso(data.timestamp) + + new_rolling_entry = {"ts": ts_iso, "count": data.count} + + if not snap.exists: + # ── brand new pattern ────────────────────────────────────────── + doc_ref.set( + { + "pattern_id": data.pattern_id, + "service": data.service, + "resource_type": data.resource_type, + "normalized_message": data.normalized_message, + "sample_message": data.sample_message, + "status": "new", + "severity": "P3", + "total_count": data.count, + "rolling_counts": [new_rolling_entry], + "first_seen": ts_iso, + "last_seen": ts_iso, + "alerted_at": None, + "created_at": ts_iso, + "updated_at": ts_iso, + } + ) + return UpsertResult(pattern_id=data.pattern_id, is_new=True) + + # ── existing pattern ─────────────────────────────────────────────── + existing = snap.to_dict() + prev_status = existing.get("status", "new") + + rolling_counts: list[dict] = list(existing.get("rolling_counts") or []) + rolling_counts.append(new_rolling_entry) + rolling_counts = self._prune_rolling_counts(rolling_counts) + + update: dict[str, Any] = { + "total_count": existing.get("total_count", 0) + data.count, + "rolling_counts": rolling_counts, + "last_seen": ts_iso, + "sample_message": data.sample_message, + "updated_at": ts_iso, + } + + # Reactivation: pattern was considered resolved/fixed but error recurred + is_new = False + if prev_status in _REACTIVATION_STATUSES: + update["status"] = "new" + update["alerted_at"] = None + is_new = True + + doc_ref.update(update) + return UpsertResult( + pattern_id=data.pattern_id, + is_new=is_new, + previous_status=prev_status, + ) + + # ------------------------------------------------------------------ + # Patterns — status changes + # ------------------------------------------------------------------ + + def auto_resolve_pattern(self, pattern_id: str, hours_silent: float) -> None: + """Mark a pattern as auto-resolved. + + Args: + pattern_id: Firestore document ID. + hours_silent: How many hours the pattern has been silent. + """ + self._db.collection(_COL_PATTERNS).document(pattern_id).update( + { + "status": "auto_resolved", + "auto_resolve_hours_silent": hours_silent, + "auto_resolved_at": _iso(_utcnow()), + "updated_at": _iso(_utcnow()), + } + ) + + def resolve_pattern( + self, + pattern_id: str, + pr_url: str | None = None, + note: str = "", + ) -> None: + """Mark a pattern as fixed (human-resolved). + + Args: + pattern_id: Firestore document ID. + pr_url: Optional URL of the PR that fixed this pattern. + note: Optional human-readable note about the fix. + """ + now_iso = _iso(_utcnow()) + self._db.collection(_COL_PATTERNS).document(pattern_id).update( + { + "status": "fixed", + "fixed_by": { + "pr_url": pr_url, + "note": note, + "fixed_at": now_iso, + }, + "updated_at": now_iso, + } + ) + + def update_pattern_alerted(self, pattern_id: str) -> None: + """Set the ``alerted_at`` timestamp on a pattern. + + Called after a Discord alert has been successfully sent for the pattern. + """ + self._db.collection(_COL_PATTERNS).document(pattern_id).update( + { + "alerted_at": _iso(_utcnow()), + "updated_at": _iso(_utcnow()), + } + ) + + def merge_pattern(self, source_id: str, target_id: str, reason: str) -> None: + """Merge *source* into *target* by muting the source pattern. + + Args: + source_id: Pattern to mute. + target_id: Pattern that the source is being merged into. + reason: Human-readable rationale for the merge. + """ + self._db.collection(_COL_PATTERNS).document(source_id).update( + { + "status": "muted", + "merged_into": target_id, + "merge_reason": reason, + "updated_at": _iso(_utcnow()), + } + ) + + # ------------------------------------------------------------------ + # Auto-resolve logic + # ------------------------------------------------------------------ + + def check_auto_resolve(self, pattern: dict) -> float | None: + """Determine whether *pattern* should be auto-resolved. + + Computes the frequency-aware silence threshold and compares it against + how long the pattern has been silent. + + Args: + pattern: A pattern document dict (as returned by get_pattern). + + Returns: + The number of hours the pattern has been silent if the threshold is + exceeded, or ``None`` if the pattern should not yet be resolved. + """ + status = pattern.get("status", "") + if status not in _AUTO_RESOLVE_ELIGIBLE: + return None + + last_seen_str = pattern.get("last_seen") + if not last_seen_str: + return None + + last_seen = datetime.fromisoformat(last_seen_str) + now = datetime.now(tz=timezone.utc) + + # Ensure both datetimes are tz-aware for subtraction + if last_seen.tzinfo is None: + last_seen = last_seen.replace(tzinfo=timezone.utc) + + hours_silent = (now - last_seen).total_seconds() / 3600.0 + + rolling_counts = pattern.get("rolling_counts") or [] + threshold = self._compute_resolve_threshold(rolling_counts) + + if hours_silent >= threshold: + return hours_silent + return None + + def _compute_resolve_threshold(self, rolling_counts: list[dict]) -> float: + """Compute the frequency-aware auto-resolve threshold in hours. + + Algorithm: + 1. If fewer than 3 data points → return AUTO_RESOLVE_FALLBACK_HOURS (48). + 2. Compute mean interval between consecutive timestamps. + 3. threshold = mean_interval_hours * AUTO_RESOLVE_MULTIPLIER. + 4. Clamp result to [AUTO_RESOLVE_MIN_HOURS, AUTO_RESOLVE_MAX_HOURS]. + + Args: + rolling_counts: List of ``{"ts": ISO-string, "count": int}`` dicts. + + Returns: + Threshold in hours (float). + """ + if len(rolling_counts) < 3: + return float(AUTO_RESOLVE_FALLBACK_HOURS) + + # Parse timestamps and sort ascending + timestamps: list[datetime] = [] + for entry in rolling_counts: + ts = datetime.fromisoformat(entry["ts"]) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + timestamps.append(ts) + timestamps.sort() + + # Compute pairwise intervals + intervals_hours: list[float] = [] + for i in range(1, len(timestamps)): + delta = (timestamps[i] - timestamps[i - 1]).total_seconds() / 3600.0 + intervals_hours.append(delta) + + if not intervals_hours: + return float(AUTO_RESOLVE_FALLBACK_HOURS) + + mean_interval = sum(intervals_hours) / len(intervals_hours) + threshold = mean_interval * AUTO_RESOLVE_MULTIPLIER + + # Clamp to configured bounds + threshold = max(float(AUTO_RESOLVE_MIN_HOURS), threshold) + threshold = min(float(AUTO_RESOLVE_MAX_HOURS), threshold) + return threshold + + def _prune_rolling_counts(self, rolling_counts: list[dict]) -> list[dict]: + """Remove rolling_count entries older than ROLLING_WINDOW_DAYS. + + Args: + rolling_counts: List of ``{"ts": ISO-string, "count": int}`` dicts. + + Returns: + Filtered list with only recent entries. + """ + if not rolling_counts: + return [] + + cutoff = _utcnow() - timedelta(days=ROLLING_WINDOW_DAYS) + result = [] + for entry in rolling_counts: + ts = datetime.fromisoformat(entry["ts"]) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + if ts >= cutoff: + result.append(entry) + return result + + # ------------------------------------------------------------------ + # Incidents + # ------------------------------------------------------------------ + + def create_incident( + self, + title: str, + root_cause: str, + severity: str, + suggested_fix: str, + primary_service: str, + pattern_ids: list[str], + used_llm: bool, + ) -> str: + """Create a new incident document and link associated patterns. + + Args: + title: Short human-readable title. + root_cause: LLM-generated or manual root-cause description. + severity: Severity string (e.g. "P1", "P2", "P3"). + suggested_fix: Suggested remediation steps. + primary_service: The primary affected service. + pattern_ids: List of pattern_ids that belong to this incident. + used_llm: Whether LLM analysis was used to generate this incident. + + Returns: + The generated incident_id string. + """ + incident_id = _generate_id("inc", "%Y%m%d_%H%M") + now_iso = _iso(_utcnow()) + + self._db.collection(_COL_INCIDENTS).document(incident_id).set( + { + "incident_id": incident_id, + "title": title, + "root_cause": root_cause, + "severity": severity, + "suggested_fix": suggested_fix, + "primary_service": primary_service, + "pattern_ids": pattern_ids, + "used_llm": used_llm, + "created_at": now_iso, + "updated_at": now_iso, + "status": "open", + } + ) + + # Link each pattern back to this incident + for pattern_id in pattern_ids: + self._db.collection(_COL_PATTERNS).document(pattern_id).update( + { + "incident_id": incident_id, + "updated_at": now_iso, + } + ) + + return incident_id + + # ------------------------------------------------------------------ + # Discord audit trail + # ------------------------------------------------------------------ + + def log_discord_alert( + self, + alert_type: str, + content: str, + success: bool, + metadata: dict | None = None, + ) -> None: + """Write an audit record to the ``discord_alerts`` collection. + + Args: + alert_type: Category of alert (e.g. "new_pattern", "digest"). + content: The message body that was (or would have been) sent. + success: Whether the Discord webhook call succeeded. + metadata: Optional additional data (pattern_ids, incident_id, etc.). + """ + alert_id = _generate_id("alert", "%Y%m%d_%H%M%S") + now_iso = _iso(_utcnow()) + + self._db.collection(_COL_ALERTS).document(alert_id).set( + { + "alert_id": alert_id, + "alert_type": alert_type, + "content": content, + "success": success, + "metadata": metadata or {}, + "created_at": now_iso, + } + ) diff --git a/backend/services/error_monitor/known_issues.py b/backend/services/error_monitor/known_issues.py new file mode 100644 index 00000000..ded31bf6 --- /dev/null +++ b/backend/services/error_monitor/known_issues.py @@ -0,0 +1,106 @@ +"""Known infrastructure noise patterns for the production error monitoring service. + +Filters out transient, expected log messages so the error monitor is not flooded +with Cloud Run cold-start probes, deploy events, spot VM preemptions, and similar +operational noise. + +Usage: + from backend.services.error_monitor.known_issues import should_ignore + + reason = should_ignore(service, message) + if reason: + # skip this log entry — it's known infrastructure noise + ... +""" + +import re +from dataclasses import dataclass + + +@dataclass +class IgnoreReason: + """Describes why a log entry should be suppressed. + + Attributes: + pattern_name: Machine-readable name of the matching ignore pattern. + reason: Human-readable explanation of why this is infrastructure noise. + """ + + pattern_name: str + reason: str + + +# --------------------------------------------------------------------------- +# Compiled ignore patterns (applied in order) +# All patterns use re.IGNORECASE for case-insensitive matching. +# --------------------------------------------------------------------------- + +_IGNORE_PATTERNS: list[tuple[str, re.Pattern, str]] = [ + ( + "startup_probe", + re.compile(r"startup probe failed", re.IGNORECASE), + "Cloud Run cold start — startup probe failure is transient and expected", + ), + ( + "ready_condition", + re.compile(r"ready condition status changed", re.IGNORECASE), + "Cloud Run deploy event — readiness condition change is expected on deploy", + ), + ( + "spot_preemption", + re.compile(r"instance was preempted", re.IGNORECASE), + "Spot/preemptible VM preemption — expected behaviour for spot instances", + ), + ( + "idle_shutdown", + re.compile(r"stopping idle encoding worker", re.IGNORECASE), + "Encoding worker idle shutdown — expected behaviour when no jobs are queued", + ), + ( + "health_check_404", + re.compile( + r"(?:GET|HEAD)\b[^\n]*?/health[^\n]*?\b(?:404|not found)\b", + re.IGNORECASE, + ), + "Load balancer health check returning 404 — transient during cold start", + ), + ( + "scheduler_retry", + re.compile(r"cloud scheduler.*retry", re.IGNORECASE), + "Cloud Scheduler transient retry — expected on intermittent job failures", + ), + ( + "runner_startup", + re.compile(r"github\.runner\..*(?:starting|stopping|idle)", re.IGNORECASE), + "GitHub Actions runner lifecycle event — expected during runner start/stop", + ), + ( + "container_shutdown", + re.compile(r"container called exit\(0\)", re.IGNORECASE), + "Container clean exit — exit code 0 indicates a graceful, expected shutdown", + ), +] + + +def should_ignore(service: str, message: str) -> IgnoreReason | None: + """Check whether a log entry matches a known infrastructure noise pattern. + + The check is purely message-based; ``service`` is accepted for API symmetry + and future service-specific overrides but does not gate any pattern today. + + Args: + service: The originating service name (e.g. ``"karaoke-backend"``). + message: The raw log message string to evaluate. + + Returns: + An :class:`IgnoreReason` if the message matches a known noise pattern, + or ``None`` if the message should be processed normally. + """ + if not message: + return None + + for pattern_name, pattern, reason in _IGNORE_PATTERNS: + if pattern.search(message): + return IgnoreReason(pattern_name=pattern_name, reason=reason) + + return None diff --git a/backend/services/error_monitor/llm_analysis.py b/backend/services/error_monitor/llm_analysis.py new file mode 100644 index 00000000..529bf602 --- /dev/null +++ b/backend/services/error_monitor/llm_analysis.py @@ -0,0 +1,429 @@ +"""LLM-powered incident analysis and duplicate detection for the error monitor. + +Uses Gemini Flash via the google-generativeai library to: + 1. Group related error patterns into incidents with root-cause analysis. + 2. Identify near-duplicate patterns that the regex normalizer missed. + +Both functions fall back gracefully when the LLM is unavailable or returns +unparseable output. +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from collections import defaultdict +from dataclasses import dataclass, field + +from google import genai +from google.genai import types as genai_types + +from backend.services.error_monitor.config import ( + GCP_PROJECT, + LLM_ANALYSIS_MODEL, + LLM_VERTEX_LOCATION, + MIN_PATTERNS_FOR_ANALYSIS, + SERVICE_DEPENDENCY_MAP, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class Incident: + """A grouped incident produced by LLM analysis or the fallback grouper.""" + + title: str + root_cause: str | None + severity: str # P0 / P1 / P2 / P3 + suggested_fix: str | None + primary_service: str + pattern_indices: list[int] + + +@dataclass +class IncidentAnalysis: + """The result of an incident grouping pass.""" + + incidents: list[Incident] + used_llm: bool = True + + +@dataclass +class DuplicateGroup: + """A canonical pattern plus the new patterns that duplicate it.""" + + canonical_index: int + duplicate_indices: list[int] + reason: str + + +# --------------------------------------------------------------------------- +# LLM call helper +# --------------------------------------------------------------------------- + +_RETRY_ATTEMPTS = 3 +_RETRY_BASE_DELAY = 2.0 # seconds + + +def _call_llm( + system_prompt: str, + user_prompt: str, + temperature: float = 0.2, +) -> str: + """Call Gemini and return the response text. + + Retries up to _RETRY_ATTEMPTS times with exponential back-off on failure. + + Args: + system_prompt: System instruction for the model. + user_prompt: User-facing prompt content. + temperature: Sampling temperature (lower = more deterministic). + + Returns: + The model's response text. + + Raises: + Exception: Re-raises the last exception after all retries are exhausted. + """ + last_exc: Exception | None = None + + for attempt in range(_RETRY_ATTEMPTS): + try: + client = genai.Client( + vertexai=True, + project=GCP_PROJECT, + location=LLM_VERTEX_LOCATION, + ) + response = client.models.generate_content( + model=LLM_ANALYSIS_MODEL, + contents=user_prompt, + config=genai_types.GenerateContentConfig( + system_instruction=system_prompt, + temperature=temperature, + ), + ) + return response.text + except Exception as exc: # noqa: BLE001 + last_exc = exc + delay = _RETRY_BASE_DELAY * (2**attempt) + logger.warning( + "LLM call failed (attempt %d/%d): %s — retrying in %.1fs", + attempt + 1, + _RETRY_ATTEMPTS, + exc, + delay, + ) + time.sleep(delay) + + raise last_exc # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# Response parsing +# --------------------------------------------------------------------------- + +_MD_CODE_BLOCK_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) + + +def _strip_markdown_code_block(text: str) -> str: + """Remove surrounding ```json ... ``` or ``` ... ``` fences if present.""" + match = _MD_CODE_BLOCK_RE.search(text) + if match: + return match.group(1).strip() + return text.strip() + + +def _parse_llm_response(text: str) -> IncidentAnalysis | None: + """Parse the LLM response text into an IncidentAnalysis. + + Args: + text: Raw LLM output, possibly wrapped in a markdown code block. + + Returns: + IncidentAnalysis on success, None on any parse failure. + """ + if not text: + return None + + cleaned = _strip_markdown_code_block(text) + + try: + data = json.loads(cleaned) + except json.JSONDecodeError: + logger.warning("LLM returned non-JSON response (first 200 chars): %.200s", text) + return None + + if "incidents" not in data: + logger.warning("LLM response missing 'incidents' key: %s", list(data.keys())) + return None + + incidents: list[Incident] = [] + for raw in data["incidents"]: + try: + incidents.append( + Incident( + title=raw.get("title", "Unknown incident"), + root_cause=raw.get("root_cause"), + severity=raw.get("severity", "P2"), + suggested_fix=raw.get("suggested_fix"), + primary_service=raw.get("primary_service", "unknown"), + pattern_indices=raw.get("pattern_indices", []), + ) + ) + except Exception as exc: # noqa: BLE001 + logger.warning("Skipping malformed incident entry: %s — %s", raw, exc) + + return IncidentAnalysis(incidents=incidents, used_llm=True) + + +# --------------------------------------------------------------------------- +# Fallback grouper +# --------------------------------------------------------------------------- + + +def _fallback_group_by_service(patterns: list[dict]) -> IncidentAnalysis: + """Group patterns by service name when the LLM is unavailable. + + Each unique service becomes one P2 incident containing all of that + service's pattern indices. + + Args: + patterns: List of error pattern dicts (must include 'service' key). + + Returns: + IncidentAnalysis with used_llm=False. + """ + service_to_indices: dict[str, list[int]] = defaultdict(list) + for idx, pattern in enumerate(patterns): + service = pattern.get("service", "unknown") + service_to_indices[service].append(idx) + + incidents = [ + Incident( + title=f"Errors in {service}", + root_cause=None, + severity="P2", + suggested_fix=None, + primary_service=service, + pattern_indices=indices, + ) + for service, indices in service_to_indices.items() + ] + + return IncidentAnalysis(incidents=incidents, used_llm=False) + + +# --------------------------------------------------------------------------- +# Main analysis functions +# --------------------------------------------------------------------------- + +_ANALYZE_SYSTEM_PROMPT = f"""\ +You are an expert site-reliability engineer for Nomad Karaoke, a karaoke video +generation platform running on Google Cloud Platform. + +Your task is to analyse a list of active error patterns detected across +production services, group them into meaningful incidents, and identify root +causes. + +## Service dependency map + +{SERVICE_DEPENDENCY_MAP} + +## Severity definitions + +- P0: Service is completely down / total outage +- P1: Major pipeline broken — orders cannot complete +- P2: Degraded performance — some orders affected +- P3: Minor / cosmetic issue with no direct user impact + +## Output format + +Respond with a single JSON object (no markdown fencing) with this schema: + +{{ + "incidents": [ + {{ + "title": "", + "root_cause": "", + "severity": "P0|P1|P2|P3", + "suggested_fix": "", + "primary_service": "", + "pattern_indices": [] + }} + ] +}} + +Group related patterns together into a single incident where they share a +common root cause (e.g., a downstream dependency failure). Unrelated patterns +should be in separate incidents. Every input pattern must appear in exactly +one incident. +""" + +_DEDUP_SYSTEM_PROMPT = """\ +You are an expert at identifying near-duplicate error patterns in production +monitoring systems. + +Your task is to compare a list of NEW error patterns against a list of +EXISTING (canonical) patterns and identify which new patterns are essentially +duplicates of existing ones — i.e., they describe the same underlying error +but with slightly different wording due to variation in log messages. + +## Output format + +Respond with a single JSON object (no markdown fencing) with this schema: + +{ + "duplicates": [ + { + "canonical_index": <0-based index into the EXISTING patterns list>, + "duplicate_indices": [<0-based indices into the NEW patterns list>], + "reason": "" + } + ] +} + +If no duplicates are found, return {"duplicates": []}. +""" + + +def analyze_patterns(patterns: list[dict]) -> IncidentAnalysis | None: + """Group error patterns into incidents using Gemini LLM analysis. + + Returns None if there are fewer than MIN_PATTERNS_FOR_ANALYSIS patterns. + Falls back to service-based grouping when the LLM fails or returns an + unparseable response. + + Args: + patterns: List of error pattern dicts to analyse. + + Returns: + IncidentAnalysis, or None if insufficient patterns. + """ + if len(patterns) < MIN_PATTERNS_FOR_ANALYSIS: + return None + + user_prompt = _build_analyze_user_prompt(patterns) + + try: + raw_response = _call_llm(_ANALYZE_SYSTEM_PROMPT, user_prompt, temperature=0.2) + result = _parse_llm_response(raw_response) + if result is not None: + return result + + logger.warning("LLM returned unparseable response; falling back to service grouping") + except Exception as exc: # noqa: BLE001 + logger.warning("LLM analysis failed: %s — falling back to service grouping", exc) + + return _fallback_group_by_service(patterns) + + +def find_duplicate_patterns( + new_patterns: list[dict], + existing_patterns: list[dict], +) -> list[DuplicateGroup]: + """Identify near-duplicate patterns that the regex normalizer missed. + + Returns an empty list if either input list is empty, or if the LLM call + fails for any reason. + + Args: + new_patterns: Newly detected error patterns to check. + existing_patterns: Currently tracked (canonical) error patterns. + + Returns: + List of DuplicateGroup objects (may be empty). + """ + if not new_patterns or not existing_patterns: + return [] + + user_prompt = _build_dedup_user_prompt(new_patterns, existing_patterns) + + try: + raw_response = _call_llm(_DEDUP_SYSTEM_PROMPT, user_prompt, temperature=0.1) + return _parse_duplicate_response(raw_response) + except Exception as exc: # noqa: BLE001 + logger.warning("Duplicate detection LLM call failed: %s", exc) + return [] + + +# --------------------------------------------------------------------------- +# Prompt builders +# --------------------------------------------------------------------------- + + +def _build_analyze_user_prompt(patterns: list[dict]) -> str: + """Build the user prompt listing patterns for incident grouping.""" + lines = ["Error patterns to analyse:\n"] + for idx, pattern in enumerate(patterns): + service = pattern.get("service", "unknown") + message = pattern.get("normalized_message", "") + count = pattern.get("count", 0) + lines.append(f"[{idx}] service={service!r} count={count} message={message!r}") + return "\n".join(lines) + + +def _build_dedup_user_prompt( + new_patterns: list[dict], + existing_patterns: list[dict], +) -> str: + """Build the user prompt listing existing and new patterns for dedup.""" + lines = ["EXISTING (canonical) patterns:\n"] + for idx, pattern in enumerate(existing_patterns): + service = pattern.get("service", "unknown") + message = pattern.get("normalized_message", "") + lines.append(f"[{idx}] service={service!r} message={message!r}") + + lines.append("\nNEW patterns to check:\n") + for idx, pattern in enumerate(new_patterns): + service = pattern.get("service", "unknown") + message = pattern.get("normalized_message", "") + lines.append(f"[{idx}] service={service!r} message={message!r}") + + return "\n".join(lines) + + +def _parse_duplicate_response(text: str) -> list[DuplicateGroup]: + """Parse the LLM response for duplicate detection. + + Args: + text: Raw LLM output. + + Returns: + List of DuplicateGroup objects, or [] on any failure. + """ + if not text: + return [] + + cleaned = _strip_markdown_code_block(text) + + try: + data = json.loads(cleaned) + except json.JSONDecodeError: + logger.warning("Dedup LLM returned non-JSON: %.200s", text) + return [] + + if "duplicates" not in data: + logger.warning("Dedup LLM response missing 'duplicates' key") + return [] + + groups: list[DuplicateGroup] = [] + for raw in data["duplicates"]: + try: + groups.append( + DuplicateGroup( + canonical_index=int(raw["canonical_index"]), + duplicate_indices=list(raw.get("duplicate_indices", [])), + reason=raw.get("reason", ""), + ) + ) + except Exception as exc: # noqa: BLE001 + logger.warning("Skipping malformed duplicate entry: %s — %s", raw, exc) + + return groups diff --git a/backend/services/error_monitor/monitor.py b/backend/services/error_monitor/monitor.py new file mode 100644 index 00000000..4bd2d82b --- /dev/null +++ b/backend/services/error_monitor/monitor.py @@ -0,0 +1,641 @@ +"""Monitor orchestrator for the production error monitoring service. + +This is the main entry point. Run as:: + + python -m backend.services.error_monitor.monitor + +The ``ErrorMonitor.run()`` method executes the full pipeline: + +* Monitor cycle (default) — query Cloud Logging, detect new patterns, send + Discord alerts, check auto-resolution. +* Daily digest mode (``DIGEST_MODE=true``) — bundle 24h stats into one message. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any + +from google.cloud import logging as cloud_logging +from google.cloud import secretmanager + +from backend.services.error_monitor.config import ( + GCP_PROJECT, + LOOKBACK_MINUTES, + MAX_LOG_ENTRIES, + MONITORED_CLOUD_FUNCTIONS, + MONITORED_CLOUD_RUN_JOBS, + MONITORED_CLOUD_RUN_SERVICES, + MONITORED_GCE_INSTANCES, + SPIKE_MIN_COUNT, + SPIKE_MULTIPLIER, + get_digest_mode, + get_discord_webhook_secret_name, + get_llm_enabled, +) +from backend.services.error_monitor.discord import ( + ErrorMonitorDiscord, + format_auto_resolved_alert, + format_daily_digest, + format_incident_alert, + format_new_pattern_alert, + format_spike_alert, +) +from backend.services.error_monitor.firestore_adapter import ( + ErrorPatternsAdapter, + PatternData, + UpsertResult, +) +from backend.services.error_monitor.known_issues import should_ignore +from backend.services.error_monitor.llm_analysis import analyze_patterns, find_duplicate_patterns +from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Module-level constants +# --------------------------------------------------------------------------- + +#: Maps log resource type → (label_key, service_name_list) +RESOURCE_TYPE_FILTERS: dict[str, tuple[str, list[str]]] = { + "cloud_run_revision": ("resource.labels.service_name", MONITORED_CLOUD_RUN_SERVICES), + "cloud_run_job": ("resource.labels.job_name", MONITORED_CLOUD_RUN_JOBS), + "cloud_function": ("resource.labels.function_name", MONITORED_CLOUD_FUNCTIONS), + "gce_instance": ('labels."compute.googleapis.com/resource_name"', MONITORED_GCE_INSTANCES), +} + +#: Maps log resource type → canonical resource_type string stored in Firestore. +RESOURCE_TYPE_MAP: dict[str, str] = { + "cloud_run_revision": "cloud_run_service", + "cloud_run_job": "cloud_run_job", + "cloud_function": "cloud_function", + "gce_instance": "gce_instance", +} + +# --------------------------------------------------------------------------- +# Free functions +# --------------------------------------------------------------------------- + + +def _build_log_filter( + resource_type: str, + resource_names: list[str], + lookback_minutes: int, +) -> str: + """Build a Cloud Logging filter string. + + Args: + resource_type: GCP resource type (e.g. ``"cloud_run_revision"``). + resource_names: List of service/job/function/instance names to include. + lookback_minutes: How many minutes back to query. + + Returns: + A filter string suitable for ``logging_client.list_entries(filter_=...)``. + """ + label_key = RESOURCE_TYPE_FILTERS[resource_type][0] + cutoff = datetime.now(tz=timezone.utc) - timedelta(minutes=lookback_minutes) + cutoff_iso = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ") + + name_clause = " OR ".join(f'{label_key}="{name}"' for name in resource_names) + if len(resource_names) > 1: + name_clause = f"({name_clause})" + + return ( + f'resource.type="{resource_type}" ' + f"severity>=ERROR " + f'timestamp>="{cutoff_iso}" ' + f"{name_clause}" + ) + + +def _extract_message(entry: Any) -> str | None: + """Extract the log message text from a Cloud Logging entry. + + Handles several payload shapes: + + * ``str`` payload — returned directly. + * ``dict`` payload — tries ``"message"``, ``"textPayload"``, ``"error"``, + ``"msg"`` keys in that order. + * ``None`` payload — returns ``None``. + + Args: + entry: A Cloud Logging ``LogEntry`` object. + + Returns: + The message string, or ``None`` if not extractable. + """ + payload = entry.payload + if payload is None: + return None + if isinstance(payload, str): + return payload if payload else None + if isinstance(payload, dict): + for key in ("message", "textPayload", "error", "msg"): + value = payload.get(key) + if value: + return str(value) + return None + # Fallback: convert to string representation + text = str(payload) + return text if text else None + + +def _extract_service_name(entry: Any, resource_type: str) -> str: + """Extract the service/job/function/instance name from a log entry. + + Args: + entry: A Cloud Logging ``LogEntry`` object. + resource_type: GCP log resource type string. + + Returns: + The service name string (falls back to ``"unknown"`` if not found). + """ + labels = getattr(entry.resource, "labels", {}) or {} + entry_labels = getattr(entry, "labels", {}) or {} + + if resource_type == "cloud_run_revision": + return labels.get("service_name") or "unknown" + if resource_type == "cloud_run_job": + return labels.get("job_name") or "unknown" + if resource_type == "cloud_function": + return labels.get("function_name") or "unknown" + if resource_type == "gce_instance": + # Prefer the human-readable resource name label + name = entry_labels.get("compute.googleapis.com/resource_name") + if name: + return name + return labels.get("instance_id") or "unknown" + return "unknown" + + +def _classify_resource_type(log_resource_type: str) -> str: + """Map a Cloud Logging resource type to the canonical type stored in Firestore. + + Args: + log_resource_type: Raw GCP resource type (e.g. ``"cloud_run_revision"``). + + Returns: + Canonical resource type string (falls back to the input if not in map). + """ + return RESOURCE_TYPE_MAP.get(log_resource_type, log_resource_type) + + +def _group_by_pattern(entries: list[dict]) -> dict[str, dict]: + """Group normalised log entries by their pattern hash. + + Args: + entries: List of dicts with ``service``, ``normalized``, and ``message`` keys. + + Returns: + Dict mapping ``pattern_id`` → group dict containing ``service``, + ``resource_type``, ``normalized``, ``sample_message``, ``count``. + """ + groups: dict[str, dict] = {} + for entry in entries: + service = entry["service"] + normalized = entry["normalized"] + pattern_id = compute_pattern_hash(service, normalized) + + if pattern_id not in groups: + groups[pattern_id] = { + "pattern_id": pattern_id, + "service": service, + "resource_type": entry.get("resource_type", "unknown"), + "normalized": normalized, + "sample_message": entry["message"], + "count": 0, + } + groups[pattern_id]["count"] += 1 + return groups + + +def _rolling_average(rolling_counts: list[dict]) -> float: + """Compute the mean count from a list of rolling_count dicts. + + Args: + rolling_counts: List of ``{"count": int, ...}`` dicts. + + Returns: + The arithmetic mean as a float, or ``0.0`` for an empty list. + """ + if not rolling_counts: + return 0.0 + total = sum(entry.get("count", 0) for entry in rolling_counts) + return total / len(rolling_counts) + + +def _is_spike(current_count: int, rolling_counts: list[dict]) -> bool: + """Determine whether ``current_count`` represents an error rate spike. + + A spike is detected when: + - ``current_count`` >= ``SPIKE_MIN_COUNT`` (avoids noise from rare errors), AND + - ``current_count`` > ``rolling_average * SPIKE_MULTIPLIER`` + + Args: + current_count: Error count in the current monitoring window. + rolling_counts: Historical rolling counts from Firestore. + + Returns: + ``True`` if a spike is detected, ``False`` otherwise. + """ + if not rolling_counts: + return False + if current_count < SPIKE_MIN_COUNT: + return False + avg = _rolling_average(rolling_counts) + if avg == 0.0: + return False + return current_count > avg * SPIKE_MULTIPLIER + + +# --------------------------------------------------------------------------- +# ErrorMonitor class +# --------------------------------------------------------------------------- + + +class ErrorMonitor: + """Orchestrates the full production error monitoring pipeline. + + On each run (typically every 15 minutes via Cloud Scheduler), it: + + 1. Queries Cloud Logging for recent errors across all monitored services. + 2. Groups errors into patterns using normalisation + hashing. + 3. Upserts patterns to Firestore, tracking new vs. recurrent. + 4. Optionally uses an LLM to deduplicate patterns and create incidents. + 5. Sends Discord alerts for new patterns, incidents, and spikes. + 6. Auto-resolves patterns that have been silent long enough. + """ + + def __init__(self) -> None: + self.logging_client = cloud_logging.Client(project=GCP_PROJECT) + self.firestore_adapter = ErrorPatternsAdapter() + self.discord = ErrorMonitorDiscord(webhook_url=self._get_discord_webhook()) + + def _get_discord_webhook(self) -> str: + """Retrieve the Discord webhook URL. + + Checks ``DISCORD_WEBHOOK_URL`` environment variable first; falls back + to Secret Manager. + + Returns: + Discord webhook URL string. + """ + url = os.environ.get("DISCORD_WEBHOOK_URL") + if url: + return url + + secret_name = get_discord_webhook_secret_name() + client = secretmanager.SecretManagerServiceClient() + secret_path = f"projects/{GCP_PROJECT}/secrets/{secret_name}/versions/latest" + response = client.access_secret_version(name=secret_path) + return response.payload.data.decode("utf-8").strip() + + # ------------------------------------------------------------------ + # Main entry point + # ------------------------------------------------------------------ + + def run(self) -> None: + """Execute the monitor pipeline (digest or cycle mode).""" + if get_digest_mode(): + self._run_daily_digest() + else: + self._run_monitor_cycle() + + # ------------------------------------------------------------------ + # Monitor cycle + # ------------------------------------------------------------------ + + def _run_monitor_cycle(self) -> None: + """Execute one monitor cycle: collect → group → upsert → alert → auto-resolve.""" + now = datetime.now(tz=timezone.utc) + now_iso = now.isoformat() + + # ── Step 1: Collect log entries ──────────────────────────────────── + raw_entries: list[dict] = [] + for resource_type, (_, resource_names) in RESOURCE_TYPE_FILTERS.items(): + log_filter = _build_log_filter(resource_type, resource_names, LOOKBACK_MINUTES) + logger.info("Querying %s with filter: %s", resource_type, log_filter) + try: + entries = self.logging_client.list_entries( + filter_=log_filter, + max_results=MAX_LOG_ENTRIES, + ) + for entry in entries: + message = _extract_message(entry) + if not message: + continue + service = _extract_service_name(entry, resource_type) + ignore_reason = should_ignore(service, message) + if ignore_reason: + logger.debug( + "Ignoring entry from %s (%s): %s", + service, + ignore_reason.pattern_name, + ignore_reason.reason, + ) + continue + normalized = normalize_message(message) + if not normalized: + continue + raw_entries.append( + { + "service": service, + "resource_type": _classify_resource_type(resource_type), + "normalized": normalized, + "message": message, + } + ) + except Exception as exc: # noqa: BLE001 + logger.error("Failed to query %s logs: %s", resource_type, exc) + + logger.info("Collected %d log entries after filtering", len(raw_entries)) + + # ── Step 2: Group by pattern ─────────────────────────────────────── + groups = _group_by_pattern(raw_entries) + logger.info("Grouped into %d distinct patterns", len(groups)) + + if not groups: + logger.info("No new error entries found this cycle; checking auto-resolve.") + self._check_auto_resolve() + return + + # ── Step 3: Upsert to Firestore ──────────────────────────────────── + new_pattern_ids: list[str] = [] + spike_patterns: list[dict] = [] + + for pattern_id, group in groups.items(): + data = PatternData( + pattern_id=pattern_id, + service=group["service"], + resource_type=group["resource_type"], + normalized_message=group["normalized"], + sample_message=group["sample_message"], + count=group["count"], + timestamp=now, + ) + result: UpsertResult = self.firestore_adapter.upsert_pattern(data) + if result.is_new: + new_pattern_ids.append(pattern_id) + logger.info("New pattern detected: %s (%s)", pattern_id, group["service"]) + + # Check for spike on existing patterns + existing = self.firestore_adapter.get_pattern(pattern_id) + if existing and not result.is_new: + rolling = existing.get("rolling_counts") or [] + if _is_spike(group["count"], rolling): + spike_patterns.append( + { + "service": group["service"], + "normalized_message": group["normalized"], + "current_count": group["count"], + "rolling_average": _rolling_average(rolling), + } + ) + logger.info("Spike detected for pattern %s", pattern_id) + + # ── Step 4: LLM dedup ───────────────────────────────────────────── + if new_pattern_ids and get_llm_enabled(): + new_patterns_data = [ + { + "service": groups[pid]["service"], + "normalized_message": groups[pid]["normalized"], + "count": groups[pid]["count"], + } + for pid in new_pattern_ids + ] + existing_patterns = self.firestore_adapter.get_active_patterns() + # Exclude newly added patterns from the canonical list + canonical = [p for p in existing_patterns if p.get("pattern_id") not in new_pattern_ids] + + if canonical: + duplicate_groups = find_duplicate_patterns(new_patterns_data, canonical) + for dup_group in duplicate_groups: + # Merge each duplicate new pattern into the canonical + canonical_pattern = canonical[dup_group.canonical_index] + canonical_id = canonical_pattern.get("pattern_id", "") + for dup_idx in dup_group.duplicate_indices: + if dup_idx < len(new_pattern_ids): + dup_id = new_pattern_ids[dup_idx] + logger.info( + "Merging duplicate pattern %s into %s: %s", + dup_id, + canonical_id, + dup_group.reason, + ) + self.firestore_adapter.merge_pattern(dup_id, canonical_id, dup_group.reason) + # Remove from new_pattern_ids since it was merged + new_pattern_ids = [pid for pid in new_pattern_ids if pid != dup_id] + + # ── Step 5: LLM incident analysis ───────────────────────────────── + incident_pattern_ids: set[str] = set() + + if len(new_pattern_ids) >= 2 and get_llm_enabled(): + patterns_for_analysis = [ + { + "service": groups[pid]["service"], + "normalized_message": groups[pid]["normalized"], + "count": groups[pid]["count"], + } + for pid in new_pattern_ids + if pid in groups + ] + analysis = analyze_patterns(patterns_for_analysis) + + if analysis and analysis.incidents: + for incident in analysis.incidents: + incident_pids = [ + new_pattern_ids[idx] + for idx in incident.pattern_indices + if idx < len(new_pattern_ids) + ] + if not incident_pids: + continue + + incident_id = self.firestore_adapter.create_incident( + title=incident.title, + root_cause=incident.root_cause or "", + severity=incident.severity, + suggested_fix=incident.suggested_fix or "", + primary_service=incident.primary_service, + pattern_ids=incident_pids, + used_llm=analysis.used_llm, + ) + logger.info( + "Created incident %s (%s) with %d patterns", + incident_id, + incident.severity, + len(incident_pids), + ) + + # Build pattern list for the Discord alert + pattern_dicts = [ + { + "service": groups[pid]["service"], + "normalized_message": groups[pid]["normalized"], + "count": groups[pid]["count"], + } + for pid in incident_pids + if pid in groups + ] + msg = format_incident_alert( + title=incident.title, + severity=incident.severity, + root_cause=incident.root_cause or "Unknown", + suggested_fix=incident.suggested_fix or "Investigate logs", + primary_service=incident.primary_service, + patterns=pattern_dicts, + ) + if self.discord.send_message(msg): + for pid in incident_pids: + try: + self.firestore_adapter.update_pattern_alerted(pid) + except Exception as exc: # noqa: BLE001 + logger.warning("Could not update alerted_at for %s: %s", pid, exc) + + incident_pattern_ids.update(incident_pids) + + # ── Step 6: Individual alerts for new patterns not in incidents ──── + for pattern_id in new_pattern_ids: + if pattern_id in incident_pattern_ids: + continue + if pattern_id not in groups: + continue + group = groups[pattern_id] + + # Fetch fresh copy to get first_seen timestamp + existing = self.firestore_adapter.get_pattern(pattern_id) + first_seen = existing.get("first_seen", now_iso) if existing else now_iso + + msg = format_new_pattern_alert( + service=group["service"], + resource_type=group["resource_type"], + normalized_message=group["normalized"], + sample_message=group["sample_message"], + count=group["count"], + first_seen=first_seen, + ) + if self.discord.send_message(msg): + try: + self.firestore_adapter.update_pattern_alerted(pattern_id) + except Exception as exc: # noqa: BLE001 + logger.warning("Could not update alerted_at for %s: %s", pattern_id, exc) + + # ── Step 7: Spike alerts ─────────────────────────────────────────── + for spike in spike_patterns: + msg = format_spike_alert( + service=spike["service"], + normalized_message=spike["normalized_message"], + current_count=spike["current_count"], + rolling_average=spike["rolling_average"], + ) + self.discord.send_message(msg) + + # ── Step 8: Auto-resolve check ───────────────────────────────────── + self._check_auto_resolve() + + # ------------------------------------------------------------------ + # Auto-resolve helper + # ------------------------------------------------------------------ + + def _check_auto_resolve(self) -> None: + """Check all active patterns for auto-resolution and send a consolidated alert.""" + patterns = self.firestore_adapter.get_patterns_for_auto_resolve() + resolved: list[dict] = [] + + for pattern in patterns: + pattern_id = pattern.get("pattern_id", "") + hours_silent = self.firestore_adapter.check_auto_resolve(pattern) + if hours_silent is not None: + logger.info( + "Auto-resolving pattern %s (silent %.1f hours)", pattern_id, hours_silent + ) + self.firestore_adapter.auto_resolve_pattern(pattern_id, round(hours_silent, 1)) + resolved.append( + { + "service": pattern.get("service", "unknown"), + "message": pattern.get("normalized_message", ""), + "hours_silent": round(hours_silent, 1), + } + ) + + if resolved: + msg = format_auto_resolved_alert(resolved) + self.discord.send_message(msg) + + # ------------------------------------------------------------------ + # Daily digest + # ------------------------------------------------------------------ + + def _run_daily_digest(self) -> None: + """Gather 24h stats and send a daily digest to Discord.""" + # Optional dedup sweep (best-effort; LLM errors are non-fatal) + if get_llm_enabled(): + try: + active = self.firestore_adapter.get_active_patterns() + if len(active) >= 2: + mid = len(active) // 2 + duplicates = find_duplicate_patterns(active[:mid], active[mid:]) + for dup in duplicates: + for idx in dup.duplicate_indices: + target_half = active[mid:] + source_half = active[:mid] + if idx < len(target_half) and dup.canonical_index < len(source_half): + self.firestore_adapter.merge_pattern( + target_half[idx]["pattern_id"], + source_half[dup.canonical_index]["pattern_id"], + dup.reason, + ) + except Exception as exc: # noqa: BLE001 + logger.warning("Digest dedup sweep failed (non-fatal): %s", exc) + + # Gather stats from active patterns + active_patterns = self.firestore_adapter.get_active_patterns() + per_service_counts: dict[str, int] = {} + new_count = 0 + total_errors = 0 + + for pattern in active_patterns: + service = pattern.get("service", "unknown") + count = pattern.get("total_count", 0) + per_service_counts[service] = per_service_counts.get(service, 0) + count + total_errors += count + if pattern.get("status") == "new": + new_count += 1 + + # Count recently auto-resolved patterns + resolved_patterns = self.firestore_adapter.get_patterns_for_auto_resolve() + resolved_count = sum( + 1 + for p in resolved_patterns + if self.firestore_adapter.check_auto_resolve(p) is not None + ) + + msg = format_daily_digest( + total_errors=total_errors, + new_patterns=new_count, + resolved_patterns=resolved_count, + active_patterns=len(active_patterns), + per_service_counts=per_service_counts, + ) + self.discord.send_message(msg) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + """Configure logging and run the error monitor.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + monitor = ErrorMonitor() + monitor.run() + + +if __name__ == "__main__": + main() diff --git a/backend/services/error_monitor/normalizer.py b/backend/services/error_monitor/normalizer.py new file mode 100644 index 00000000..1c54b63e --- /dev/null +++ b/backend/services/error_monitor/normalizer.py @@ -0,0 +1,137 @@ +"""Message normalizer for the production error monitoring service. + +Replaces variable parts of error messages (IDs, timestamps, paths, etc.) +with stable placeholders so the same logical error always produces the same +pattern hash, regardless of which specific resource or timestamp was involved. + +Normalizer rules are applied in order — more specific patterns first. +""" + +import hashlib +import re + +from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH + +# --------------------------------------------------------------------------- +# Compiled regex patterns (applied in order — more specific first) +# --------------------------------------------------------------------------- + +_NORMALIZERS: list[tuple[re.Pattern, str]] = [ + # 1. GCS paths: gs://bucket/path/to/object → gs:/// + # Must come before Firestore and URL normalization. + ( + re.compile(r"gs://([^/\s]+)/([^\s]+)"), + r"gs:///", + ), + # 1b. GCS bucket root (no object path): gs://bucket/ or gs://bucket + # Uses a lookahead so the slash/end-of-string is not consumed, and uses + # a character class that excludes '<' to avoid re-matching placeholders. + ( + re.compile(r"gs://([^/<\s]+)(?=[/\s]|$)"), + r"gs://", + ), + # 2. Firestore document paths (collection/doc/…/doc — at least 2 segments) + # Negative lookbehind for '.' prevents matching inside domain names (URLs). + # Negative lookbehind for '/' prevents matching inside already-replaced paths. + ( + re.compile( + r"(?", + ), + # 3. UUIDs (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx) + ( + re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" + ), + "", + ), + # 4. ISO 8601 timestamps (2024-01-15T10:30:45Z / …+05:30 / ….123Z) + ( + re.compile( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})" + ), + "", + ), + # 5. Epoch timestamps: 10+ digit number optionally followed by fractional seconds + ( + re.compile(r"\b\d{10,}(?:\.\d+)?\b"), + "", + ), + # 6. Email addresses + ( + re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"), + "", + ), + # 7. IP addresses (IPv4) — must come before large-number replacement + ( + re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), + "", + ), + # 8. URLs — strip to domain only (https://host/path → https://host) + # Handles http and https; preserves the scheme+domain for context. + ( + re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"), + r"https://\1", + ), + # 9. Job IDs in paths (/jobs/) + ( + re.compile(r"/jobs/[A-Za-z0-9]+"), + "/jobs/", + ), + # 10. Firebase UIDs (20–28 mixed-case alphanumeric characters) + # Firebase Auth UIDs are typically 28 chars; allow 20-28 for flexibility. + # Must contain both letters and digits (to avoid replacing plain words). + ( + re.compile(r"\b(?=[A-Za-z0-9]{20,28}\b)(?=[A-Za-z0-9]*[A-Z])(?=[A-Za-z0-9]*[a-z])(?=[A-Za-z0-9]*\d)[A-Za-z0-9]{20,28}\b"), + "", + ), + # 11. Hex/alphanumeric IDs: 8+ chars containing both letters and digits + # (e.g. git SHAs, Cloud Run revision IDs, job IDs) + # Uses a lookahead to require at least one digit AND one letter. + ( + re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"), + "", + ), + # 12. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements + ( + re.compile(r"\b\d{4,}\b"), + "", + ), +] + + +def normalize_message(message: str) -> str: + """Normalize a raw log message by replacing variable parts with placeholders. + + Applies each normalizer rule in order (more specific patterns first) and + truncates the result to MAX_NORMALIZED_MESSAGE_LENGTH characters. + + Args: + message: Raw log message string. + + Returns: + Normalized message string, at most MAX_NORMALIZED_MESSAGE_LENGTH chars. + """ + if not message: + return "" + + result = message + for pattern, replacement in _NORMALIZERS: + result = pattern.sub(replacement, result) + + return result[:MAX_NORMALIZED_MESSAGE_LENGTH] + + +def compute_pattern_hash(service: str, normalized_message: str) -> str: + """Compute a stable SHA-256 fingerprint for an error pattern. + + Args: + service: The originating service name (e.g. "karaoke-backend"). + normalized_message: The output of normalize_message(). + + Returns: + 64-character lowercase hex SHA-256 digest. + """ + payload = f"{service}:{normalized_message}" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6190d27a..460bf8a9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,6 +36,21 @@ Uses immutable deployment pattern - see infrastructure/encoding-worker/README.md. ``` +### Error Monitor + +```text +Cloud Logging (all services) ──→ Error Monitor (Cloud Run Job, */15 min) + │ + ├─ Normalize & deduplicate + ├─ LLM incident grouping (Gemini Flash) + ├─ Discord alerts (new patterns, spikes, daily digest) + ├─ Firestore: error_patterns, error_incidents + └─ Auto-resolve silent patterns + +Entry point: python -m backend.services.error_monitor.monitor +Design spec: docs/archive/2026-04-12-error-monitor-design.md +``` + ## Processing Pipeline ```text diff --git a/docs/README.md b/docs/README.md index 13cc18b7..703b2ff0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,6 +38,7 @@ | **Review reminder & auto-expiry** | Working (24h reminder, 48h cancel + refund) | | **i18n / Multilingual (frontend)** | Working (next-intl, en/es/de, 1062 strings, [locale] routing) | | **Referral system** | Working (referral links with configurable discounts, 20% cash kickback via Stripe Connect, auto-payout at $20) | +| **Error monitor** | New (Cloud Run Job every 15 min — log collection, normalization, LLM dedup/grouping, Discord alerts, auto-resolution, daily digest) | ## Known Issues diff --git a/docs/archive/2026-04-12-error-monitor-design.md b/docs/archive/2026-04-12-error-monitor-design.md new file mode 100644 index 00000000..085c7e51 --- /dev/null +++ b/docs/archive/2026-04-12-error-monitor-design.md @@ -0,0 +1,407 @@ +# Nomad Karaoke Error Monitor — Design Spec + +> **Date:** 2026-04-12 +> **Status:** Approved +> **Scope:** Error Monitor service for production error detection, normalization, dedup, alerting, and resolution tracking across all Nomad Karaoke services. +> **Reference:** Based on the Aquarius Error Bot system (`/Users/andrew/Projects/aquarius/docs/archive/error-bot-system-reference.md`) + +--- + +## 1. System Overview + +A Cloud Run Job that runs every 15 minutes, queries GCP Cloud Logging for errors across all production services, normalizes and deduplicates them, uses Gemini Flash to group related errors into incidents, sends alerts to Discord, tracks patterns in Firestore, and auto-resolves patterns that go silent. + +### End-to-End Flow + +``` +Production Errors (Cloud Logging) + | + v +Error Monitor (Cloud Run Job, every 15 min) + |-- Query Cloud Logging (ERROR severity, last 15 min) + | |-- cloud_run_revision: karaoke-backend, karaoke-decide, audio-separator + | |-- cloud_run_job: video-encoding-job, lyrics-transcription-job, + | | audio-separation-job, audio-download-job + | |-- cloud_function: gdrive-validator, runner_manager, backup_to_aws, + | | divebar_mirror, kn_data_sync, divebar_lookup, + | | encoding_worker_idle + | |-- gce_instance: encoding-worker-a, encoding-worker-b, + | flacfetch-vm, divebar-sync-vm + |-- Normalize messages (strip IDs, timestamps, emails, job IDs, GCS paths) + |-- Group by pattern hash (SHA-256 of service::normalized_message) + |-- Check Firestore for existing patterns + |-- LLM dedup: check new patterns against existing (Gemini Flash) + |-- If 2+ new patterns: LLM incident analysis (group by root cause) + |-- Send Discord alerts (per-incident or per-pattern) + |-- Auto-resolve stale patterns (frequency-aware thresholds) + | + v +Daily Digest (08:00 UTC, same job with DIGEST_MODE=true) + |-- Dedup sweep of all active patterns + |-- 24h summary: total errors, new patterns, frequency stats + |-- Per-service breakdown + |-- Send digest to Discord +``` + +### Monitored Resources + +| Category | Resources | +|----------|-----------| +| **Cloud Run Services** | karaoke-backend, karaoke-decide, audio-separator | +| **Cloud Run Jobs** | video-encoding-job, lyrics-transcription-job, audio-separation-job, audio-download-job | +| **Cloud Functions** | gdrive-validator, runner_manager, backup_to_aws, divebar_mirror, kn_data_sync, divebar_lookup, encoding_worker_idle | +| **GCE Instances** | encoding-worker-a, encoding-worker-b, flacfetch-vm, divebar-sync-vm | + +--- + +## 2. Log Collection + +Four separate Cloud Logging queries, one per resource type, each scoped to: +- Severity: `>=ERROR` +- Time window: last 15 minutes (configurable via `LOOKBACK_MINUTES`) +- Max entries: 500 total across all queries + +### Query Resource Types + +| Resource Type | Filter Field | Values | +|---------------|-------------|--------| +| `cloud_run_revision` | `resource.labels.service_name` | karaoke-backend, karaoke-decide, audio-separator | +| `cloud_run_job` | `resource.labels.job_name` | video-encoding-job, lyrics-transcription-job, audio-separation-job, audio-download-job | +| `cloud_function` | `resource.labels.function_name` | gdrive-validator, runner_manager, backup_to_aws, divebar_mirror, kn_data_sync, divebar_lookup, encoding_worker_idle | +| `gce_instance` | `labels."compute.googleapis.com/resource_name"` | encoding-worker-a, encoding-worker-b, flacfetch-vm, divebar-sync-vm | + +### Payload Handling + +Three payload types (same as Aquarius): +- Structured JSON dict → extract message from `textPayload` or `jsonPayload.message` +- Plain text string → use directly +- HTTP request logs (no payload) → skip + +--- + +## 3. Message Normalization + +### Standard Normalizers (from Aquarius) + +| Pattern | Placeholder | +|---------|-------------| +| UUIDs | `` | +| ISO timestamps | `` | +| Epoch timestamps | `` | +| Hex/alphanumeric IDs (8+ chars) | `` | +| Email addresses | `` | +| IP addresses | `` | +| URLs | domain only | +| Numbers (4+ digits) | `` | + +### NK-Specific Normalizers + +| Pattern | Placeholder | +|---------|-------------| +| Job IDs (`/jobs/`) | `/jobs/` | +| GCS paths (`gs://nomadkaraoke-*/**`) | `gs:///` | +| Firebase UIDs | `` | +| Audio file references (hashes, filenames) | `` | +| Firestore document paths | `` | + +### Pattern Hash + +`SHA-256(service::normalized_message)` — deterministic, collision-resistant. + +--- + +## 4. LLM Analysis + +### Model & Provider + +- **Model:** Gemini 2.0 Flash via Vertex AI +- **Auth:** Application Default Credentials (service account) +- **Temperature:** 0.2 for incident analysis, 0.1 for dedup +- **Retry:** Exponential backoff, 3 attempts, 2s base delay +- **Fallback:** If LLM unavailable, group by service name (no root cause analysis) + +### Incident Grouping + +When 2+ new patterns are detected, the LLM analyzes them together. The system prompt provides NK-specific service dependency context: + +``` +karaoke-backend (API) → orchestrates all workers + ├── audio-download-job → flacfetch-vm (downloads audio) + ├── audio-separation-job / audio-separator (GPU stem separation) + ├── lyrics-transcription-job (AudioShake + AI correction) + ├── video-encoding-job → encoding-worker-a/b (FFmpeg rendering) + └── youtube-upload (distribution) + +karaoke-decide (independent) → BigQuery, Spotify, Last.fm + +Cloud Functions: gdrive-validator, backup_to_aws, divebar_mirror, + kn_data_sync, divebar_lookup, runner_manager, + encoding_worker_idle +``` + +### Output Format (JSON) + +```json +{ + "incidents": [ + { + "title": "max 60 chars", + "root_cause": "one-liner", + "severity": "P0|P1|P2|P3", + "suggested_fix": "actionable, 1-2 sentences", + "primary_service": "upstream root cause service", + "pattern_indices": [1, 3] + } + ] +} +``` + +### Severity Levels + +- **P0:** Service completely down, all jobs failing, payment processing broken +- **P1:** Major pipeline stage broken (audio separation, lyrics, encoding), auth failures +- **P2:** Degraded but functional — intermittent errors, slow responses, single-job failures +- **P3:** Minor — deprecation warnings, cleanup failures, spot VM preemptions + +### Duplicate Detection + +Identifies near-duplicate patterns that regex normalization missed (different phrasing of same error). Only merges new patterns into existing canonical ones to preserve alert history. + +--- + +## 5. Firestore Data Model + +### `error_patterns` Collection + +| Field | Type | Description | +|-------|------|-------------| +| `pattern_id` | string | SHA-256(service::normalized_message) — document ID | +| `service` | string | Resource name (e.g., `karaoke-backend`, `video-encoding-job`) | +| `resource_type` | string | `cloud_run_service` / `cloud_run_job` / `cloud_function` / `gce_instance` | +| `normalized_message` | string | Regex-normalized error (max 200 chars) | +| `sample_message` | string | Raw error sample | +| `first_seen` | string (ISO) | First occurrence | +| `last_seen` | string (ISO) | Most recent occurrence | +| `total_count` | int | Cumulative count | +| `rolling_counts` | map | `{YYYY-MM-DDTHH:MM → count}` (7-day window) | +| `status` | string | `new` / `acknowledged` / `known` / `muted` / `fixed` / `auto_resolved` | +| `incident_id` | string? | Link to grouped incident | +| `severity` | string | `P0` / `P1` / `P2` / `P3` | +| `alerted_at` | string (ISO) | When Discord alert was sent | +| `notes` | string? | Free-text notes | +| `merged_into` | string? | Pattern ID this was merged into | +| `fixed_by` | map? | `{pr_display, pr_url, note, resolved_at}` | + +### `error_incidents` Collection + +| Field | Type | Description | +|-------|------|-------------| +| `incident_id` | string | `inc_YYYYMMDD_HHMM_<8-hex>` — document ID | +| `title` | string | Max 60 chars | +| `root_cause` | string? | LLM-identified | +| `severity` | string | `P0` / `P1` / `P2` / `P3` | +| `suggested_fix` | string? | Actionable next step | +| `primary_service` | string | Upstream root cause service | +| `pattern_ids` | list | Related pattern IDs | +| `used_llm` | bool | Whether LLM analysis was used | +| `created_at` | string (ISO) | Creation timestamp | +| `status` | string | `open` / `investigating` / `fixed` / `false_positive` | + +### `discord_alerts` Collection + +| Field | Type | Description | +|-------|------|-------------| +| Document ID | string | `alert_YYYYMMDD_HHMMSS_<8-hex>` | +| `timestamp` | string (ISO) | When sent | +| `alert_type` | string | `new_pattern` / `spike` / `incident` / `resolved_batch` / `daily_digest` | +| `content` | string | Message text (max 2000 chars) | +| `success` | bool | Whether webhook POST succeeded | +| + alert-specific metadata | varies | service, pattern, count, severity, etc. | + +### Migration from `known-issues.yaml` + +Existing entries in `known-issues.yaml` are migrated to `error_patterns` documents with status `known` or `muted`. The YAML file is then deprecated. + +--- + +## 6. Discord Notifications + +### Webhook + +Reuses the existing `discord-alert-webhook` secret in Secret Manager. + +### Alert Types + +| Alert Type | Trigger | Emoji | Content | +|------------|---------|-------|---------| +| **New Pattern** | Unrecognized error | `🔴` | Service, resource type, normalized msg, sample, count, timestamps, `/prod-investigate` prompt | +| **Incident** | 2+ patterns grouped by LLM | `🚨`/`🔴`/`⚠️`/`🟡` (P0-P3) | Title, root cause, per-pattern details, suggested fix | +| **Auto-Resolved** | Pattern silent past threshold | `✅` | Pattern, hours silent, consolidated | +| **Spike** | Count > 5x rolling avg (min 5) | `⚠️` | Pattern, current vs. normal count | +| **Daily Digest** | 08:00 UTC | `📊` | 24h totals, per-service breakdown, frequency stats | + +### Format Rules + +- Compact inline formatting (no bullet lists) +- Show actual error message text, never summarize +- Include copy-paste `/prod-investigate` prompts +- Rate limited: max 10 messages per run +- 2000-char limit with truncation +- Timestamps in Eastern time (America/New_York, DST-aware) +- Honest when LLM fails: "LLM grouping unavailable" + +--- + +## 7. Auto-Resolution + +Frequency-aware thresholds based on historical occurrence rate: + +``` +threshold_hours = mean_interval_between_occurrences * AUTO_RESOLVE_MULTIPLIER +``` + +| Setting | Value | +|---------|-------| +| `AUTO_RESOLVE_MULTIPLIER` | 8 (99.97% confidence, Poisson) | +| `AUTO_RESOLVE_MIN_HOURS` | 6 | +| `AUTO_RESOLVE_MAX_HOURS` | 168 (1 week) | +| `AUTO_RESOLVE_FALLBACK_HOURS` | 48 (when <3 data points) | + +High-frequency errors resolve faster; rare errors wait longer. + +--- + +## 8. Known Issues / Ignore Patterns + +Regex patterns for infrastructure noise that never triggers alerts: + +- Cloud Run cold start / startup probe failures +- Spot VM preemption events (GitHub runners) +- GitHub runner startup/shutdown noise +- Encoding worker idle shutdown (expected) +- Cloud Scheduler retry transients +- Health check 404s from load balancers +- Ready condition changes during deploys + +Defined in `known_issues.py`, easy to add to without code changes. + +--- + +## 9. Infrastructure (Pulumi) + +### New Resources (`infrastructure/modules/error_monitor.py`) + +1. **Cloud Run Job** (`nomad-error-monitor`) + - Image: existing karaoke-backend Docker image + - Command: `["python", "-m", "backend.services.error_monitor.monitor"]` + - Memory: 512Mi, CPU: 1, Timeout: 300s, Max retries: 0 + - Env vars: `GCP_PROJECT_ID`, `ENVIRONMENT`, `DISCORD_WEBHOOK_URL` (Secret Manager ref) + +2. **Cloud Scheduler — Monitor** (`error-monitor-trigger`) + - Schedule: `*/15 * * * *` (UTC) + - Target: Cloud Run Jobs API `:run` + - Auth: OAuth token via scheduler service account + - Retry: 1 attempt, 60s max + +3. **Cloud Scheduler — Daily Digest** (`error-monitor-daily-digest`) + - Schedule: `0 8 * * *` (UTC) + - Same job, with `DIGEST_MODE=true` environment override + +4. **Service Account** (`error-monitor-scheduler`) + - `roles/run.invoker` on the Cloud Run Job + +### IAM for Error Monitor Job + +The job runs under the existing backend service account (`karaoke-backend-sa`), which already has Firestore and Secret Manager access. Additional role needed: +- `roles/logging.viewer` (read Cloud Logging — not currently on the backend SA) + +--- + +## 10. Slash Command Updates + +All five `/prod-*` commands updated to use `error_patterns` as primary data source. + +### `/prod-health` +Add error pattern summary (active count, new patterns in last hour) alongside existing health endpoint checks. + +### `/prod-errors` +**Primary source becomes Firestore `error_patterns`** — fast, pre-deduplicated, pre-categorized. Cloud Logging becomes secondary, used only for stack traces when investigating a specific pattern. + +### `/prod-investigate` +Query `error_patterns` first for pattern context (history, frequency, related incidents), then Cloud Logging for stack traces and detailed samples. + +### `/prod-review` +Replace raw Cloud Logging categorization with pattern statuses: NEW, REGRESSIONS (previously fixed reoccurred), IN PROGRESS (acknowledged), KNOWN, VERIFIED FIXED. Remove `known-issues.yaml` dependency. + +### `/prod-known-issue` +Write to Firestore `error_patterns` (status: `known`/`muted`) instead of YAML file. + +### Helper Scripts + +- `scripts/query-error-patterns.py` — CLI to query patterns from Firestore (used by slash commands) +- `scripts/resolve-error-pattern.py` — CLI to mark patterns as fixed with PR reference + +--- + +## 11. Code Structure + +``` +backend/services/error_monitor/ +├── __init__.py +├── monitor.py # Entry point — query, normalize, group, analyze, alert +├── config.py # Configuration constants, monitored services list +├── normalizer.py # Message normalization (regex replacements) +├── known_issues.py # Ignore pattern matching +├── llm_analysis.py # Vertex AI Gemini incident grouping + dedup +├── discord.py # Discord webhook client +└── firestore_adapter.py # CRUD for error_patterns, incidents, alerts +``` + +### Configuration (`config.py`) + +| Setting | Value | +|---------|-------| +| `GCP_PROJECT` | `nomadkaraoke` | +| `LOOKBACK_MINUTES` | 15 | +| `MAX_LOG_ENTRIES` | 500 | +| `LLM_ANALYSIS_ENABLED` | true (env var) | +| `LLM_ANALYSIS_MODEL` | `gemini-2.0-flash` | +| `MIN_PATTERNS_FOR_ANALYSIS` | 2 | +| `SPIKE_MULTIPLIER` | 5.0 | +| `SPIKE_MIN_COUNT` | 5 | +| `MAX_DISCORD_MESSAGES_PER_RUN` | 10 | +| `MAX_ACTIVE_PATTERNS` | 500 | +| `ROLLING_WINDOW_DAYS` | 7 | +| `DISCORD_WEBHOOK_SECRET` | `discord-alert-webhook` | + +--- + +## 12. Testing + +### Unit Tests + +``` +tests/unit/services/error_monitor/ +├── test_monitor.py # Core grouping, windowing logic +├── test_normalizer.py # Message normalization +├── test_llm_analysis.py # LLM grouping, dedup +├── test_firestore_adapter.py # CRUD, rolling counts, resolution +├── test_discord.py # Message formatting +├── test_known_issues.py # Ignore pattern matching +├── test_dedup.py # Duplicate detection +└── test_monitor_orchestration.py # Full pipeline integration +``` + +### Integration Tests + +- Test against Firestore emulator for pattern CRUD +- Test Cloud Logging query construction (mock the API client) +- Test Discord webhook formatting (mock HTTP) + +--- + +## 13. Out of Scope (Future Work) + +- **Auto-fixer (Nomad-fix):** Autonomous Claude Code agent that receives errors via Pub/Sub, creates fix PRs. Designed as a separate follow-up project. +- **Frontend admin dashboard:** Visual error monitoring UI. The slash commands serve this purpose for now. +- **Cross-repo error correlation:** Correlating errors between karaoke-gen and karaoke-decide Firestore. Both write to the same `error_patterns` collection, so patterns from both are visible together. diff --git a/docs/archive/2026-04-12-error-monitor-plan.md b/docs/archive/2026-04-12-error-monitor-plan.md new file mode 100644 index 00000000..4da3ee17 --- /dev/null +++ b/docs/archive/2026-04-12-error-monitor-plan.md @@ -0,0 +1,3369 @@ +# Nomad Karaoke Error Monitor — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a production error monitoring service that queries Cloud Logging every 15 minutes, normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and tracks patterns in Firestore. + +**Architecture:** Cloud Run Job sharing the karaoke-backend Docker image with command override. Firestore for pattern/incident persistence. Gemini 2.0 Flash via Vertex AI for LLM dedup and incident grouping. Discord webhook for alerts. Cloud Scheduler for 15-min trigger and daily digest. + +**Tech Stack:** Python 3.12, google-cloud-logging, google-cloud-firestore, google-generativeai (Vertex AI), requests, Pulumi (GCP infrastructure) + +**Design Spec:** `docs/archive/2026-04-12-error-monitor-design.md` + +**Aquarius Reference:** `/Users/andrew/Projects/aquarius/docs/archive/error-bot-system-reference.md` + +--- + +## File Structure + +### New Files + +``` +backend/services/error_monitor/ +├── __init__.py # Package init, version +├── config.py # All configuration constants +├── normalizer.py # Message normalization (regex replacements) +├── known_issues.py # Ignore pattern matching +├── firestore_adapter.py # CRUD for error_patterns, incidents, alerts +├── discord.py # Discord webhook client for alerts +├── llm_analysis.py # Vertex AI Gemini incident grouping + dedup +└── monitor.py # Entry point — orchestrates full pipeline + +scripts/ +├── query-error-patterns.py # CLI to query patterns from Firestore +└── resolve-error-pattern.py # CLI to mark patterns as fixed + +infrastructure/modules/ +└── error_monitor.py # Pulumi: Cloud Run Job + Cloud Schedulers + +tests/unit/services/error_monitor/ +├── __init__.py +├── test_config.py +├── test_normalizer.py +├── test_known_issues.py +├── test_firestore_adapter.py +├── test_discord.py +├── test_llm_analysis.py +└── test_monitor.py +``` + +### Modified Files + +``` +infrastructure/__main__.py # Wire in error_monitor module +infrastructure/config.py # Add ErrorMonitorConfig class +infrastructure/modules/iam/backend_sa.py # Add roles/logging.viewer +pyproject.toml # Add google-cloud-logging dependency +``` + +### Slash Commands (workspace level, updated) + +``` +/Users/andrew/Projects/nomadkaraoke/.claude/commands/ +├── prod-errors.md # Update to use error_patterns as primary source +├── prod-health.md # Add error pattern summary +├── prod-investigate.md # Query error_patterns first +├── prod-review.md # Replace Cloud Logging categorization with pattern statuses +└── prod-known-issue.md # Write to Firestore instead of YAML +``` + +--- + +## Task 1: Configuration & Dependencies + +**Files:** +- Create: `backend/services/error_monitor/__init__.py` +- Create: `backend/services/error_monitor/config.py` +- Create: `tests/unit/services/error_monitor/__init__.py` +- Create: `tests/unit/services/error_monitor/test_config.py` +- Modify: `pyproject.toml` (add google-cloud-logging) + +- [ ] **Step 1: Add google-cloud-logging dependency** + +In `pyproject.toml`, add to `[tool.poetry.dependencies]`: + +``` +google-cloud-logging = ">=3.9.0" +``` + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && poetry lock --no-update` + +- [ ] **Step 2: Create package init** + +Create `backend/services/error_monitor/__init__.py`: + +```python +"""Production error monitoring service for Nomad Karaoke.""" +``` + +Create `tests/unit/services/error_monitor/__init__.py`: + +```python +``` + +- [ ] **Step 3: Write config tests** + +Create `tests/unit/services/error_monitor/test_config.py`: + +```python +"""Tests for error monitor configuration.""" + +import os +from unittest.mock import patch + +from backend.services.error_monitor.config import ( + MONITORED_CLOUD_RUN_SERVICES, + MONITORED_CLOUD_RUN_JOBS, + MONITORED_CLOUD_FUNCTIONS, + MONITORED_GCE_INSTANCES, + GCP_PROJECT, + LOOKBACK_MINUTES, + MAX_LOG_ENTRIES, + SPIKE_MULTIPLIER, + SPIKE_MIN_COUNT, + AUTO_RESOLVE_MULTIPLIER, + AUTO_RESOLVE_MIN_HOURS, + AUTO_RESOLVE_MAX_HOURS, + AUTO_RESOLVE_FALLBACK_HOURS, + MAX_DISCORD_MESSAGES_PER_RUN, + MAX_ACTIVE_PATTERNS, + ROLLING_WINDOW_DAYS, + get_llm_enabled, + get_discord_webhook_secret_name, +) + + +class TestMonitoredServices: + def test_cloud_run_services_are_defined(self): + assert "karaoke-backend" in MONITORED_CLOUD_RUN_SERVICES + assert "karaoke-decide" in MONITORED_CLOUD_RUN_SERVICES + assert "audio-separator" in MONITORED_CLOUD_RUN_SERVICES + assert len(MONITORED_CLOUD_RUN_SERVICES) == 3 + + def test_cloud_run_jobs_are_defined(self): + assert "video-encoding-job" in MONITORED_CLOUD_RUN_JOBS + assert "lyrics-transcription-job" in MONITORED_CLOUD_RUN_JOBS + assert "audio-separation-job" in MONITORED_CLOUD_RUN_JOBS + assert "audio-download-job" in MONITORED_CLOUD_RUN_JOBS + assert len(MONITORED_CLOUD_RUN_JOBS) == 4 + + def test_cloud_functions_are_defined(self): + assert "gdrive-validator" in MONITORED_CLOUD_FUNCTIONS + assert "runner_manager" in MONITORED_CLOUD_FUNCTIONS + assert len(MONITORED_CLOUD_FUNCTIONS) >= 7 + + def test_gce_instances_are_defined(self): + assert "encoding-worker-a" in MONITORED_GCE_INSTANCES + assert "flacfetch-vm" in MONITORED_GCE_INSTANCES + assert len(MONITORED_GCE_INSTANCES) >= 4 + + +class TestDefaults: + def test_gcp_project(self): + assert GCP_PROJECT == "nomadkaraoke" + + def test_lookback_minutes(self): + assert LOOKBACK_MINUTES == 15 + + def test_max_log_entries(self): + assert MAX_LOG_ENTRIES == 500 + + def test_spike_settings(self): + assert SPIKE_MULTIPLIER == 5.0 + assert SPIKE_MIN_COUNT == 5 + + def test_auto_resolve_settings(self): + assert AUTO_RESOLVE_MULTIPLIER == 8 + assert AUTO_RESOLVE_MIN_HOURS == 6 + assert AUTO_RESOLVE_MAX_HOURS == 168 + assert AUTO_RESOLVE_FALLBACK_HOURS == 48 + + def test_discord_limits(self): + assert MAX_DISCORD_MESSAGES_PER_RUN == 10 + + def test_rolling_window(self): + assert ROLLING_WINDOW_DAYS == 7 + + +class TestEnvOverrides: + @patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "false"}) + def test_llm_disabled_via_env(self): + assert get_llm_enabled() is False + + @patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "true"}) + def test_llm_enabled_via_env(self): + assert get_llm_enabled() is True + + def test_llm_enabled_by_default(self): + with patch.dict(os.environ, {}, clear=True): + # Remove the key if present + os.environ.pop("LLM_ANALYSIS_ENABLED", None) + assert get_llm_enabled() is True +``` + +- [ ] **Step 4: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_config.py -v` + +Expected: ImportError — `backend.services.error_monitor.config` does not exist yet. + +- [ ] **Step 5: Implement config module** + +Create `backend/services/error_monitor/config.py`: + +```python +"""Configuration for the error monitor service.""" + +import os + +# GCP +GCP_PROJECT = os.environ.get("GCP_PROJECT_ID", "nomadkaraoke") +GCP_REGION = os.environ.get("GCP_REGION", "us-central1") + +# Log collection +LOOKBACK_MINUTES = 15 +MAX_LOG_ENTRIES = 500 + +# Monitored resources +MONITORED_CLOUD_RUN_SERVICES = [ + "karaoke-backend", + "karaoke-decide", + "audio-separator", +] + +MONITORED_CLOUD_RUN_JOBS = [ + "video-encoding-job", + "lyrics-transcription-job", + "audio-separation-job", + "audio-download-job", +] + +MONITORED_CLOUD_FUNCTIONS = [ + "gdrive-validator", + "runner_manager", + "backup_to_aws", + "divebar_mirror", + "kn_data_sync", + "divebar_lookup", + "encoding_worker_idle", +] + +MONITORED_GCE_INSTANCES = [ + "encoding-worker-a", + "encoding-worker-b", + "flacfetch-vm", + "divebar-sync-vm", +] + +# Service dependency map (for LLM system prompt) +SERVICE_DEPENDENCY_MAP = """ +karaoke-backend (API) → orchestrates all workers + ├── audio-download-job → flacfetch-vm (downloads audio) + ├── audio-separation-job / audio-separator (GPU stem separation) + ├── lyrics-transcription-job (AudioShake + AI correction) + ├── video-encoding-job → encoding-worker-a/b (FFmpeg rendering) + └── youtube-upload (distribution) + +karaoke-decide (independent) → BigQuery, Spotify, Last.fm + +Cloud Functions: gdrive-validator, backup_to_aws, divebar_mirror, + kn_data_sync, divebar_lookup, runner_manager, + encoding_worker_idle +""".strip() + +# LLM analysis +LLM_ANALYSIS_MODEL = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-2.0-flash") +LLM_VERTEX_LOCATION = os.environ.get("LLM_VERTEX_LOCATION", "us-central1") +MIN_PATTERNS_FOR_ANALYSIS = int(os.environ.get("MIN_PATTERNS_FOR_ANALYSIS", "2")) + +# Spike detection +SPIKE_MULTIPLIER = 5.0 +SPIKE_MIN_COUNT = 5 + +# Auto-resolution +AUTO_RESOLVE_MULTIPLIER = 8 +AUTO_RESOLVE_MIN_HOURS = 6 +AUTO_RESOLVE_MAX_HOURS = 168 # 1 week +AUTO_RESOLVE_FALLBACK_HOURS = 48 + +# Discord +MAX_DISCORD_MESSAGES_PER_RUN = 10 +DISCORD_MAX_MESSAGE_LENGTH = 2000 + +# Pattern tracking +MAX_ACTIVE_PATTERNS = 500 +ROLLING_WINDOW_DAYS = 7 +MAX_NORMALIZED_MESSAGE_LENGTH = 200 + + +def get_llm_enabled() -> bool: + """Check if LLM analysis is enabled (default: True).""" + return os.environ.get("LLM_ANALYSIS_ENABLED", "true").lower() == "true" + + +def get_discord_webhook_secret_name() -> str: + """Get the Secret Manager secret name for the Discord webhook.""" + return os.environ.get("DISCORD_WEBHOOK_SECRET", "discord-alert-webhook") + + +def get_digest_mode() -> bool: + """Check if running in daily digest mode.""" + return os.environ.get("DIGEST_MODE", "false").lower() == "true" +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_config.py -v` + +Expected: All tests PASS. + +- [ ] **Step 7: Commit** + +```bash +git add backend/services/error_monitor/__init__.py backend/services/error_monitor/config.py tests/unit/services/error_monitor/__init__.py tests/unit/services/error_monitor/test_config.py pyproject.toml +git commit -m "feat(error-monitor): add config module and dependencies" +``` + +--- + +## Task 2: Message Normalizer + +**Files:** +- Create: `backend/services/error_monitor/normalizer.py` +- Create: `tests/unit/services/error_monitor/test_normalizer.py` + +- [ ] **Step 1: Write normalizer tests** + +Create `tests/unit/services/error_monitor/test_normalizer.py`: + +```python +"""Tests for error message normalization.""" + +from backend.services.error_monitor.normalizer import normalize_message, compute_pattern_hash + + +class TestNormalizeMessage: + def test_replaces_uuids(self): + msg = "Failed to process job 550e8400-e29b-41d4-a716-446655440000" + assert "" in normalize_message(msg) + assert "550e8400" not in normalize_message(msg) + + def test_replaces_iso_timestamps(self): + msg = "Error at 2026-04-10T14:30:00.123Z in handler" + assert "" in normalize_message(msg) + assert "2026-04-10" not in normalize_message(msg) + + def test_replaces_epoch_timestamps(self): + msg = "Timeout after 1712345678.123 seconds elapsed" + result = normalize_message(msg) + assert "" in result + + def test_replaces_emails(self): + msg = "Auth failed for user@example.com" + assert "" in normalize_message(msg) + assert "user@example.com" not in normalize_message(msg) + + def test_replaces_ip_addresses(self): + msg = "Connection refused from 192.168.1.100" + assert "" in normalize_message(msg) + + def test_replaces_hex_ids(self): + msg = "Document abc123def456 not found" + assert "" in normalize_message(msg) + + def test_replaces_large_numbers(self): + msg = "Request took 12345 ms" + assert "" in normalize_message(msg) + + def test_preserves_small_numbers(self): + msg = "Error code 404" + result = normalize_message(msg) + assert "404" in result + + def test_replaces_job_ids_in_paths(self): + msg = "Failed /jobs/abc123def456ghi789 processing" + result = normalize_message(msg) + assert "/jobs/" in result + + def test_replaces_gcs_paths(self): + msg = "File not found: gs://nomadkaraoke-prod/audio/abc123/vocals.wav" + result = normalize_message(msg) + assert "gs:///" in result + + def test_replaces_firebase_uids(self): + msg = "User XLsws9aOZ5hJA0hJHQ7ymw5x4pw2 not found" + result = normalize_message(msg) + assert "" in result + + def test_replaces_firestore_doc_paths(self): + msg = "Error reading jobs/abc123def/state_data/current" + result = normalize_message(msg) + assert "" in result + + def test_truncates_long_messages(self): + msg = "x" * 300 + result = normalize_message(msg) + assert len(result) <= 200 + + def test_strips_urls_to_domain(self): + msg = "Failed to reach https://api.audioshake.ai/v1/transcribe?key=abc123" + result = normalize_message(msg) + assert "api.audioshake.ai" in result + assert "key=abc123" not in result + + def test_deterministic(self): + msg = "Error processing job 550e8400-e29b-41d4-a716-446655440000 at 2026-04-10T14:30:00Z" + assert normalize_message(msg) == normalize_message(msg) + + def test_empty_message(self): + assert normalize_message("") == "" + + def test_no_replacements_needed(self): + msg = "Simple error message" + assert normalize_message(msg) == "Simple error message" + + +class TestComputePatternHash: + def test_returns_hex_string(self): + result = compute_pattern_hash("karaoke-backend", "Some error") + assert len(result) == 64 # SHA-256 hex + assert all(c in "0123456789abcdef" for c in result) + + def test_deterministic(self): + h1 = compute_pattern_hash("karaoke-backend", "Error X") + h2 = compute_pattern_hash("karaoke-backend", "Error X") + assert h1 == h2 + + def test_different_services_different_hash(self): + h1 = compute_pattern_hash("karaoke-backend", "Error X") + h2 = compute_pattern_hash("karaoke-decide", "Error X") + assert h1 != h2 + + def test_different_messages_different_hash(self): + h1 = compute_pattern_hash("karaoke-backend", "Error X") + h2 = compute_pattern_hash("karaoke-backend", "Error Y") + assert h1 != h2 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_normalizer.py -v` + +Expected: ImportError. + +- [ ] **Step 3: Implement normalizer** + +Create `backend/services/error_monitor/normalizer.py`: + +```python +"""Error message normalization for pattern deduplication. + +Replaces variable parts (IDs, timestamps, emails, etc.) with placeholders +so the same logical error produces the same pattern hash. +""" + +import hashlib +import re + +from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH + +# Order matters — more specific patterns first to prevent partial matches. +_NORMALIZERS: list[tuple[re.Pattern, str]] = [ + # GCS paths (before general URLs) + (re.compile(r"gs://[a-zA-Z0-9._-]+/[^\s]+"), "gs:///"), + # Firestore document paths (collections/docId/subcollections/docId) + (re.compile(r"(?"), + # UUIDs + (re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I), ""), + # ISO timestamps (with optional fractional seconds and timezone) + (re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?"), ""), + # Epoch timestamps (10+ digit float) + (re.compile(r"\b\d{10,13}\.\d+\b"), ""), + # Email addresses + (re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"), ""), + # IP addresses + (re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), ""), + # URLs — strip to domain only + (re.compile(r"https?://([a-zA-Z0-9.-]+)[^\s]*"), r"\1"), + # Job IDs in paths (before general hex IDs) + (re.compile(r"/jobs/[a-zA-Z0-9_-]{8,}"), "/jobs/"), + # Firebase UIDs (20-28 alphanumeric chars, typically mixed case) + (re.compile(r"\b[a-zA-Z0-9]{20,28}\b"), ""), + # Hex/alphanumeric IDs (8+ chars with at least one digit and one letter) + (re.compile(r"\b(?=[a-f0-9]*[a-f])(?=[a-f0-9]*[0-9])[a-f0-9]{8,}\b", re.I), ""), + # Large numbers (4+ digits) + (re.compile(r"\b\d{4,}\b"), ""), +] + + +def normalize_message(message: str) -> str: + """Normalize an error message by replacing variable parts with placeholders. + + Returns the normalized message, truncated to MAX_NORMALIZED_MESSAGE_LENGTH. + """ + if not message: + return "" + + result = message + for pattern, replacement in _NORMALIZERS: + result = pattern.sub(replacement, result) + + if len(result) > MAX_NORMALIZED_MESSAGE_LENGTH: + result = result[:MAX_NORMALIZED_MESSAGE_LENGTH] + + return result + + +def compute_pattern_hash(service: str, normalized_message: str) -> str: + """Compute a deterministic hash for a service::message pattern. + + Returns a 64-char lowercase hex SHA-256 digest. + """ + key = f"{service}::{normalized_message}" + return hashlib.sha256(key.encode("utf-8")).hexdigest() +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_normalizer.py -v` + +Expected: All tests PASS. If any regex tests fail, adjust the regex patterns and re-run until all pass. + +- [ ] **Step 5: Commit** + +```bash +git add backend/services/error_monitor/normalizer.py tests/unit/services/error_monitor/test_normalizer.py +git commit -m "feat(error-monitor): add message normalizer with NK-specific patterns" +``` + +--- + +## Task 3: Known Issues / Ignore Patterns + +**Files:** +- Create: `backend/services/error_monitor/known_issues.py` +- Create: `tests/unit/services/error_monitor/test_known_issues.py` + +- [ ] **Step 1: Write known issues tests** + +Create `tests/unit/services/error_monitor/test_known_issues.py`: + +```python +"""Tests for known issue / ignore pattern matching.""" + +from backend.services.error_monitor.known_issues import should_ignore, IgnoreReason + + +class TestShouldIgnore: + def test_ignores_cloud_run_startup_probe(self): + result = should_ignore( + service="karaoke-backend", + message="Startup probe failed: HTTP probe failed with statuscode: 503", + ) + assert result is not None + assert isinstance(result, IgnoreReason) + assert "startup" in result.reason.lower() + + def test_ignores_ready_condition_change(self): + result = should_ignore( + service="karaoke-backend", + message="Ready condition status changed to True", + ) + assert result is not None + + def test_ignores_spot_vm_preemption(self): + result = should_ignore( + service="github-runner-1", + message="Instance was preempted", + ) + assert result is not None + + def test_ignores_encoding_worker_idle_shutdown(self): + result = should_ignore( + service="encoding_worker_idle", + message="Stopping idle encoding worker encoding-worker-a", + ) + assert result is not None + + def test_ignores_health_check_404(self): + result = should_ignore( + service="karaoke-backend", + message='GET /health returned 404 "Not Found"', + ) + assert result is not None + + def test_does_not_ignore_real_errors(self): + result = should_ignore( + service="karaoke-backend", + message="Firestore transaction failed: DEADLINE_EXCEEDED", + ) + assert result is None + + def test_does_not_ignore_unknown_service(self): + result = should_ignore( + service="karaoke-backend", + message="NullPointerException in processJob", + ) + assert result is None + + def test_case_insensitive_matching(self): + result = should_ignore( + service="karaoke-backend", + message="STARTUP PROBE FAILED: connection refused", + ) + assert result is not None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_known_issues.py -v` + +Expected: ImportError. + +- [ ] **Step 3: Implement known issues module** + +Create `backend/services/error_monitor/known_issues.py`: + +```python +"""Ignore patterns for infrastructure noise that should never trigger alerts.""" + +import re +from dataclasses import dataclass + + +@dataclass +class IgnoreReason: + """Why a log entry was ignored.""" + pattern_name: str + reason: str + + +# Each entry: (pattern_name, compiled_regex, reason) +# Regex is matched against the full message, case-insensitive. +_IGNORE_PATTERNS: list[tuple[str, re.Pattern, str]] = [ + ( + "startup_probe", + re.compile(r"startup probe failed", re.I), + "Startup probe failure during Cloud Run cold start (transient)", + ), + ( + "ready_condition", + re.compile(r"ready condition status changed", re.I), + "Ready condition change during deploy (expected)", + ), + ( + "spot_preemption", + re.compile(r"instance was preempted", re.I), + "Spot VM preemption (expected for GitHub runners)", + ), + ( + "idle_shutdown", + re.compile(r"stopping idle encoding worker", re.I), + "Encoding worker idle shutdown (expected behavior)", + ), + ( + "health_check_404", + re.compile(r"(?:GET|HEAD)\s+/health.*(?:404|not found)", re.I), + "Health check 404 from load balancer (transient during deploys)", + ), + ( + "scheduler_retry", + re.compile(r"cloud scheduler.*retry", re.I), + "Cloud Scheduler retry noise (transient)", + ), + ( + "runner_startup", + re.compile(r"github.runner.*(?:starting|stopping|idle)", re.I), + "GitHub runner lifecycle event (expected)", + ), + ( + "container_shutdown", + re.compile(r"container called exit\(0\)", re.I), + "Clean container shutdown (not an error)", + ), +] + + +def should_ignore(service: str, message: str) -> IgnoreReason | None: + """Check if a log entry should be ignored. + + Returns an IgnoreReason if the message matches a known ignore pattern, + or None if it should be processed. + """ + for pattern_name, regex, reason in _IGNORE_PATTERNS: + if regex.search(message): + return IgnoreReason(pattern_name=pattern_name, reason=reason) + return None +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_known_issues.py -v` + +Expected: All tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/services/error_monitor/known_issues.py tests/unit/services/error_monitor/test_known_issues.py +git commit -m "feat(error-monitor): add ignore patterns for infrastructure noise" +``` + +--- + +## Task 4: Firestore Adapter + +**Files:** +- Create: `backend/services/error_monitor/firestore_adapter.py` +- Create: `tests/unit/services/error_monitor/test_firestore_adapter.py` + +- [ ] **Step 1: Write Firestore adapter tests** + +Create `tests/unit/services/error_monitor/test_firestore_adapter.py`: + +```python +"""Tests for Firestore adapter (error_patterns, incidents, alerts).""" + +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch, call + +import pytest + +from backend.services.error_monitor.firestore_adapter import ( + ErrorPatternsAdapter, + PatternData, +) + + +@pytest.fixture +def mock_db(): + return MagicMock() + + +@pytest.fixture +def adapter(mock_db): + return ErrorPatternsAdapter(db=mock_db) + + +class TestGetPattern: + def test_returns_none_for_missing_pattern(self, adapter, mock_db): + mock_db.collection.return_value.document.return_value.get.return_value.exists = False + result = adapter.get_pattern("nonexistent_hash") + assert result is None + + def test_returns_pattern_data_for_existing(self, adapter, mock_db): + doc_mock = MagicMock() + doc_mock.exists = True + doc_mock.to_dict.return_value = { + "pattern_id": "abc123", + "service": "karaoke-backend", + "status": "new", + "total_count": 5, + } + mock_db.collection.return_value.document.return_value.get.return_value = doc_mock + result = adapter.get_pattern("abc123") + assert result is not None + assert result["pattern_id"] == "abc123" + + +class TestUpsertPattern: + def test_creates_new_pattern(self, adapter, mock_db): + doc_mock = MagicMock() + doc_mock.exists = False + mock_db.collection.return_value.document.return_value.get.return_value = doc_mock + + adapter.upsert_pattern(PatternData( + pattern_id="hash1", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Error in ", + sample_message="Error in abc123", + count=3, + timestamp=datetime(2026, 4, 12, 10, 0, tzinfo=timezone.utc), + )) + + mock_db.collection.return_value.document.return_value.set.assert_called_once() + call_args = mock_db.collection.return_value.document.return_value.set.call_args + data = call_args[0][0] + assert data["pattern_id"] == "hash1" + assert data["status"] == "new" + assert data["total_count"] == 3 + + def test_updates_existing_pattern_counts(self, adapter, mock_db): + doc_mock = MagicMock() + doc_mock.exists = True + doc_mock.to_dict.return_value = { + "pattern_id": "hash1", + "service": "karaoke-backend", + "status": "acknowledged", + "total_count": 10, + "rolling_counts": {}, + } + mock_db.collection.return_value.document.return_value.get.return_value = doc_mock + + adapter.upsert_pattern(PatternData( + pattern_id="hash1", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Error in ", + sample_message="Error in xyz789", + count=5, + timestamp=datetime(2026, 4, 12, 10, 0, tzinfo=timezone.utc), + )) + + mock_db.collection.return_value.document.return_value.update.assert_called_once() + + def test_reactivates_auto_resolved_pattern(self, adapter, mock_db): + doc_mock = MagicMock() + doc_mock.exists = True + doc_mock.to_dict.return_value = { + "pattern_id": "hash1", + "status": "auto_resolved", + "total_count": 10, + "rolling_counts": {}, + } + mock_db.collection.return_value.document.return_value.get.return_value = doc_mock + + result = adapter.upsert_pattern(PatternData( + pattern_id="hash1", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Error in ", + sample_message="Error again", + count=1, + timestamp=datetime(2026, 4, 12, 10, 0, tzinfo=timezone.utc), + )) + + assert result.is_new is True # Treat as new since it was resolved + + +class TestGetActivePatterns: + def test_returns_active_patterns(self, adapter, mock_db): + doc1 = MagicMock() + doc1.to_dict.return_value = {"pattern_id": "a", "status": "new"} + doc2 = MagicMock() + doc2.to_dict.return_value = {"pattern_id": "b", "status": "acknowledged"} + + mock_query = MagicMock() + mock_query.stream.return_value = [doc1, doc2] + mock_db.collection.return_value.where.return_value.where.return_value = mock_query + + result = adapter.get_active_patterns() + assert len(result) == 2 + + +class TestAutoResolve: + def test_computes_threshold_from_rolling_counts(self, adapter): + rolling_counts = { + "2026-04-12T09:00": 2, + "2026-04-12T09:15": 3, + "2026-04-12T09:30": 1, + "2026-04-12T09:45": 4, + } + threshold = adapter._compute_resolve_threshold(rolling_counts) + # Mean interval = ~15 min = 0.25 hours + # threshold = 0.25 * 8 = 2 hours, clamped to min 6 + assert threshold >= 6 + assert threshold <= 168 + + +class TestCreateIncident: + def test_creates_incident_document(self, adapter, mock_db): + adapter.create_incident( + title="Audio separation failures", + root_cause="GPU OOM on large files", + severity="P1", + suggested_fix="Increase GPU memory limit", + primary_service="audio-separator", + pattern_ids=["hash1", "hash2"], + used_llm=True, + ) + + mock_db.collection.return_value.document.return_value.set.assert_called_once() + call_args = mock_db.collection.return_value.document.return_value.set.call_args + data = call_args[0][0] + assert data["title"] == "Audio separation failures" + assert data["severity"] == "P1" + assert len(data["pattern_ids"]) == 2 + assert data["status"] == "open" + + +class TestMergePattern: + def test_marks_source_as_muted(self, adapter, mock_db): + adapter.merge_pattern("source_hash", "target_hash", "Same error, different phrasing") + mock_db.collection.return_value.document.return_value.update.assert_called_once() + call_args = mock_db.collection.return_value.document.return_value.update.call_args + data = call_args[0][0] + assert data["status"] == "muted" + assert data["merged_into"] == "target_hash" + assert "Same error" in data["merged_reason"] + + +class TestUpdatePatternAlerted: + def test_sets_alerted_timestamp(self, adapter, mock_db): + adapter.update_pattern_alerted("hash1") + mock_db.collection.return_value.document.return_value.update.assert_called_once() + call_args = mock_db.collection.return_value.document.return_value.update.call_args + data = call_args[0][0] + assert "alerted_at" in data + assert data["alerted_at"] is not None + + +class TestLogDiscordAlert: + def test_logs_alert_to_firestore(self, adapter, mock_db): + adapter.log_discord_alert( + alert_type="new_pattern", + content="🔴 New error in karaoke-backend...", + success=True, + metadata={"service": "karaoke-backend", "pattern_id": "hash1"}, + ) + + mock_db.collection.return_value.document.return_value.set.assert_called_once() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_firestore_adapter.py -v` + +Expected: ImportError. + +- [ ] **Step 3: Implement Firestore adapter** + +Create `backend/services/error_monitor/firestore_adapter.py`: + +```python +"""Firestore adapter for error_patterns, error_incidents, and discord_alerts.""" + +import logging +import os +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone + +from google.cloud import firestore + +from backend.services.error_monitor.config import ( + AUTO_RESOLVE_FALLBACK_HOURS, + AUTO_RESOLVE_MAX_HOURS, + AUTO_RESOLVE_MIN_HOURS, + AUTO_RESOLVE_MULTIPLIER, + ROLLING_WINDOW_DAYS, +) + +logger = logging.getLogger(__name__) + +PATTERNS_COLLECTION = "error_patterns" +INCIDENTS_COLLECTION = "error_incidents" +ALERTS_COLLECTION = "discord_alerts" + +# Statuses that count as "active" (not resolved/fixed) +ACTIVE_STATUSES = ["new", "acknowledged", "known"] + +# Statuses that should be treated as "new" when seen again +REACTIVATE_STATUSES = ["auto_resolved", "fixed"] + + +@dataclass +class PatternData: + """Input data for upserting a pattern.""" + pattern_id: str + service: str + resource_type: str + normalized_message: str + sample_message: str + count: int + timestamp: datetime + + +@dataclass +class UpsertResult: + """Result of upserting a pattern.""" + pattern_id: str + is_new: bool + previous_status: str | None = None + + +class ErrorPatternsAdapter: + """Firestore CRUD for error patterns, incidents, and alerts.""" + + def __init__(self, db: firestore.Client | None = None): + self.db = db or firestore.Client( + project=os.environ.get("GCP_PROJECT_ID", "nomadkaraoke") + ) + + def get_pattern(self, pattern_id: str) -> dict | None: + """Get a single pattern by ID. Returns None if not found.""" + doc = self.db.collection(PATTERNS_COLLECTION).document(pattern_id).get() + if doc.exists: + return doc.to_dict() + return None + + def upsert_pattern(self, data: PatternData) -> UpsertResult: + """Create or update an error pattern. + + Returns UpsertResult indicating whether this is a new pattern. + A pattern is considered "new" if it didn't exist before, or if + it was previously auto_resolved/fixed and has reappeared. + """ + doc_ref = self.db.collection(PATTERNS_COLLECTION).document(data.pattern_id) + doc = doc_ref.get() + ts_iso = data.timestamp.isoformat() + window_key = data.timestamp.strftime("%Y-%m-%dT%H:%M") + + if not doc.exists: + doc_ref.set({ + "pattern_id": data.pattern_id, + "service": data.service, + "resource_type": data.resource_type, + "normalized_message": data.normalized_message, + "sample_message": data.sample_message, + "first_seen": ts_iso, + "last_seen": ts_iso, + "total_count": data.count, + "rolling_counts": {window_key: data.count}, + "status": "new", + "severity": "P3", # Default, upgraded by LLM analysis + "alerted_at": None, + "incident_id": None, + "notes": None, + "merged_into": None, + "fixed_by": None, + }) + return UpsertResult(pattern_id=data.pattern_id, is_new=True) + + existing = doc.to_dict() + previous_status = existing.get("status", "new") + is_reactivation = previous_status in REACTIVATE_STATUSES + + rolling_counts = existing.get("rolling_counts", {}) + rolling_counts[window_key] = rolling_counts.get(window_key, 0) + data.count + self._prune_rolling_counts(rolling_counts) + + update_data = { + "last_seen": ts_iso, + "total_count": existing.get("total_count", 0) + data.count, + "rolling_counts": rolling_counts, + "sample_message": data.sample_message, # Keep latest sample + } + + if is_reactivation: + update_data["status"] = "new" + update_data["alerted_at"] = None # Will be re-alerted + + doc_ref.update(update_data) + return UpsertResult( + pattern_id=data.pattern_id, + is_new=is_reactivation, + previous_status=previous_status, + ) + + def get_active_patterns(self) -> list[dict]: + """Get all patterns with active status (new, acknowledged, known).""" + results = [] + for status in ACTIVE_STATUSES: + docs = ( + self.db.collection(PATTERNS_COLLECTION) + .where("status", "in", ACTIVE_STATUSES) + .stream() + ) + results = [doc.to_dict() for doc in docs] + break # Only need one query with "in" + return results + + def get_patterns_for_auto_resolve(self) -> list[dict]: + """Get active patterns that might be eligible for auto-resolution.""" + return [ + p for p in self.get_active_patterns() + if p.get("status") in ("new", "acknowledged") + ] + + def auto_resolve_pattern(self, pattern_id: str, hours_silent: float) -> None: + """Mark a pattern as auto-resolved.""" + self.db.collection(PATTERNS_COLLECTION).document(pattern_id).update({ + "status": "auto_resolved", + "notes": f"Auto-resolved after {hours_silent:.1f} hours of silence", + }) + logger.info(f"Auto-resolved pattern {pattern_id} after {hours_silent:.1f}h") + + def resolve_pattern( + self, pattern_id: str, pr_url: str | None = None, note: str = "" + ) -> None: + """Manually resolve a pattern (mark as fixed).""" + fixed_by = { + "resolved_at": datetime.now(timezone.utc).isoformat(), + "note": note, + } + if pr_url: + fixed_by["pr_url"] = pr_url + # Extract "org/repo#N" display from URL + parts = pr_url.rstrip("/").split("/") + if len(parts) >= 2: + fixed_by["pr_display"] = f"{parts[-3]}/{parts[-2]}#{parts[-1]}" + + self.db.collection(PATTERNS_COLLECTION).document(pattern_id).update({ + "status": "fixed", + "fixed_by": fixed_by, + }) + + def _compute_resolve_threshold(self, rolling_counts: dict) -> float: + """Compute auto-resolve threshold in hours from rolling counts. + + Uses frequency-aware thresholds: threshold = mean_interval * multiplier. + Clamped between MIN and MAX hours. + """ + if len(rolling_counts) < 3: + return AUTO_RESOLVE_FALLBACK_HOURS + + # Parse timestamps and sort + timestamps = sorted(rolling_counts.keys()) + if len(timestamps) < 2: + return AUTO_RESOLVE_FALLBACK_HOURS + + # Compute mean interval between occurrences in hours + intervals = [] + for i in range(1, len(timestamps)): + try: + t1 = datetime.fromisoformat(timestamps[i - 1]) + t2 = datetime.fromisoformat(timestamps[i]) + hours = (t2 - t1).total_seconds() / 3600 + if hours > 0: + intervals.append(hours) + except (ValueError, TypeError): + continue + + if not intervals: + return AUTO_RESOLVE_FALLBACK_HOURS + + mean_interval = sum(intervals) / len(intervals) + threshold = mean_interval * AUTO_RESOLVE_MULTIPLIER + return max(AUTO_RESOLVE_MIN_HOURS, min(AUTO_RESOLVE_MAX_HOURS, threshold)) + + def check_auto_resolve(self, pattern: dict) -> float | None: + """Check if a pattern should be auto-resolved. + + Returns hours_silent if it should be resolved, None otherwise. + """ + last_seen = pattern.get("last_seen") + if not last_seen: + return None + + try: + last_seen_dt = datetime.fromisoformat(last_seen) + except (ValueError, TypeError): + return None + + if last_seen_dt.tzinfo is None: + last_seen_dt = last_seen_dt.replace(tzinfo=timezone.utc) + + hours_silent = ( + datetime.now(timezone.utc) - last_seen_dt + ).total_seconds() / 3600 + + threshold = self._compute_resolve_threshold( + pattern.get("rolling_counts", {}) + ) + + if hours_silent >= threshold: + return hours_silent + return None + + def _prune_rolling_counts(self, rolling_counts: dict) -> None: + """Remove rolling count entries older than ROLLING_WINDOW_DAYS.""" + from datetime import timedelta + cutoff_dt = datetime.now(timezone.utc) - timedelta(days=ROLLING_WINDOW_DAYS) + cutoff_str = cutoff_dt.strftime("%Y-%m-%dT%H:%M") + keys_to_remove = [k for k in rolling_counts if k < cutoff_str] + for k in keys_to_remove: + del rolling_counts[k] + + def create_incident( + self, + title: str, + root_cause: str | None, + severity: str, + suggested_fix: str | None, + primary_service: str, + pattern_ids: list[str], + used_llm: bool, + ) -> str: + """Create an incident grouping related patterns. Returns incident_id.""" + now = datetime.now(timezone.utc) + incident_id = f"inc_{now.strftime('%Y%m%d_%H%M')}_{uuid.uuid4().hex[:8]}" + + self.db.collection(INCIDENTS_COLLECTION).document(incident_id).set({ + "incident_id": incident_id, + "title": title[:60], + "root_cause": root_cause, + "severity": severity, + "suggested_fix": suggested_fix, + "primary_service": primary_service, + "pattern_ids": pattern_ids, + "pattern_count": len(pattern_ids), + "used_llm": used_llm, + "created_at": now.isoformat(), + "status": "open", + }) + + # Link patterns to incident + for pid in pattern_ids: + self.db.collection(PATTERNS_COLLECTION).document(pid).update({ + "incident_id": incident_id, + "severity": severity, + }) + + return incident_id + + def log_discord_alert( + self, + alert_type: str, + content: str, + success: bool, + metadata: dict | None = None, + ) -> None: + """Log a Discord alert to Firestore for audit trail.""" + now = datetime.now(timezone.utc) + alert_id = f"alert_{now.strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}" + + data = { + "timestamp": now.isoformat(), + "alert_type": alert_type, + "content": content[:2000], + "success": success, + } + if metadata: + data.update(metadata) + + self.db.collection(ALERTS_COLLECTION).document(alert_id).set(data) + + def update_pattern_alerted(self, pattern_id: str) -> None: + """Mark a pattern as having been alerted.""" + self.db.collection(PATTERNS_COLLECTION).document(pattern_id).update({ + "alerted_at": datetime.now(timezone.utc).isoformat(), + }) + + def merge_pattern( + self, source_id: str, target_id: str, reason: str + ) -> None: + """Merge a duplicate pattern into a canonical one.""" + now = datetime.now(timezone.utc).isoformat() + self.db.collection(PATTERNS_COLLECTION).document(source_id).update({ + "status": "muted", + "merged_into": target_id, + "merged_at": now, + "merged_reason": reason, + }) +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_firestore_adapter.py -v` + +Expected: All tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/services/error_monitor/firestore_adapter.py tests/unit/services/error_monitor/test_firestore_adapter.py +git commit -m "feat(error-monitor): add Firestore adapter for patterns, incidents, alerts" +``` + +--- + +## Task 5: Discord Alert Client + +**Files:** +- Create: `backend/services/error_monitor/discord.py` +- Create: `tests/unit/services/error_monitor/test_discord.py` + +- [ ] **Step 1: Write Discord client tests** + +Create `tests/unit/services/error_monitor/test_discord.py`: + +```python +"""Tests for Discord alert formatting and sending.""" + +from unittest.mock import MagicMock, patch +from datetime import datetime, timezone + +from backend.services.error_monitor.discord import ( + ErrorMonitorDiscord, + format_new_pattern_alert, + format_incident_alert, + format_auto_resolved_alert, + format_spike_alert, + format_daily_digest, +) + + +@pytest.fixture +def discord_client(): + return ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + + +class TestFormatNewPatternAlert: + def test_includes_service_and_message(self): + msg = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Firestore DEADLINE_EXCEEDED in /jobs/", + sample_message="Firestore DEADLINE_EXCEEDED in /jobs/abc123", + count=3, + first_seen="2026-04-12T10:00:00Z", + ) + assert "karaoke-backend" in msg + assert "Firestore DEADLINE_EXCEEDED" in msg + assert "🔴" in msg + + def test_includes_investigate_prompt(self): + msg = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Error X", + sample_message="Error X detail", + count=1, + first_seen="2026-04-12T10:00:00Z", + ) + assert "/prod-investigate" in msg + + def test_truncates_to_discord_limit(self): + long_message = "x" * 3000 + msg = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message=long_message, + sample_message=long_message, + count=1, + first_seen="2026-04-12T10:00:00Z", + ) + assert len(msg) <= 2000 + + +class TestFormatIncidentAlert: + def test_p0_uses_siren_emoji(self): + msg = format_incident_alert( + title="Backend completely down", + severity="P0", + root_cause="Firestore unavailable", + suggested_fix="Check Firestore status page", + primary_service="karaoke-backend", + patterns=[ + {"service": "karaoke-backend", "normalized_message": "Error A", "count": 10}, + {"service": "video-encoding-job", "normalized_message": "Error B", "count": 5}, + ], + ) + assert "🚨" in msg + + def test_p3_uses_yellow_emoji(self): + msg = format_incident_alert( + title="Minor cleanup issue", + severity="P3", + root_cause="Stale temp files", + suggested_fix="Add cleanup cron", + primary_service="backup_to_aws", + patterns=[ + {"service": "backup_to_aws", "normalized_message": "Cleanup failed", "count": 1}, + ], + ) + assert "🟡" in msg + + +class TestFormatAutoResolvedAlert: + def test_consolidated_format(self): + msg = format_auto_resolved_alert( + resolved_patterns=[ + {"service": "karaoke-backend", "normalized_message": "Error A", "hours_silent": 24.5}, + {"service": "karaoke-decide", "normalized_message": "Error B", "hours_silent": 48.0}, + ], + ) + assert "✅" in msg + assert "karaoke-backend" in msg + assert "karaoke-decide" in msg + + +class TestFormatSpikeAlert: + def test_includes_current_vs_normal(self): + msg = format_spike_alert( + service="karaoke-backend", + normalized_message="Timeout on /api/jobs", + current_count=50, + rolling_average=5.0, + ) + assert "⚠️" in msg + assert "50" in msg + assert "5" in msg + + +class TestFormatDailyDigest: + def test_includes_totals(self): + msg = format_daily_digest( + total_errors=150, + new_patterns=3, + resolved_patterns=2, + active_patterns=12, + per_service_counts={"karaoke-backend": 80, "karaoke-decide": 40, "audio-separator": 30}, + ) + assert "📊" in msg + assert "150" in msg + assert "karaoke-backend" in msg + + +import pytest + + +class TestSendMessage: + @patch("backend.services.error_monitor.discord.requests.post") + def test_sends_webhook_message(self, mock_post, discord_client): + mock_post.return_value.status_code = 204 + mock_post.return_value.ok = True + result = discord_client.send_message("Test message") + assert result is True + mock_post.assert_called_once() + + @patch("backend.services.error_monitor.discord.requests.post") + def test_returns_false_on_failure(self, mock_post, discord_client): + mock_post.return_value.status_code = 429 + mock_post.return_value.ok = False + mock_post.return_value.text = "Rate limited" + result = discord_client.send_message("Test message") + assert result is False + + def test_tracks_message_count(self, discord_client): + assert discord_client.messages_sent == 0 + + @patch("backend.services.error_monitor.discord.requests.post") + def test_respects_rate_limit(self, mock_post, discord_client): + mock_post.return_value.ok = True + discord_client.messages_sent = 10 # At limit + result = discord_client.send_message("Should not send") + assert result is False + mock_post.assert_not_called() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_discord.py -v` + +Expected: ImportError. + +- [ ] **Step 3: Implement Discord client** + +Create `backend/services/error_monitor/discord.py`: + +```python +"""Discord webhook client for error monitor alerts.""" + +import logging +from datetime import datetime, timezone + +import requests + +from backend.services.error_monitor.config import ( + DISCORD_MAX_MESSAGE_LENGTH, + MAX_DISCORD_MESSAGES_PER_RUN, +) + +logger = logging.getLogger(__name__) + +SEVERITY_EMOJI = { + "P0": "🚨", + "P1": "🔴", + "P2": "⚠️", + "P3": "🟡", +} + + +def _now_eastern() -> str: + """Current time formatted in Eastern time for Discord messages.""" + from zoneinfo import ZoneInfo + now = datetime.now(ZoneInfo("America/New_York")) + return now.strftime("%I:%M %p ET") + + +def _truncate(msg: str) -> str: + """Truncate message to Discord's limit.""" + if len(msg) <= DISCORD_MAX_MESSAGE_LENGTH: + return msg + return msg[: DISCORD_MAX_MESSAGE_LENGTH - 3] + "..." + + +def format_new_pattern_alert( + service: str, + resource_type: str, + normalized_message: str, + sample_message: str, + count: int, + first_seen: str, +) -> str: + """Format a new error pattern alert.""" + resource_label = resource_type.replace("cloud_run_", "").replace("_", " ") + msg = ( + f"🔴 **New error pattern** in `{service}` ({resource_label})\n" + f"**Pattern:** `{normalized_message[:120]}`\n" + f"**Sample:** `{sample_message[:200]}`\n" + f"**Count:** {count} | **First seen:** {first_seen[:16]} | {_now_eastern()}\n" + f"_Investigate:_ `/prod-investigate '{service} {normalized_message[:60]}'`" + ) + return _truncate(msg) + + +def format_incident_alert( + title: str, + severity: str, + root_cause: str | None, + suggested_fix: str | None, + primary_service: str, + patterns: list[dict], +) -> str: + """Format an incident alert (2+ patterns grouped by root cause).""" + emoji = SEVERITY_EMOJI.get(severity, "⚠️") + lines = [ + f"{emoji} **[{severity}] {title}**", + f"**Root cause:** {root_cause or 'Unknown'} (primary: `{primary_service}`)", + ] + if suggested_fix: + lines.append(f"**Fix:** {suggested_fix}") + + for p in patterns[:5]: # Max 5 patterns shown + lines.append( + f" → `{p['service']}`: `{p['normalized_message'][:80]}` ({p.get('count', '?')}x)" + ) + + if len(patterns) > 5: + lines.append(f" _...and {len(patterns) - 5} more patterns_") + + lines.append(f"_{_now_eastern()}_") + return _truncate("\n".join(lines)) + + +def format_auto_resolved_alert(resolved_patterns: list[dict]) -> str: + """Format a consolidated auto-resolved alert.""" + lines = [f"✅ **{len(resolved_patterns)} pattern(s) auto-resolved**"] + for p in resolved_patterns[:8]: + hours = p.get("hours_silent", 0) + lines.append( + f" → `{p['service']}`: `{p['normalized_message'][:60]}` (silent {hours:.0f}h)" + ) + if len(resolved_patterns) > 8: + lines.append(f" _...and {len(resolved_patterns) - 8} more_") + lines.append(f"_{_now_eastern()}_") + return _truncate("\n".join(lines)) + + +def format_spike_alert( + service: str, + normalized_message: str, + current_count: int, + rolling_average: float, +) -> str: + """Format a spike detection alert.""" + msg = ( + f"⚠️ **Error spike** in `{service}`\n" + f"**Pattern:** `{normalized_message[:120]}`\n" + f"**Current:** {current_count} (normal avg: {rolling_average:.1f}) — " + f"{current_count / rolling_average:.0f}x increase\n" + f"_{_now_eastern()}_" + ) + return _truncate(msg) + + +def format_daily_digest( + total_errors: int, + new_patterns: int, + resolved_patterns: int, + active_patterns: int, + per_service_counts: dict[str, int], +) -> str: + """Format the daily digest summary.""" + lines = [ + f"📊 **Daily Error Digest** — {datetime.now(timezone.utc).strftime('%Y-%m-%d')}", + f"**24h total:** {total_errors} errors | **Active patterns:** {active_patterns} | " + f"**New:** {new_patterns} | **Resolved:** {resolved_patterns}", + "", + "**By service:**", + ] + for service, count in sorted(per_service_counts.items(), key=lambda x: -x[1]): + lines.append(f" `{service}`: {count}") + lines.append(f"\n_{_now_eastern()}_") + return _truncate("\n".join(lines)) + + +class ErrorMonitorDiscord: + """Discord webhook client with rate limiting and audit logging.""" + + def __init__(self, webhook_url: str): + self.webhook_url = webhook_url + self.messages_sent = 0 + + def send_message(self, content: str) -> bool: + """Send a message to Discord. Returns True on success.""" + if self.messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN: + logger.warning("Discord rate limit reached (%d messages), skipping", self.messages_sent) + return False + + try: + response = requests.post( + self.webhook_url, + json={"content": content}, + timeout=30, + ) + self.messages_sent += 1 + if response.ok: + return True + else: + logger.error( + "Discord webhook failed: %d %s", response.status_code, response.text[:200] + ) + return False + except requests.RequestException as e: + logger.error("Discord webhook error: %s", e) + self.messages_sent += 1 + return False +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_discord.py -v` + +Expected: All tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/services/error_monitor/discord.py tests/unit/services/error_monitor/test_discord.py +git commit -m "feat(error-monitor): add Discord alert client with formatting" +``` + +--- + +## Task 6: LLM Analysis (Gemini) + +**Files:** +- Create: `backend/services/error_monitor/llm_analysis.py` +- Create: `tests/unit/services/error_monitor/test_llm_analysis.py` + +- [ ] **Step 1: Write LLM analysis tests** + +Create `tests/unit/services/error_monitor/test_llm_analysis.py`: + +```python +"""Tests for LLM incident analysis and dedup.""" + +import json +from unittest.mock import MagicMock, patch, AsyncMock + +import pytest + +from backend.services.error_monitor.llm_analysis import ( + analyze_patterns, + find_duplicate_patterns, + IncidentAnalysis, + Incident, + DuplicateGroup, + _parse_llm_response, +) + + +class TestParseResponse: + def test_parses_valid_json(self): + response = json.dumps({ + "incidents": [ + { + "title": "Audio separation failures", + "root_cause": "GPU OOM", + "severity": "P1", + "suggested_fix": "Increase memory", + "primary_service": "audio-separator", + "pattern_indices": [0, 1], + } + ] + }) + result = _parse_llm_response(response) + assert result is not None + assert len(result.incidents) == 1 + assert result.incidents[0].title == "Audio separation failures" + assert result.incidents[0].severity == "P1" + + def test_returns_none_for_invalid_json(self): + assert _parse_llm_response("not json") is None + + def test_returns_none_for_missing_incidents(self): + assert _parse_llm_response('{"other": "data"}') is None + + def test_handles_markdown_code_block(self): + response = '```json\n{"incidents": [{"title": "Test", "root_cause": null, "severity": "P3", "suggested_fix": null, "primary_service": "backend", "pattern_indices": [0]}]}\n```' + result = _parse_llm_response(response) + assert result is not None + assert len(result.incidents) == 1 + + +class TestAnalyzePatterns: + @patch("backend.services.error_monitor.llm_analysis._call_llm") + def test_groups_patterns_into_incidents(self, mock_llm): + mock_llm.return_value = json.dumps({ + "incidents": [ + { + "title": "Firestore connectivity", + "root_cause": "Firestore overloaded", + "severity": "P1", + "suggested_fix": "Check Firestore dashboard", + "primary_service": "karaoke-backend", + "pattern_indices": [0, 1], + } + ] + }) + + patterns = [ + {"service": "karaoke-backend", "normalized_message": "Firestore DEADLINE_EXCEEDED"}, + {"service": "video-encoding-job", "normalized_message": "Firestore unavailable"}, + ] + + result = analyze_patterns(patterns) + assert result is not None + assert len(result.incidents) == 1 + assert result.incidents[0].pattern_indices == [0, 1] + + @patch("backend.services.error_monitor.llm_analysis._call_llm") + def test_falls_back_on_llm_failure(self, mock_llm): + mock_llm.side_effect = Exception("LLM unavailable") + patterns = [ + {"service": "karaoke-backend", "normalized_message": "Error A"}, + {"service": "karaoke-decide", "normalized_message": "Error B"}, + ] + result = analyze_patterns(patterns) + # Fallback: group by service + assert result is not None + assert len(result.incidents) == 2 # One per service + + @patch("backend.services.error_monitor.llm_analysis._call_llm") + def test_skips_analysis_for_single_pattern(self, mock_llm): + patterns = [ + {"service": "karaoke-backend", "normalized_message": "Error A"}, + ] + result = analyze_patterns(patterns) + assert result is None # Not enough patterns + mock_llm.assert_not_called() + + +class TestFindDuplicates: + @patch("backend.services.error_monitor.llm_analysis._call_llm") + def test_identifies_near_duplicates(self, mock_llm): + mock_llm.return_value = json.dumps({ + "duplicates": [ + { + "canonical_index": 0, + "duplicate_indices": [1], + "reason": "Same Firestore timeout, different phrasing", + } + ] + }) + new_patterns = [ + {"pattern_id": "new1", "normalized_message": "Firestore timeout on read"}, + ] + existing_patterns = [ + {"pattern_id": "existing1", "normalized_message": "Firestore read timed out"}, + ] + result = find_duplicate_patterns(new_patterns, existing_patterns) + assert len(result) == 1 + assert result[0].canonical_index == 0 + + @patch("backend.services.error_monitor.llm_analysis._call_llm") + def test_returns_empty_on_no_duplicates(self, mock_llm): + mock_llm.return_value = json.dumps({"duplicates": []}) + result = find_duplicate_patterns( + [{"pattern_id": "a", "normalized_message": "Error X"}], + [{"pattern_id": "b", "normalized_message": "Completely different"}], + ) + assert result == [] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_llm_analysis.py -v` + +Expected: ImportError. + +- [ ] **Step 3: Implement LLM analysis module** + +Create `backend/services/error_monitor/llm_analysis.py`: + +```python +"""LLM-powered incident analysis and duplicate detection using Gemini Flash.""" + +import json +import logging +import re +import time +from dataclasses import dataclass, field + +import google.generativeai as genai + +from backend.services.error_monitor.config import ( + GCP_PROJECT, + LLM_ANALYSIS_MODEL, + LLM_VERTEX_LOCATION, + MIN_PATTERNS_FOR_ANALYSIS, + SERVICE_DEPENDENCY_MAP, +) + +logger = logging.getLogger(__name__) + +# Retry settings +MAX_RETRIES = 3 +BASE_DELAY = 2.0 + + +@dataclass +class Incident: + title: str + root_cause: str | None + severity: str + suggested_fix: str | None + primary_service: str + pattern_indices: list[int] + + +@dataclass +class IncidentAnalysis: + incidents: list[Incident] + used_llm: bool = True + + +@dataclass +class DuplicateGroup: + canonical_index: int + duplicate_indices: list[int] + reason: str + + +def _call_llm( + system_prompt: str, user_prompt: str, temperature: float = 0.2 +) -> str: + """Call Gemini Flash via Vertex AI. Returns raw response text.""" + model = genai.GenerativeModel( + model_name=LLM_ANALYSIS_MODEL, + system_instruction=system_prompt, + generation_config=genai.GenerationConfig(temperature=temperature), + ) + + for attempt in range(MAX_RETRIES): + try: + response = model.generate_content(user_prompt) + return response.text + except Exception as e: + if attempt < MAX_RETRIES - 1: + delay = BASE_DELAY * (2**attempt) + logger.warning( + "LLM call attempt %d failed: %s, retrying in %.1fs", + attempt + 1, e, delay, + ) + time.sleep(delay) + else: + raise + + +def _parse_llm_response(text: str) -> IncidentAnalysis | None: + """Parse LLM JSON response into IncidentAnalysis.""" + # Strip markdown code blocks if present + cleaned = re.sub(r"```(?:json)?\s*", "", text).strip() + cleaned = cleaned.rstrip("`") + + try: + data = json.loads(cleaned) + except json.JSONDecodeError: + logger.error("Failed to parse LLM response as JSON: %s", text[:200]) + return None + + if "incidents" not in data: + logger.error("LLM response missing 'incidents' key") + return None + + incidents = [] + for inc in data["incidents"]: + incidents.append(Incident( + title=inc.get("title", "Unknown"), + root_cause=inc.get("root_cause"), + severity=inc.get("severity", "P3"), + suggested_fix=inc.get("suggested_fix"), + primary_service=inc.get("primary_service", "unknown"), + pattern_indices=inc.get("pattern_indices", []), + )) + + return IncidentAnalysis(incidents=incidents, used_llm=True) + + +def _fallback_group_by_service(patterns: list[dict]) -> IncidentAnalysis: + """Fallback grouping when LLM is unavailable: one incident per service.""" + by_service: dict[str, list[int]] = {} + for i, p in enumerate(patterns): + svc = p.get("service", "unknown") + by_service.setdefault(svc, []).append(i) + + incidents = [] + for service, indices in by_service.items(): + incidents.append(Incident( + title=f"Errors in {service}", + root_cause=None, + severity="P2", + suggested_fix=None, + primary_service=service, + pattern_indices=indices, + )) + + return IncidentAnalysis(incidents=incidents, used_llm=False) + + +def analyze_patterns(patterns: list[dict]) -> IncidentAnalysis | None: + """Analyze 2+ new patterns to identify root causes and group into incidents. + + Returns None if fewer than MIN_PATTERNS_FOR_ANALYSIS patterns. + Falls back to service-based grouping if LLM fails. + """ + if len(patterns) < MIN_PATTERNS_FOR_ANALYSIS: + return None + + system_prompt = f"""You are a production error analyst for Nomad Karaoke, a karaoke video generation platform. + +Service dependency map: +{SERVICE_DEPENDENCY_MAP} + +Analyze the error patterns below. Group related errors that likely share a root cause into incidents. + +Respond with JSON only: +{{ + "incidents": [ + {{ + "title": "max 60 chars", + "root_cause": "one-liner explanation", + "severity": "P0|P1|P2|P3", + "suggested_fix": "actionable, 1-2 sentences", + "primary_service": "upstream root cause service name", + "pattern_indices": [0, 2] + }} + ] +}} + +Severity: +- P0: Service completely down, all jobs failing, payment processing broken +- P1: Major pipeline stage broken (audio separation, lyrics, encoding), auth failures +- P2: Degraded but functional — intermittent errors, slow responses +- P3: Minor — deprecation warnings, cleanup failures, spot VM preemptions + +Every pattern must appear in exactly one incident's pattern_indices.""" + + patterns_text = "\n".join( + f"[{i}] service={p['service']}, message={p.get('normalized_message', '')}, count={p.get('count', 1)}" + for i, p in enumerate(patterns) + ) + + try: + response = _call_llm(system_prompt, patterns_text) + result = _parse_llm_response(response) + if result: + return result + logger.warning("Could not parse LLM response, falling back to service grouping") + except Exception as e: + logger.error("LLM analysis failed: %s, falling back to service grouping", e) + + return _fallback_group_by_service(patterns) + + +def find_duplicate_patterns( + new_patterns: list[dict], + existing_patterns: list[dict], +) -> list[DuplicateGroup]: + """Identify near-duplicate patterns that regex normalization missed. + + Compares new patterns against existing canonical patterns. + Returns groups where new patterns should be merged into existing ones. + """ + if not new_patterns or not existing_patterns: + return [] + + system_prompt = """You are checking for near-duplicate error patterns. Two patterns are duplicates if they describe the same error with slightly different phrasing. + +Only merge a NEW pattern into an EXISTING one — never merge two existing patterns. + +Respond with JSON only: +{ + "duplicates": [ + { + "canonical_index": 0, + "duplicate_indices": [1], + "reason": "why they're the same error" + } + ] +} + +If no duplicates, return {"duplicates": []}.""" + + text = "EXISTING patterns (canonical):\n" + for i, p in enumerate(existing_patterns): + text += f" [{i}] {p.get('service', '?')}: {p.get('normalized_message', '')}\n" + text += "\nNEW patterns (check for duplicates):\n" + for i, p in enumerate(new_patterns): + text += f" [{i}] {p.get('service', '?')}: {p.get('normalized_message', '')}\n" + + try: + response = _call_llm(system_prompt, text, temperature=0.1) + cleaned = re.sub(r"```(?:json)?\s*", "", response).strip().rstrip("`") + data = json.loads(cleaned) + groups = [] + for d in data.get("duplicates", []): + groups.append(DuplicateGroup( + canonical_index=d["canonical_index"], + duplicate_indices=d["duplicate_indices"], + reason=d["reason"], + )) + return groups + except Exception as e: + logger.error("Duplicate detection failed: %s", e) + return [] +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_llm_analysis.py -v` + +Expected: All tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/services/error_monitor/llm_analysis.py tests/unit/services/error_monitor/test_llm_analysis.py +git commit -m "feat(error-monitor): add LLM incident analysis and dedup via Gemini" +``` + +--- + +## Task 7: Monitor Orchestrator (Main Entry Point) + +**Files:** +- Create: `backend/services/error_monitor/monitor.py` +- Create: `tests/unit/services/error_monitor/test_monitor.py` + +- [ ] **Step 1: Write monitor orchestrator tests** + +Create `tests/unit/services/error_monitor/test_monitor.py`: + +```python +"""Tests for the error monitor orchestrator.""" + +import json +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch, call + +import pytest + +from backend.services.error_monitor.monitor import ( + ErrorMonitor, + _build_log_filter, + _extract_message, + _extract_service_name, + _classify_resource_type, +) + + +class TestBuildLogFilter: + def test_cloud_run_service_filter(self): + f = _build_log_filter( + resource_type="cloud_run_revision", + resource_names=["karaoke-backend", "karaoke-decide"], + lookback_minutes=15, + ) + assert 'resource.type="cloud_run_revision"' in f + assert "severity>=ERROR" in f + assert "karaoke-backend" in f + + def test_cloud_run_job_filter(self): + f = _build_log_filter( + resource_type="cloud_run_job", + resource_names=["video-encoding-job"], + lookback_minutes=15, + ) + assert 'resource.type="cloud_run_job"' in f + assert "video-encoding-job" in f + + def test_gce_instance_filter(self): + f = _build_log_filter( + resource_type="gce_instance", + resource_names=["encoding-worker-a"], + lookback_minutes=15, + ) + assert 'resource.type="gce_instance"' in f + + +class TestExtractMessage: + def test_extracts_text_payload(self): + entry = MagicMock() + entry.payload = "Simple error text" + entry.payload_type = "text_payload" + assert _extract_message(entry) == "Simple error text" + + def test_extracts_json_message(self): + entry = MagicMock() + entry.payload = {"message": "JSON error message"} + entry.payload_type = "json_payload" + assert _extract_message(entry) == "JSON error message" + + def test_extracts_json_textpayload_field(self): + entry = MagicMock() + entry.payload = {"textPayload": "Text from JSON"} + entry.payload_type = "json_payload" + assert _extract_message(entry) == "Text from JSON" + + def test_returns_none_for_empty(self): + entry = MagicMock() + entry.payload = None + entry.payload_type = "proto_payload" + assert _extract_message(entry) is None + + +class TestClassifyResourceType: + def test_cloud_run_service(self): + assert _classify_resource_type("cloud_run_revision") == "cloud_run_service" + + def test_cloud_run_job(self): + assert _classify_resource_type("cloud_run_job") == "cloud_run_job" + + def test_cloud_function(self): + assert _classify_resource_type("cloud_function") == "cloud_function" + + def test_gce_instance(self): + assert _classify_resource_type("gce_instance") == "gce_instance" + + +class TestExtractServiceName: + def test_cloud_run_service(self): + entry = MagicMock() + entry.resource.labels = {"service_name": "karaoke-backend"} + assert _extract_service_name(entry, "cloud_run_revision") == "karaoke-backend" + + def test_cloud_run_job(self): + entry = MagicMock() + entry.resource.labels = {"job_name": "video-encoding-job"} + assert _extract_service_name(entry, "cloud_run_job") == "video-encoding-job" + + def test_cloud_function(self): + entry = MagicMock() + entry.resource.labels = {"function_name": "backup_to_aws"} + assert _extract_service_name(entry, "cloud_function") == "backup_to_aws" + + def test_returns_unknown_for_missing_labels(self): + entry = MagicMock() + entry.resource.labels = {} + assert _extract_service_name(entry, "cloud_run_revision") == "unknown" + + +class TestMonitorPipeline: + @pytest.fixture + def monitor(self): + m = ErrorMonitor.__new__(ErrorMonitor) + m.logging_client = MagicMock() + m.firestore_adapter = MagicMock() + m.discord = MagicMock() + m.discord.messages_sent = 0 + m.logger = MagicMock() + return m + + def test_groups_entries_by_pattern(self, monitor): + entries = [ + {"service": "karaoke-backend", "normalized": "Error A", "raw": "Error A detail", "resource_type": "cloud_run_service", "timestamp": datetime.now(timezone.utc)}, + {"service": "karaoke-backend", "normalized": "Error A", "raw": "Error A detail 2", "resource_type": "cloud_run_service", "timestamp": datetime.now(timezone.utc)}, + {"service": "karaoke-decide", "normalized": "Error B", "raw": "Error B detail", "resource_type": "cloud_run_service", "timestamp": datetime.now(timezone.utc)}, + ] + groups = monitor._group_by_pattern(entries) + assert len(groups) == 2 # Two distinct patterns + + def test_detects_spike(self, monitor): + pattern = { + "rolling_counts": { + "2026-04-12T09:00": 2, + "2026-04-12T09:15": 3, + "2026-04-12T09:30": 2, + "2026-04-12T09:45": 1, + } + } + # Normal avg ~2, current count 15 → spike (15/2 = 7.5x > 5x) + assert monitor._is_spike(pattern, current_count=15) is True + # Normal avg ~2, current count 5 → not spike (5/2 = 2.5x < 5x) + assert monitor._is_spike(pattern, current_count=5) is False + + def test_rolling_average(self, monitor): + assert monitor._rolling_average({"a": 2, "b": 4, "c": 6}) == 4.0 + assert monitor._rolling_average({}) == 0.0 + + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + def test_run_cycle_no_errors(self, mock_llm, mock_digest, monitor): + """End-to-end: no errors found, still checks auto-resolve.""" + monitor._collect_logs = MagicMock(return_value=[]) + monitor._check_auto_resolve = MagicMock() + monitor._run_monitor_cycle() + monitor._check_auto_resolve.assert_called_once() + + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + def test_run_cycle_with_new_pattern(self, mock_llm, mock_digest, monitor): + """End-to-end: one new error triggers individual alert.""" + from backend.services.error_monitor.firestore_adapter import UpsertResult + monitor._collect_logs = MagicMock(return_value=[ + {"service": "karaoke-backend", "resource_type": "cloud_run_service", + "raw": "Firestore timeout", "normalized": "Firestore timeout", + "timestamp": datetime.now(timezone.utc)}, + ]) + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="hash1", is_new=True + ) + monitor.firestore_adapter.get_pattern.return_value = None + monitor._check_auto_resolve = MagicMock() + + monitor._run_monitor_cycle() + + monitor.discord.send_message.assert_called_once() # One alert sent + monitor.firestore_adapter.update_pattern_alerted.assert_called_once() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_monitor.py -v` + +Expected: ImportError. + +- [ ] **Step 3: Implement monitor orchestrator** + +Create `backend/services/error_monitor/monitor.py`: + +```python +"""Error monitor orchestrator — main entry point. + +Queries Cloud Logging, normalizes, deduplicates, analyzes, and alerts. +Run as: python -m backend.services.error_monitor.monitor +""" + +import logging +import os +import sys +from datetime import datetime, timezone, timedelta + +from google.cloud import logging as cloud_logging +from google.cloud import secretmanager + +from backend.services.error_monitor.config import ( + GCP_PROJECT, + LOOKBACK_MINUTES, + MAX_LOG_ENTRIES, + MONITORED_CLOUD_FUNCTIONS, + MONITORED_CLOUD_RUN_JOBS, + MONITORED_CLOUD_RUN_SERVICES, + MONITORED_GCE_INSTANCES, + SPIKE_MIN_COUNT, + SPIKE_MULTIPLIER, + get_digest_mode, + get_discord_webhook_secret_name, + get_llm_enabled, +) +from backend.services.error_monitor.discord import ( + ErrorMonitorDiscord, + format_auto_resolved_alert, + format_daily_digest, + format_incident_alert, + format_new_pattern_alert, + format_spike_alert, +) +from backend.services.error_monitor.firestore_adapter import ( + ErrorPatternsAdapter, + PatternData, +) +from backend.services.error_monitor.known_issues import should_ignore +from backend.services.error_monitor.llm_analysis import analyze_patterns, find_duplicate_patterns +from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message + +logger = logging.getLogger(__name__) + +# Maps Cloud Logging resource types to our filter fields +RESOURCE_TYPE_FILTERS = { + "cloud_run_revision": ("resource.labels.service_name", MONITORED_CLOUD_RUN_SERVICES), + "cloud_run_job": ("resource.labels.job_name", MONITORED_CLOUD_RUN_JOBS), + "cloud_function": ("resource.labels.function_name", MONITORED_CLOUD_FUNCTIONS), + "gce_instance": ('labels."compute.googleapis.com/resource_name"', MONITORED_GCE_INSTANCES), +} + +RESOURCE_TYPE_MAP = { + "cloud_run_revision": "cloud_run_service", + "cloud_run_job": "cloud_run_job", + "cloud_function": "cloud_function", + "gce_instance": "gce_instance", +} + + +def _build_log_filter( + resource_type: str, resource_names: list[str], lookback_minutes: int +) -> str: + """Build a Cloud Logging filter string.""" + cutoff = datetime.now(timezone.utc) - timedelta(minutes=lookback_minutes) + cutoff_str = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ") + + filter_field, _ = RESOURCE_TYPE_FILTERS.get(resource_type, ("", [])) + names_filter = " OR ".join(f'{filter_field}="{name}"' for name in resource_names) + + return ( + f'resource.type="{resource_type}" ' + f"severity>=ERROR " + f'timestamp>="{cutoff_str}" ' + f"({names_filter})" + ) + + +def _extract_message(entry) -> str | None: + """Extract the error message from a Cloud Logging entry.""" + payload = entry.payload + if payload is None: + return None + + if isinstance(payload, str): + return payload if payload.strip() else None + + if isinstance(payload, dict): + # Try common fields + for key in ("message", "textPayload", "error", "msg"): + if key in payload and payload[key]: + return str(payload[key]) + # Fallback to string representation + return str(payload)[:500] + + return None + + +def _extract_service_name(entry, resource_type: str) -> str: + """Extract the service/resource name from a log entry.""" + labels = entry.resource.labels or {} + if resource_type == "cloud_run_revision": + return labels.get("service_name", "unknown") + elif resource_type == "cloud_run_job": + return labels.get("job_name", "unknown") + elif resource_type == "cloud_function": + return labels.get("function_name", "unknown") + elif resource_type == "gce_instance": + # Try instance name from labels + entry_labels = getattr(entry, "labels", {}) or {} + return entry_labels.get("compute.googleapis.com/resource_name", labels.get("instance_id", "unknown")) + return "unknown" + + +def _classify_resource_type(log_resource_type: str) -> str: + """Map Cloud Logging resource type to our resource type.""" + return RESOURCE_TYPE_MAP.get(log_resource_type, "unknown") + + +class ErrorMonitor: + """Main error monitor orchestrator.""" + + def __init__(self): + self.logging_client = cloud_logging.Client(project=GCP_PROJECT) + self.firestore_adapter = ErrorPatternsAdapter() + self.logger = logger + + # Get Discord webhook URL from Secret Manager + webhook_url = self._get_discord_webhook() + self.discord = ErrorMonitorDiscord(webhook_url=webhook_url) + + def _get_discord_webhook(self) -> str: + """Retrieve Discord webhook URL from Secret Manager.""" + # Check env var first (injected by Cloud Run Job) + url = os.environ.get("DISCORD_WEBHOOK_URL") + if url: + return url.strip() + + # Fall back to Secret Manager + client = secretmanager.SecretManagerServiceClient() + secret_name = get_discord_webhook_secret_name() + name = f"projects/{GCP_PROJECT}/secrets/{secret_name}/versions/latest" + response = client.access_secret_version(name=name) + return response.payload.data.decode("utf-8").strip() + + def run(self) -> None: + """Run the full error monitor pipeline.""" + if get_digest_mode(): + self._run_daily_digest() + return + + self._run_monitor_cycle() + + def _run_monitor_cycle(self) -> None: + """Standard 15-minute monitor cycle.""" + logger.info("Starting error monitor cycle") + + # 1. Collect logs + entries = self._collect_logs() + logger.info("Collected %d log entries", len(entries)) + + if not entries: + # Still check for auto-resolution even with no new errors + self._check_auto_resolve() + logger.info("No errors found, cycle complete") + return + + # 2. Group by pattern + groups = self._group_by_pattern(entries) + logger.info("Grouped into %d patterns", len(groups)) + + # 3. Upsert patterns to Firestore, identify new ones + new_patterns = [] + spike_patterns = [] + for pattern_hash, group in groups.items(): + result = self.firestore_adapter.upsert_pattern(PatternData( + pattern_id=pattern_hash, + service=group["service"], + resource_type=group["resource_type"], + normalized_message=group["normalized"], + sample_message=group["sample"], + count=group["count"], + timestamp=group["latest_timestamp"], + )) + + if result.is_new: + new_patterns.append({ + "pattern_id": pattern_hash, + "service": group["service"], + "resource_type": group["resource_type"], + "normalized_message": group["normalized"], + "sample_message": group["sample"], + "count": group["count"], + "first_seen": group["earliest_timestamp"].isoformat(), + }) + + # Check for spike on existing patterns + existing = self.firestore_adapter.get_pattern(pattern_hash) + if existing and self._is_spike(existing, group["count"]): + spike_patterns.append({ + "service": group["service"], + "normalized_message": group["normalized"], + "current_count": group["count"], + "rolling_average": self._rolling_average(existing.get("rolling_counts", {})), + }) + + # 4. LLM dedup new patterns against existing + if new_patterns and get_llm_enabled(): + existing = self.firestore_adapter.get_active_patterns() + duplicates = find_duplicate_patterns(new_patterns, existing) + for dup in duplicates: + for dup_idx in dup.duplicate_indices: + if dup_idx < len(new_patterns): + merged = new_patterns[dup_idx] + canonical = existing[dup.canonical_index] if dup.canonical_index < len(existing) else None + if canonical: + self.firestore_adapter.merge_pattern( + merged["pattern_id"], + canonical["pattern_id"], + dup.reason, + ) + logger.info( + "Merged duplicate %s into %s: %s", + merged["pattern_id"][:8], + canonical["pattern_id"][:8], + dup.reason, + ) + + # Remove merged patterns from new list + merged_ids = set() + for dup in duplicates: + for idx in dup.duplicate_indices: + if idx < len(new_patterns): + merged_ids.add(new_patterns[idx]["pattern_id"]) + new_patterns = [p for p in new_patterns if p["pattern_id"] not in merged_ids] + + # 5. LLM incident analysis for new patterns + if len(new_patterns) >= 2 and get_llm_enabled(): + analysis = analyze_patterns(new_patterns) + if analysis: + for incident in analysis.incidents: + pattern_ids = [ + new_patterns[i]["pattern_id"] + for i in incident.pattern_indices + if i < len(new_patterns) + ] + incident_id = self.firestore_adapter.create_incident( + title=incident.title, + root_cause=incident.root_cause, + severity=incident.severity, + suggested_fix=incident.suggested_fix, + primary_service=incident.primary_service, + pattern_ids=pattern_ids, + used_llm=analysis.used_llm, + ) + + # Send incident alert + incident_patterns = [ + new_patterns[i] for i in incident.pattern_indices + if i < len(new_patterns) + ] + msg = format_incident_alert( + title=incident.title, + severity=incident.severity, + root_cause=incident.root_cause, + suggested_fix=incident.suggested_fix, + primary_service=incident.primary_service, + patterns=incident_patterns, + ) + success = self.discord.send_message(msg) + self.firestore_adapter.log_discord_alert( + alert_type="incident", + content=msg, + success=success, + metadata={"incident_id": incident_id, "severity": incident.severity}, + ) + for pid in pattern_ids: + self.firestore_adapter.update_pattern_alerted(pid) + else: + # Send individual alerts for new patterns + for pattern in new_patterns: + msg = format_new_pattern_alert( + service=pattern["service"], + resource_type=pattern["resource_type"], + normalized_message=pattern["normalized_message"], + sample_message=pattern["sample_message"], + count=pattern["count"], + first_seen=pattern["first_seen"], + ) + success = self.discord.send_message(msg) + self.firestore_adapter.log_discord_alert( + alert_type="new_pattern", + content=msg, + success=success, + metadata={"service": pattern["service"], "pattern_id": pattern["pattern_id"]}, + ) + self.firestore_adapter.update_pattern_alerted(pattern["pattern_id"]) + + # 6. Send spike alerts + for spike in spike_patterns: + msg = format_spike_alert( + service=spike["service"], + normalized_message=spike["normalized_message"], + current_count=spike["current_count"], + rolling_average=spike["rolling_average"], + ) + success = self.discord.send_message(msg) + self.firestore_adapter.log_discord_alert( + alert_type="spike", + content=msg, + success=success, + metadata={"service": spike["service"]}, + ) + + # 7. Check for auto-resolution + self._check_auto_resolve() + + logger.info( + "Cycle complete: %d entries, %d patterns, %d new, %d spikes", + len(entries), len(groups), len(new_patterns), len(spike_patterns), + ) + + def _collect_logs(self) -> list[dict]: + """Query Cloud Logging for errors across all resource types.""" + all_entries = [] + + for resource_type, (filter_field, names) in RESOURCE_TYPE_FILTERS.items(): + if not names: + continue + + log_filter = _build_log_filter(resource_type, names, LOOKBACK_MINUTES) + try: + entries = list( + self.logging_client.list_entries( + filter_=log_filter, + max_results=MAX_LOG_ENTRIES // len(RESOURCE_TYPE_FILTERS), + project_ids=[GCP_PROJECT], + ) + ) + except Exception as e: + logger.error("Failed to query %s logs: %s", resource_type, e) + continue + + for entry in entries: + message = _extract_message(entry) + if not message: + continue + + service = _extract_service_name(entry, resource_type) + our_resource_type = _classify_resource_type(resource_type) + + # Check ignore patterns + if should_ignore(service, message): + continue + + normalized = normalize_message(message) + all_entries.append({ + "service": service, + "resource_type": our_resource_type, + "raw": message, + "normalized": normalized, + "timestamp": entry.timestamp or datetime.now(timezone.utc), + }) + + return all_entries[:MAX_LOG_ENTRIES] + + def _group_by_pattern(self, entries: list[dict]) -> dict: + """Group log entries by normalized pattern hash.""" + groups = {} + for entry in entries: + pattern_hash = compute_pattern_hash(entry["service"], entry["normalized"]) + if pattern_hash not in groups: + groups[pattern_hash] = { + "service": entry["service"], + "resource_type": entry["resource_type"], + "normalized": entry["normalized"], + "sample": entry["raw"], + "count": 0, + "earliest_timestamp": entry["timestamp"], + "latest_timestamp": entry["timestamp"], + } + groups[pattern_hash]["count"] += 1 + if entry["timestamp"] < groups[pattern_hash]["earliest_timestamp"]: + groups[pattern_hash]["earliest_timestamp"] = entry["timestamp"] + if entry["timestamp"] > groups[pattern_hash]["latest_timestamp"]: + groups[pattern_hash]["latest_timestamp"] = entry["timestamp"] + + return groups + + def _is_spike(self, pattern: dict, current_count: int) -> bool: + """Check if current count represents a spike vs. rolling average.""" + if current_count < SPIKE_MIN_COUNT: + return False + avg = self._rolling_average(pattern.get("rolling_counts", {})) + if avg <= 0: + return False + return current_count / avg >= SPIKE_MULTIPLIER + + def _rolling_average(self, rolling_counts: dict) -> float: + """Compute rolling average from rolling_counts map.""" + if not rolling_counts: + return 0.0 + values = list(rolling_counts.values()) + return sum(values) / len(values) + + def _check_auto_resolve(self) -> None: + """Check all active patterns for auto-resolution.""" + patterns = self.firestore_adapter.get_patterns_for_auto_resolve() + resolved = [] + + for pattern in patterns: + hours_silent = self.firestore_adapter.check_auto_resolve(pattern) + if hours_silent is not None: + self.firestore_adapter.auto_resolve_pattern( + pattern["pattern_id"], hours_silent + ) + resolved.append({ + "service": pattern.get("service", "unknown"), + "normalized_message": pattern.get("normalized_message", ""), + "hours_silent": hours_silent, + }) + + if resolved: + msg = format_auto_resolved_alert(resolved) + success = self.discord.send_message(msg) + self.firestore_adapter.log_discord_alert( + alert_type="resolved_batch", + content=msg, + success=success, + metadata={"count": len(resolved)}, + ) + + def _run_daily_digest(self) -> None: + """Generate and send the daily digest.""" + logger.info("Running daily digest") + + # Dedup sweep + if get_llm_enabled(): + active = self.firestore_adapter.get_active_patterns() + if len(active) >= 2: + duplicates = find_duplicate_patterns(active[:50], active[:50]) + for dup in duplicates: + for idx in dup.duplicate_indices: + if idx < len(active): + canonical = active[dup.canonical_index] if dup.canonical_index < len(active) else None + if canonical: + self.firestore_adapter.merge_pattern( + active[idx]["pattern_id"], + canonical["pattern_id"], + dup.reason, + ) + + # Gather stats + active = self.firestore_adapter.get_active_patterns() + per_service: dict[str, int] = {} + total_24h = 0 + new_count = 0 + cutoff = datetime.now(timezone.utc) - timedelta(hours=24) + + for p in active: + svc = p.get("service", "unknown") + # Sum rolling counts from last 24h + for ts_key, count in p.get("rolling_counts", {}).items(): + try: + if datetime.fromisoformat(ts_key) >= cutoff: + per_service[svc] = per_service.get(svc, 0) + count + total_24h += count + except (ValueError, TypeError): + pass + if p.get("status") == "new": + first_seen = p.get("first_seen", "") + try: + if datetime.fromisoformat(first_seen) >= cutoff: + new_count += 1 + except (ValueError, TypeError): + pass + + # Count resolved in last 24h from discord_alerts + cutoff_iso = cutoff.isoformat() + resolved_docs = ( + self.db.collection("discord_alerts") + .where("alert_type", "==", "resolved_batch") + .where("timestamp", ">=", cutoff_iso) + .stream() + ) + resolved_count = sum(doc.to_dict().get("count", 0) for doc in resolved_docs) + + msg = format_daily_digest( + total_errors=total_24h, + new_patterns=new_count, + resolved_patterns=resolved_count, + active_patterns=len(active), + per_service_counts=per_service, + ) + success = self.discord.send_message(msg) + self.firestore_adapter.log_discord_alert( + alert_type="daily_digest", + content=msg, + success=success, + metadata={"total_24h": total_24h, "active": len(active)}, + ) + logger.info("Daily digest sent: %d errors, %d active patterns", total_24h, len(active)) + + +def main(): + """Entry point for Cloud Run Job.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + logger.info("Error monitor starting (project=%s, digest=%s)", GCP_PROJECT, get_digest_mode()) + + try: + monitor = ErrorMonitor() + monitor.run() + except Exception as e: + logger.exception("Error monitor failed: %s", e) + sys.exit(1) + + logger.info("Error monitor completed successfully") + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot && python -m pytest tests/unit/services/error_monitor/test_monitor.py -v` + +Expected: All tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/services/error_monitor/monitor.py tests/unit/services/error_monitor/test_monitor.py +git commit -m "feat(error-monitor): add monitor orchestrator (main entry point)" +``` + +--- + +## Task 8: Pulumi Infrastructure + +**Files:** +- Create: `infrastructure/modules/error_monitor.py` +- Modify: `infrastructure/__main__.py` +- Modify: `infrastructure/config.py` +- Modify: `infrastructure/modules/iam/backend_sa.py` + +- [ ] **Step 1: Add ErrorMonitorConfig to infrastructure config** + +Read `infrastructure/config.py` and add a new config class following existing patterns (like `RunnerManagerConfig`). + +Add to `infrastructure/config.py`: + +```python +class ErrorMonitorConfig: + """Configuration for the error monitor Cloud Run Job.""" + JOB_NAME = "nomad-error-monitor" + MEMORY = "512Mi" + CPU = "1" + TIMEOUT = "300s" + MAX_RETRIES = 0 + MONITOR_SCHEDULE = "*/15 * * * *" # Every 15 minutes UTC + DIGEST_SCHEDULE = "0 8 * * *" # 08:00 UTC daily + SCHEDULER_SA_NAME = "error-monitor-scheduler" +``` + +- [ ] **Step 2: Add logging.viewer role to backend SA** + +Read `infrastructure/modules/iam/backend_sa.py` and add: + +```python + "logging_viewer": "roles/logging.viewer", # Error monitor: read Cloud Logging +``` + +to the existing roles dict/list for the backend service account. + +- [ ] **Step 3: Create error monitor Pulumi module** + +Create `infrastructure/modules/error_monitor.py`: + +```python +"""Pulumi module for error monitor infrastructure. + +Creates: +- Cloud Run Job (nomad-error-monitor) — shares karaoke-backend image +- Cloud Scheduler (error-monitor-trigger) — every 15 minutes +- Cloud Scheduler (error-monitor-daily-digest) — 08:00 UTC daily +- Service Account (error-monitor-scheduler) — for scheduler auth +""" + +import pulumi +import pulumi_gcp as gcp + +from infrastructure.config import ( + PROJECT_ID, + REGION, + ErrorMonitorConfig as Config, +) + + +def create_error_monitor( + backend_image: pulumi.Input[str], + backend_sa_email: pulumi.Input[str], + depends_on: list | None = None, +): + """Create error monitor infrastructure. + + Args: + backend_image: Docker image URL for karaoke-backend + backend_sa_email: Backend service account email (for job execution) + depends_on: Resources that must be created first + """ + + # 1. Service Account for Cloud Scheduler + scheduler_sa = gcp.serviceaccount.Account( + "error-monitor-scheduler-sa", + account_id=Config.SCHEDULER_SA_NAME, + display_name="Error Monitor Scheduler", + project=PROJECT_ID, + ) + + # 2. Cloud Run Job + job = gcp.cloudrunv2.Job( + "error-monitor-job", + name=Config.JOB_NAME, + location=REGION, + template=gcp.cloudrunv2.JobTemplateArgs( + template=gcp.cloudrunv2.JobTemplateTemplateArgs( + service_account=backend_sa_email, + max_retries=Config.MAX_RETRIES, + timeout=Config.TIMEOUT, + containers=[ + gcp.cloudrunv2.JobTemplateTemplateContainerArgs( + image=backend_image, + args=["python", "-m", "backend.services.error_monitor.monitor"], + resources=gcp.cloudrunv2.JobTemplateTemplateContainerResourcesArgs( + limits={ + "memory": Config.MEMORY, + "cpu": Config.CPU, + }, + ), + envs=[ + gcp.cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="GCP_PROJECT_ID", + value=PROJECT_ID, + ), + gcp.cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="ENVIRONMENT", + value="production", + ), + gcp.cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="DISCORD_WEBHOOK_URL", + value_source=gcp.cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs( + secret_key_ref=gcp.cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs( + secret=f"projects/{PROJECT_ID}/secrets/discord-alert-webhook", + version="latest", + ), + ), + ), + ], + ), + ], + ), + ), + opts=pulumi.ResourceOptions(depends_on=depends_on or []), + ) + + # 3. Grant scheduler SA permission to invoke the job + gcp.cloudrunv2.JobIamMember( + "error-monitor-scheduler-invoker", + name=job.name, + location=REGION, + role="roles/run.invoker", + member=pulumi.Output.concat("serviceAccount:", scheduler_sa.email), + ) + + # 4. Cloud Scheduler — Monitor (every 15 min) + job_uri = pulumi.Output.concat( + "https://", REGION, "-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/", + PROJECT_ID, "/jobs/", Config.JOB_NAME, ":run", + ) + + gcp.cloudscheduler.Job( + "error-monitor-trigger", + name="error-monitor-trigger", + description="Trigger error monitor every 15 minutes", + region=REGION, + schedule=Config.MONITOR_SCHEDULE, + time_zone="UTC", + http_target=gcp.cloudscheduler.JobHttpTargetArgs( + uri=job_uri, + http_method="POST", + oauth_token=gcp.cloudscheduler.JobHttpTargetOauthTokenArgs( + service_account_email=scheduler_sa.email, + ), + ), + retry_config=gcp.cloudscheduler.JobRetryConfigArgs( + retry_count=1, + max_retry_duration="60s", + ), + ) + + # 5. Cloud Scheduler — Daily Digest (08:00 UTC) + gcp.cloudscheduler.Job( + "error-monitor-daily-digest", + name="error-monitor-daily-digest", + description="Trigger daily error digest at 08:00 UTC", + region=REGION, + schedule=Config.DIGEST_SCHEDULE, + time_zone="UTC", + http_target=gcp.cloudscheduler.JobHttpTargetArgs( + uri=job_uri, + http_method="POST", + body=pulumi.Output.from_input( + '{"overrides":{"containerOverrides":[{"env":[{"name":"DIGEST_MODE","value":"true"}]}]}}' + ).apply(lambda s: s.encode("utf-8")), + headers={"Content-Type": "application/json"}, + oauth_token=gcp.cloudscheduler.JobHttpTargetOauthTokenArgs( + service_account_email=scheduler_sa.email, + ), + ), + retry_config=gcp.cloudscheduler.JobRetryConfigArgs( + retry_count=1, + max_retry_duration="60s", + ), + ) + + return { + "job": job, + "scheduler_sa": scheduler_sa, + } +``` + +- [ ] **Step 4: Wire into infrastructure/__main__.py** + +Read `infrastructure/__main__.py` and add the error monitor module call. Find where other modules are invoked and add: + +```python +from infrastructure.modules.error_monitor import create_error_monitor + +# ... after backend image and SA are defined ... +error_monitor = create_error_monitor( + backend_image=backend_image_url, # Use the actual variable name from __main__.py + backend_sa_email=backend_sa.email, # Use the actual variable name +) +``` + +The exact variable names depend on what's already in `__main__.py` — read it first and match the existing patterns. + +- [ ] **Step 5: Commit** + +```bash +git add infrastructure/modules/error_monitor.py infrastructure/__main__.py infrastructure/config.py infrastructure/modules/iam/backend_sa.py +git commit -m "feat(error-monitor): add Pulumi infrastructure (Cloud Run Job + Schedulers)" +``` + +--- + +## Task 9: Helper Scripts + +**Files:** +- Create: `scripts/query-error-patterns.py` +- Create: `scripts/resolve-error-pattern.py` + +- [ ] **Step 1: Create query-error-patterns.py** + +Create `scripts/query-error-patterns.py`: + +```python +#!/usr/bin/env python3 +"""CLI to query error patterns from Firestore. + +Usage: + python scripts/query-error-patterns.py # All active + python scripts/query-error-patterns.py --status new # Only new + python scripts/query-error-patterns.py --service karaoke-backend + python scripts/query-error-patterns.py --hours 24 # Last 24h + python scripts/query-error-patterns.py --json # JSON output +""" + +import argparse +import json +import sys +from datetime import datetime, timezone, timedelta + +from google.cloud import firestore + + +def main(): + parser = argparse.ArgumentParser(description="Query error patterns from Firestore") + parser.add_argument("--status", help="Filter by status (new, acknowledged, known, muted, fixed, auto_resolved)") + parser.add_argument("--service", help="Filter by service name") + parser.add_argument("--hours", type=int, help="Only patterns seen in last N hours") + parser.add_argument("--json", action="store_true", help="Output as JSON") + parser.add_argument("--project", default="nomadkaraoke", help="GCP project ID") + args = parser.parse_args() + + db = firestore.Client(project=args.project) + query = db.collection("error_patterns") + + if args.status: + query = query.where("status", "==", args.status) + else: + # Default: active patterns only + query = query.where("status", "in", ["new", "acknowledged", "known"]) + + if args.service: + query = query.where("service", "==", args.service) + + patterns = [doc.to_dict() for doc in query.stream()] + + # Filter by time if requested + if args.hours: + cutoff = (datetime.now(timezone.utc) - timedelta(hours=args.hours)).isoformat() + patterns = [p for p in patterns if p.get("last_seen", "") >= cutoff] + + # Sort by last_seen descending + patterns.sort(key=lambda p: p.get("last_seen", ""), reverse=True) + + if args.json: + print(json.dumps(patterns, indent=2, default=str)) + return + + # Table output + if not patterns: + print("No matching patterns found.") + return + + print(f"{'Status':<14} {'Service':<25} {'Count':>6} {'Last Seen':<20} Message") + print("-" * 120) + for p in patterns: + status = p.get("status", "?") + service = p.get("service", "?")[:24] + count = p.get("total_count", 0) + last_seen = p.get("last_seen", "?")[:19] + message = p.get("normalized_message", "?")[:50] + print(f"{status:<14} {service:<25} {count:>6} {last_seen:<20} {message}") + + print(f"\nTotal: {len(patterns)} pattern(s)") + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 2: Create resolve-error-pattern.py** + +Create `scripts/resolve-error-pattern.py`: + +```python +#!/usr/bin/env python3 +"""CLI to resolve error patterns in Firestore. + +Usage: + python scripts/resolve-error-pattern.py --pr 42 --note "Fixed timeout" + python scripts/resolve-error-pattern.py --note "Added retry logic" +""" + +import argparse +import sys +from datetime import datetime, timezone + +from google.cloud import firestore + + +def find_pattern(db, prefix: str) -> dict | None: + """Find a pattern by ID prefix.""" + docs = list(db.collection("error_patterns").stream()) + matches = [ + doc.to_dict() for doc in docs + if doc.id.startswith(prefix) + ] + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + print(f"Ambiguous prefix '{prefix}', matches {len(matches)} patterns:") + for m in matches[:5]: + print(f" {m['pattern_id'][:16]} {m.get('service', '?')} {m.get('normalized_message', '?')[:60]}") + return None + return None + + +def main(): + parser = argparse.ArgumentParser(description="Resolve an error pattern") + parser.add_argument("pattern_prefix", help="Pattern ID or prefix (at least 8 chars)") + parser.add_argument("--pr", type=int, help="PR number that fixes this") + parser.add_argument("--note", required=True, help="Description of the fix") + parser.add_argument("--repo", default="nomadkaraoke/karaoke-gen", help="GitHub repo for PR URL") + parser.add_argument("--project", default="nomadkaraoke", help="GCP project ID") + args = parser.parse_args() + + db = firestore.Client(project=args.project) + pattern = find_pattern(db, args.pattern_prefix) + + if not pattern: + print(f"Pattern not found for prefix: {args.pattern_prefix}") + sys.exit(1) + + pattern_id = pattern["pattern_id"] + print(f"Resolving: {pattern.get('service', '?')} — {pattern.get('normalized_message', '?')[:80]}") + + fixed_by = { + "resolved_at": datetime.now(timezone.utc).isoformat(), + "note": args.note, + } + if args.pr: + fixed_by["pr_url"] = f"https://github.com/{args.repo}/pull/{args.pr}" + fixed_by["pr_display"] = f"{args.repo}#{args.pr}" + + db.collection("error_patterns").document(pattern_id).update({ + "status": "fixed", + "fixed_by": fixed_by, + }) + + print(f"Resolved pattern {pattern_id[:16]}...") + if args.pr: + print(f" PR: {fixed_by['pr_url']}") + print(f" Note: {args.note}") + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 3: Make scripts executable** + +Run: +```bash +chmod +x scripts/query-error-patterns.py scripts/resolve-error-pattern.py +``` + +- [ ] **Step 4: Commit** + +```bash +git add scripts/query-error-patterns.py scripts/resolve-error-pattern.py +git commit -m "feat(error-monitor): add CLI scripts for querying and resolving patterns" +``` + +--- + +## Task 10: Update Slash Commands + +**Files:** +- Modify: `/Users/andrew/Projects/nomadkaraoke/.claude/commands/prod-errors.md` +- Modify: `/Users/andrew/Projects/nomadkaraoke/.claude/commands/prod-health.md` +- Modify: `/Users/andrew/Projects/nomadkaraoke/.claude/commands/prod-investigate.md` +- Modify: `/Users/andrew/Projects/nomadkaraoke/.claude/commands/prod-review.md` +- Modify: `/Users/andrew/Projects/nomadkaraoke/.claude/commands/prod-known-issue.md` + +**Note:** These files live in the workspace root, not the worktree. They are shared Claude Code commands, not repo code. Read each file, understand its current structure, and update to use `error_patterns` Firestore as the primary data source. + +- [ ] **Step 1: Read all five current slash commands** + +Read each command file to understand current structure before editing. + +- [ ] **Step 2: Update /prod-errors** + +Key changes: +- Primary source: `python scripts/query-error-patterns.py --json` (from karaoke-gen repo) +- Fallback: Cloud Logging direct query (if error_patterns is empty — first-run scenario) +- Group output by pattern status (new → acknowledged → known) instead of raw log categorization +- Include pattern_id prefix in output for easy `/prod-investigate` follow-up + +- [ ] **Step 3: Update /prod-health** + +Key changes: +- Add a section after health endpoint checks: "Error Patterns: X active (Y new in last hour)" +- Query: `python scripts/query-error-patterns.py --hours 1 --status new --json | python -c "import sys,json; print(len(json.load(sys.stdin)))"` + +- [ ] **Step 4: Update /prod-investigate** + +Key changes: +- First query `error_patterns` for the pattern matching the search term +- Show pattern history: first_seen, last_seen, total_count, status, rolling frequency +- Then query Cloud Logging for stack traces and detailed samples +- Show related incident if pattern has an incident_id + +- [ ] **Step 5: Update /prod-review** + +Key changes: +- Replace `known-issues.yaml` loading with `error_patterns` query +- Categorize: NEW (status=new), REGRESSIONS (status=new AND fixed_by is not null), IN PROGRESS (acknowledged), KNOWN (known/muted), FIXED (fixed in last 24h) +- Remove references to `known-issues.yaml` + +- [ ] **Step 6: Update /prod-known-issue** + +Key changes: +- Write to Firestore `error_patterns` (status: known/muted) instead of YAML +- Use `scripts/query-error-patterns.py --status known` to list +- Add/update via Python Firestore update (not YAML file editing) + +- [ ] **Step 7: Commit slash command changes** + +Note: These are in the workspace root, committed separately from the worktree. + +```bash +cd /Users/andrew/Projects/nomadkaraoke +git add .claude/commands/prod-errors.md .claude/commands/prod-health.md .claude/commands/prod-investigate.md .claude/commands/prod-review.md .claude/commands/prod-known-issue.md +git commit -m "feat(error-monitor): update prod slash commands to use error_patterns Firestore" +``` + +--- + +## Task 11: Run Full Test Suite and Verify + +- [ ] **Step 1: Run all error monitor tests** + +```bash +cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot +python -m pytest tests/unit/services/error_monitor/ -v +``` + +Expected: All tests PASS. + +- [ ] **Step 2: Run the full project test suite** + +```bash +cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot +make test 2>&1 | tail -n 100 +``` + +Expected: No regressions. All existing tests still pass. + +- [ ] **Step 3: Verify Pulumi preview** + +```bash +cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-error-bot/infrastructure +pulumi preview 2>&1 | tail -n 50 +``` + +Expected: Shows new resources to create (Cloud Run Job, 2 Cloud Schedulers, Service Account, IAM binding). No unexpected changes to existing resources. + +- [ ] **Step 4: Commit any final fixes** + +If any tests failed or Pulumi preview showed issues, fix and commit. + +--- + +## Task 12: Migrate known-issues.yaml + +- [ ] **Step 1: Read current known-issues.yaml** + +Read `/Users/andrew/Projects/nomadkaraoke/.claude/known-issues.yaml` to understand existing entries. + +- [ ] **Step 2: Write migration script** + +Create a one-time migration script (can be deleted after running): + +```python +#!/usr/bin/env python3 +"""One-time migration: known-issues.yaml → Firestore error_patterns.""" + +import hashlib +import yaml +from google.cloud import firestore +from datetime import datetime, timezone + + +def main(): + with open("/Users/andrew/Projects/nomadkaraoke/.claude/known-issues.yaml") as f: + data = yaml.safe_load(f) + + db = firestore.Client(project="nomadkaraoke") + now = datetime.now(timezone.utc).isoformat() + + for issue in data.get("issues", []): + pattern_id = hashlib.sha256( + f"{issue.get('service', 'unknown')}::{issue.get('pattern', '')}".encode() + ).hexdigest() + + status = "known" if issue.get("status") in ("known-acceptable", "investigating", "backlog") else "muted" + if issue.get("status") == "fixed": + status = "fixed" + + doc = { + "pattern_id": pattern_id, + "service": issue.get("service", "unknown"), + "resource_type": "cloud_run_service", # Most are backend + "normalized_message": issue.get("pattern", "")[:200], + "sample_message": issue.get("description", ""), + "first_seen": issue.get("fixed_date", now), + "last_seen": issue.get("fixed_date", now), + "total_count": 0, + "rolling_counts": {}, + "status": status, + "severity": "P3", + "notes": issue.get("reason", issue.get("description", "")), + "alerted_at": None, + "incident_id": None, + "merged_into": None, + "fixed_by": {"note": issue.get("fixed_by", ""), "resolved_at": issue.get("fixed_date", "")} if issue.get("status") == "fixed" else None, + } + + db.collection("error_patterns").document(pattern_id).set(doc, merge=True) + print(f"Migrated: {issue.get('id', '?')} → {status}") + + print(f"\nMigrated {len(data.get('issues', []))} issues to Firestore") + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 3: Run migration** + +```bash +python scripts/migrate-known-issues.py +``` + +- [ ] **Step 4: Verify migration** + +```bash +python scripts/query-error-patterns.py --status known +python scripts/query-error-patterns.py --status fixed +``` + +- [ ] **Step 5: Clean up** + +Delete the migration script (one-time use): +```bash +rm scripts/migrate-known-issues.py +``` + +- [ ] **Step 6: Commit** + +```bash +git add -A +git commit -m "chore(error-monitor): migrate known-issues.yaml to Firestore" +``` diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index ecf17dc3..c7983ca8 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -42,6 +42,7 @@ from modules import divebar_mirror, kn_data_sync, divebar_lookup, backup from modules import audio_separator_service from modules import gpu_artifact_registry from modules import encoding_worker_manager +from modules import error_monitor as error_monitor_module from modules.iam import backend_sa, github_actions_sa, claude_automation_sa, claude_readonly_sa, worker_sas from compute import encoding_worker_vm, github_runners, divebar_sync_vm @@ -181,6 +182,14 @@ lyrics_transcription_job = cloud_run.create_lyrics_transcription_job(bucket, bac audio_separation_job = cloud_run.create_audio_separation_job(bucket, backend_service_account) audio_download_job = cloud_run.create_audio_download_job(bucket, backend_service_account, vpc_connector) +# ==================== Error Monitor ==================== + +# Cloud Run Job + Cloud Scheduler for automated error detection and digest reports +error_monitor_resources = error_monitor_module.create_error_monitor( + backend_image=f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest", + backend_sa_email=backend_service_account.email, +) + # ==================== Monitoring ==================== # Alert policies @@ -668,3 +677,9 @@ pulumi.export("backup_function_url", backup_resources["function"].url) pulumi.export("backup_staging_bucket", backup_resources["staging_bucket"].name) pulumi.export("backup_scheduler_name", backup_resources["scheduler"].name) pulumi.export("backup_service_account", backup_resources["service_account"].email) + +# Error Monitor +pulumi.export("error_monitor_job_name", error_monitor_resources["job"].name) +pulumi.export("error_monitor_scheduler_sa", error_monitor_resources["scheduler_sa"].email) +pulumi.export("error_monitor_scheduler_name", error_monitor_resources["monitor_scheduler"].name) +pulumi.export("error_monitor_digest_scheduler_name", error_monitor_resources["digest_scheduler"].name) diff --git a/infrastructure/config.py b/infrastructure/config.py index 670abda2..ae126b04 100644 --- a/infrastructure/config.py +++ b/infrastructure/config.py @@ -133,3 +133,15 @@ class EncodingWorkerConfig: FUNCTION_NAME = "encoding-worker-idle-shutdown" FUNCTION_MEMORY = "256M" FUNCTION_TIMEOUT = 120 # 2 minutes + + +class ErrorMonitorConfig: + """Configuration for the error monitor Cloud Run Job.""" + JOB_NAME = "nomad-error-monitor" + MEMORY = "512Mi" + CPU = "1" + TIMEOUT = "300s" + MAX_RETRIES = 0 + MONITOR_SCHEDULE = "*/15 * * * *" # Every 15 minutes UTC + DIGEST_SCHEDULE = "0 8 * * *" # 08:00 UTC daily + SCHEDULER_SA_NAME = "error-monitor-scheduler" diff --git a/infrastructure/modules/error_monitor.py b/infrastructure/modules/error_monitor.py new file mode 100644 index 00000000..d75d7d13 --- /dev/null +++ b/infrastructure/modules/error_monitor.py @@ -0,0 +1,187 @@ +""" +Error Monitor Cloud Run Job and Cloud Scheduler resources. + +Deploys the error monitor as a Cloud Run Job triggered by Cloud Scheduler +every 15 minutes (error checks) and daily at 08:00 UTC (digest report). +""" + +import base64 +import json + +import pulumi +import pulumi_gcp as gcp +from pulumi_gcp import cloudrunv2, cloudscheduler, serviceaccount + +from config import PROJECT_ID, REGION, ErrorMonitorConfig + +# Empty body for the monitor trigger (Cloud Run Jobs API requires POST body) +_EMPTY_BODY_B64 = base64.b64encode(b"{}").decode() + +# Digest trigger body — sets DIGEST_MODE=true via Cloud Run Job override +_DIGEST_BODY_B64 = base64.b64encode( + json.dumps({ + "overrides": { + "containerOverrides": [ + { + "env": [ + {"name": "DIGEST_MODE", "value": "true"}, + ] + } + ] + } + }).encode() +).decode() + + +def create_error_monitor( + backend_image: pulumi.Input[str], + backend_sa_email: pulumi.Input[str], + depends_on: list = None, +) -> dict: + """ + Create the error monitor Cloud Run Job and Cloud Scheduler triggers. + + The error monitor runs as the backend SA (same permissions: Firestore, + Secrets, Logging). A dedicated scheduler SA is created for Cloud Scheduler + OAuth authentication. + + Args: + backend_image: Docker image URL for the backend (Artifact Registry). + backend_sa_email: Email of the backend service account (runs the job). + depends_on: Optional list of Pulumi resources to depend on. + + Returns: + dict with keys: 'job', 'scheduler_sa', 'monitor_scheduler', 'digest_scheduler'. + """ + job_opts = pulumi.ResourceOptions(depends_on=depends_on) if depends_on else None + + # ---- Service Account for Cloud Scheduler ---- + # Scheduler SA authenticates when invoking the Cloud Run Job via OAuth token. + scheduler_sa = serviceaccount.Account( + "error-monitor-scheduler-sa", + account_id=ErrorMonitorConfig.SCHEDULER_SA_NAME, + display_name="Error Monitor Scheduler Service Account", + ) + + # ---- Cloud Run Job ---- + error_monitor_job = cloudrunv2.Job( + "error-monitor-job", + name=ErrorMonitorConfig.JOB_NAME, + location=REGION, + template=cloudrunv2.JobTemplateArgs( + template=cloudrunv2.JobTemplateTemplateArgs( + containers=[ + cloudrunv2.JobTemplateTemplateContainerArgs( + image=backend_image, + args=["python", "-m", "backend.services.error_monitor.monitor"], + resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs( + limits={ + "cpu": ErrorMonitorConfig.CPU, + "memory": ErrorMonitorConfig.MEMORY, + }, + ), + envs=[ + # GCP project for Firestore, Logging, etc. + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="GCP_PROJECT_ID", + value=PROJECT_ID, + ), + # Environment tag for log filtering + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="ENVIRONMENT", + value="production", + ), + # Discord webhook for alert notifications + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="DISCORD_WEBHOOK_URL", + value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs( + secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs( + secret=f"projects/{PROJECT_ID}/secrets/discord-alert-webhook", + version="latest", + ), + ), + ), + ], + ) + ], + service_account=backend_sa_email, + timeout=ErrorMonitorConfig.TIMEOUT, + max_retries=ErrorMonitorConfig.MAX_RETRIES, + ), + ), + opts=job_opts, + ) + + # ---- IAM: grant scheduler SA permission to invoke (run) the job ---- + error_monitor_job_invoker = cloudrunv2.JobIamMember( + "error-monitor-scheduler-invoker", + project=PROJECT_ID, + location=REGION, + name=error_monitor_job.name, + role="roles/run.invoker", + member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"), + ) + + # Cloud Run Jobs API endpoint — Cloud Scheduler calls this to trigger a run. + # Format: https://run.googleapis.com/v2/projects/{project}/locations/{region}/jobs/{job}:run + job_run_uri = error_monitor_job.name.apply( + lambda name: ( + f"https://run.googleapis.com/v2/projects/{PROJECT_ID}" + f"/locations/{REGION}/jobs/{name}:run" + ) + ) + + # Shared opts: schedulers depend on IAM binding being in place first + scheduler_opts = pulumi.ResourceOptions(depends_on=[error_monitor_job_invoker]) + + # ---- Cloud Scheduler: every-15-min monitor ---- + monitor_scheduler = cloudscheduler.Job( + "error-monitor-scheduler", + name="error-monitor-trigger", + description="Trigger error monitor Cloud Run Job every 15 minutes", + region=REGION, + schedule=ErrorMonitorConfig.MONITOR_SCHEDULE, + time_zone="UTC", + http_target=cloudscheduler.JobHttpTargetArgs( + uri=job_run_uri, + http_method="POST", + body=_EMPTY_BODY_B64, + oauth_token=cloudscheduler.JobHttpTargetOauthTokenArgs( + service_account_email=scheduler_sa.email, + ), + ), + retry_config=cloudscheduler.JobRetryConfigArgs( + retry_count=0, + ), + opts=scheduler_opts, + ) + + # ---- Cloud Scheduler: daily digest at 08:00 UTC ---- + # Passes DIGEST_MODE=true in the request body as a Cloud Run Job override. + digest_scheduler = cloudscheduler.Job( + "error-monitor-digest-scheduler", + name="error-monitor-daily-digest", + description="Trigger error monitor Cloud Run Job daily at 08:00 UTC for digest report", + region=REGION, + schedule=ErrorMonitorConfig.DIGEST_SCHEDULE, + time_zone="UTC", + http_target=cloudscheduler.JobHttpTargetArgs( + uri=job_run_uri, + http_method="POST", + body=_DIGEST_BODY_B64, + oauth_token=cloudscheduler.JobHttpTargetOauthTokenArgs( + service_account_email=scheduler_sa.email, + ), + ), + retry_config=cloudscheduler.JobRetryConfigArgs( + retry_count=0, + ), + opts=scheduler_opts, + ) + + return { + "job": error_monitor_job, + "scheduler_sa": scheduler_sa, + "monitor_scheduler": monitor_scheduler, + "digest_scheduler": digest_scheduler, + } diff --git a/infrastructure/modules/iam/backend_sa.py b/infrastructure/modules/iam/backend_sa.py index 3a1867e7..6a17a650 100644 --- a/infrastructure/modules/iam/backend_sa.py +++ b/infrastructure/modules/iam/backend_sa.py @@ -120,6 +120,14 @@ def grant_backend_permissions( member=service_account.email.apply(lambda email: f"serviceAccount:{email}"), ) + # Cloud Logging Viewer - error monitor reads Cloud Logging for error detection + bindings["logging_viewer"] = gcp.projects.IAMMember( + "karaoke-backend-logging-viewer", + project=PROJECT_ID, + role="roles/logging.viewer", + member=service_account.email.apply(lambda email: f"serviceAccount:{email}"), + ) + return bindings diff --git a/poetry.lock b/poetry.lock index 7c08320e..94b5ddac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2546,6 +2546,44 @@ requests-oauthlib = ">=0.7.0" [package.extras] tool = ["click (>=6.0.0)"] +[[package]] +name = "google-cloud-appengine-logging" +version = "1.9.0" +description = "Google Cloud Appengine Logging API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_appengine_logging-1.9.0-py3-none-any.whl", hash = "sha256:bbf3a7e4dc171678f7f481259d1f68c3ae7d337530f1f2361f8a0b214dbcfe36"}, + {file = "google_cloud_appengine_logging-1.9.0.tar.gz", hash = "sha256:ff397f0bbc1485f979ab45767c38e0f676c9598c97c384f7412216e6ea22f805"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = ">=1.33.2,<2.0.0" +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, +] +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-audit-log" +version = "0.5.0" +description = "Google Cloud Audit Protos" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_audit_log-0.5.0-py3-none-any.whl", hash = "sha256:3f4632f25bf67446fa9085c52868f3cb42fb1afbab9489ba8978e30991afc79f"}, + {file = "google_cloud_audit_log-0.5.0.tar.gz", hash = "sha256:3b32d5e77db634c46fbd6c5e01f5bda836f420dfbb21d730501c75e9fab4e4a4"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.56.2,<2.0.0" +protobuf = ">=4.25.8,<8.0.0" + [[package]] name = "google-cloud-compute" version = "1.46.0" @@ -2610,6 +2648,33 @@ proto-plus = [ ] protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" +[[package]] +name = "google-cloud-logging" +version = "3.15.0" +description = "Google Cloud Logging API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_logging-3.15.0-py3-none-any.whl", hash = "sha256:7dcc67434c4e7181510c133d5ac8fd4ce60c23fa4158661f67e54bf440c32450"}, + {file = "google_cloud_logging-3.15.0.tar.gz", hash = "sha256:72168a1e98bbfc27c75f0b8f630a7f5d786065f3f1f7e9e53d2d787a03693a4a"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +google-cloud-appengine-logging = ">=0.1.3,<2.0.0" +google-cloud-audit-log = ">=0.3.1,<1.0.0" +google-cloud-core = ">=2.0.0,<3.0.0" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0" +grpcio = ">=1.33.2,<2.0.0" +opentelemetry-api = ">=1.9.0" +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, +] +protobuf = ">=4.25.8,<8.0.0" + [[package]] name = "google-cloud-monitoring" version = "2.29.1" @@ -10702,4 +10767,4 @@ local-whisper = ["whisper-timestamped"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "6ed6be0dbeef6f31529246a14d2ad4286edc5ab2fe455433a886429a78f8054b" +content-hash = "7b0bb0bf110afca09d3d7e7c7b829777091be76916b4a29add8ecbcf50372eef" diff --git a/pyproject.toml b/pyproject.toml index 5c57d78c..26ddbeb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.168.2" +version = "0.169.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" @@ -94,6 +94,7 @@ google-cloud-secret-manager = "2.26.0" google-cloud-monitoring = "2.29.1" # Cloud Monitoring for YouTube API quota metrics google-cloud-tasks = "2.21.0" # Cloud Tasks for worker coordination (Phase 1 scalability) google-cloud-run = "0.15.0" # Cloud Run Jobs for long-running video encoding (Phase 2 scalability) +google-cloud-logging = ">=3.9.0" # Cloud Logging for error monitoring python-dotenv = ">=1.0.0" pydub = ">=0.25.1" # Payment and email services diff --git a/scripts/query-error-patterns.py b/scripts/query-error-patterns.py new file mode 100755 index 00000000..59eb56ce --- /dev/null +++ b/scripts/query-error-patterns.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +""" +Query error patterns from Firestore. + +Usage: + python scripts/query-error-patterns.py # All active + python scripts/query-error-patterns.py --status new # Only new + python scripts/query-error-patterns.py --service karaoke-backend + python scripts/query-error-patterns.py --hours 24 # Last 24h + python scripts/query-error-patterns.py --json # JSON output + +Requires: + - google-cloud-firestore + - GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login +""" +import argparse +import json +import sys +from datetime import datetime, timedelta, timezone +from typing import Any, Optional + +from google.cloud import firestore +from google.cloud.firestore_v1 import FieldFilter + + +ACTIVE_STATUSES = ["new", "acknowledged", "known"] + + +def query_error_patterns( + status: Optional[str] = None, + service: Optional[str] = None, + hours: Optional[int] = None, + output_json: bool = False, + project: str = "nomadkaraoke", +) -> list[dict[str, Any]]: + """ + Query error patterns from Firestore. + + Args: + status: Filter by status (new/acknowledged/known/fixed). Default: active (new/acknowledged/known). + service: Filter by service name. + hours: Only return patterns last seen within this many hours. + output_json: Print JSON output instead of a table. + project: GCP project ID. + + Returns: + List of pattern dicts. + """ + db = firestore.Client(project=project) + query = db.collection("error_patterns") + + # Status filter + if status: + query = query.where(filter=FieldFilter("status", "==", status)) + else: + query = query.where(filter=FieldFilter("status", "in", ACTIVE_STATUSES)) + + # Service filter + if service: + query = query.where(filter=FieldFilter("service", "==", service)) + + # Fetch all matching docs (streaming) + patterns = [] + for doc in query.stream(): + data = doc.to_dict() + data["_id"] = doc.id + patterns.append(data) + + # Hours filter (applied in Python — Firestore requires composite index for multi-field filters) + if hours is not None: + cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) + filtered = [] + for p in patterns: + last_seen = p.get("last_seen") + if last_seen is not None: + # Firestore Timestamps have a .timestamp() method + ts = last_seen.timestamp() if hasattr(last_seen, "timestamp") else float(last_seen) + if ts >= cutoff.timestamp(): + filtered.append(p) + patterns = filtered + + # Sort by last_seen descending (most recent first) + def sort_key(p): + ls = p.get("last_seen") + if ls is None: + return 0.0 + return ls.timestamp() if hasattr(ls, "timestamp") else float(ls) + + patterns.sort(key=sort_key, reverse=True) + + if output_json: + print(json.dumps(patterns, indent=2, default=str)) + else: + _print_table(patterns) + + return patterns + + +def _fmt_last_seen(value) -> str: + """Return first 19 chars of ISO timestamp, or empty string.""" + if value is None: + return "" + if hasattr(value, "isoformat"): + return value.isoformat()[:19] + return str(value)[:19] + + +def _print_table(patterns: list[dict[str, Any]]) -> None: + """Print patterns as a human-readable table.""" + if not patterns: + print("No error patterns found.") + return + + # Column widths + col_status = 14 + col_service = 24 + col_count = 6 + col_last_seen = 19 + col_message = 50 + + header = ( + f"{'Status':<{col_status}} " + f"{'Service':<{col_service}} " + f"{'Count':>{col_count}} " + f"{'Last Seen':<{col_last_seen}} " + f"{'Message':<{col_message}}" + ) + separator = "-" * len(header) + + print(separator) + print(header) + print(separator) + + for p in patterns: + status = str(p.get("status", ""))[:col_status] + service = str(p.get("service", ""))[:col_service] + count = p.get("total_count", p.get("occurrence_count", p.get("count", 0))) + last_seen = _fmt_last_seen(p.get("last_seen")) + message = str(p.get("normalized_message", p.get("error_message", p.get("message", ""))))[:col_message] + + print( + f"{status:<{col_status}} " + f"{service:<{col_service}} " + f"{count:>{col_count}} " + f"{last_seen:<{col_last_seen}} " + f"{message:<{col_message}}" + ) + + print(separator) + print(f"Total: {len(patterns)} pattern(s)") + + +def main(): + parser = argparse.ArgumentParser( + description="Query error patterns from Firestore" + ) + parser.add_argument( + "--status", + help="Filter by status (new/acknowledged/known/fixed). Default: active patterns.", + ) + parser.add_argument( + "--service", + help="Filter by service name (e.g. karaoke-backend).", + ) + parser.add_argument( + "--hours", + type=int, + help="Only show patterns last seen within this many hours.", + ) + parser.add_argument( + "--json", + action="store_true", + dest="output_json", + help="Output as JSON.", + ) + parser.add_argument( + "--project", + default="nomadkaraoke", + help="GCP project ID (default: nomadkaraoke).", + ) + args = parser.parse_args() + + query_error_patterns( + status=args.status, + service=args.service, + hours=args.hours, + output_json=args.output_json, + project=args.project, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/resolve-error-pattern.py b/scripts/resolve-error-pattern.py new file mode 100755 index 00000000..76e97967 --- /dev/null +++ b/scripts/resolve-error-pattern.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +Resolve an error pattern by marking it as fixed. + +Usage: + python scripts/resolve-error-pattern.py --pr 42 --note "Fixed timeout" + python scripts/resolve-error-pattern.py --note "Added retry logic" + +Requires: + - google-cloud-firestore + - GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login +""" +import argparse +import sys +from datetime import datetime, timezone +from typing import Optional + +from google.cloud import firestore + + +def resolve_error_pattern( + pattern_prefix: str, + note: str, + pr: Optional[int] = None, + repo: str = "nomadkaraoke/karaoke-gen", + project: str = "nomadkaraoke", +) -> bool: + """ + Find an error pattern by ID prefix and mark it as fixed. + + Args: + pattern_prefix: Prefix of the pattern document ID. + note: Resolution note explaining how it was fixed. + pr: Optional PR number that fixed the issue. + repo: GitHub repo slug used to build the PR URL. + project: GCP project ID. + + Returns: + True if the pattern was resolved, False otherwise. + """ + try: + db = firestore.Client(project=project) + collection = db.collection("error_patterns") + except Exception as exc: + print(f"Error: Could not connect to Firestore (project={project}): {exc}", file=sys.stderr) + return False + + # Stream all docs and find those whose ID starts with the prefix + matches = [] + try: + for doc in collection.stream(): + if doc.id.startswith(pattern_prefix): + matches.append(doc) + except Exception as exc: + print(f"Error: Could not query error_patterns: {exc}", file=sys.stderr) + return False + + if len(matches) == 0: + print(f"Error: No pattern found with ID prefix '{pattern_prefix}'", file=sys.stderr) + return False + + if len(matches) > 1: + print( + f"Ambiguous prefix '{pattern_prefix}' — matches {len(matches)} pattern(s). " + "Please provide a longer prefix.", + file=sys.stderr, + ) + print("\nFirst 5 matches:", file=sys.stderr) + for doc in matches[:5]: + data = doc.to_dict() + service = data.get("service", "") + message = str(data.get("error_message", data.get("message", "")))[:60] + print(f" {doc.id} [{service}] {message}", file=sys.stderr) + return False + + doc = matches[0] + data = doc.to_dict() + resolved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + fixed_by: dict = { + "resolved_at": resolved_at, + "note": note, + } + + if pr is not None: + pr_url = f"https://github.com/{repo}/pull/{pr}" + fixed_by["pr_url"] = pr_url + fixed_by["pr_display"] = f"#{pr}" + + update_payload = { + "status": "fixed", + "fixed_by": fixed_by, + } + + try: + doc.reference.update(update_payload) + except Exception as exc: + print(f"Error: Could not update pattern {doc.id}: {exc}", file=sys.stderr) + return False + + # Confirmation output + service = data.get("service", "") + message = str(data.get("error_message", data.get("message", "")))[:80] + print(f"Resolved pattern: {doc.id}") + print(f" Service: {service}") + print(f" Message: {message}") + print(f" Status: fixed") + print(f" Note: {note}") + if pr is not None: + print(f" PR: {fixed_by['pr_url']}") + print(f" Resolved: {resolved_at}") + + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Resolve an error pattern by marking it as fixed" + ) + parser.add_argument( + "pattern_prefix", + help="Prefix of the error pattern document ID.", + ) + parser.add_argument( + "--note", + required=True, + help="Resolution note (e.g. 'Fixed timeout in encoding worker').", + ) + parser.add_argument( + "--pr", + type=int, + help="PR number that fixed the issue (optional).", + ) + parser.add_argument( + "--repo", + default="nomadkaraoke/karaoke-gen", + help="GitHub repo slug for PR URL (default: nomadkaraoke/karaoke-gen).", + ) + parser.add_argument( + "--project", + default="nomadkaraoke", + help="GCP project ID (default: nomadkaraoke).", + ) + args = parser.parse_args() + + success = resolve_error_pattern( + pattern_prefix=args.pattern_prefix, + note=args.note, + pr=args.pr, + repo=args.repo, + project=args.project, + ) + + if not success: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/services/error_monitor/__init__.py b/tests/unit/services/error_monitor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/services/error_monitor/test_config.py b/tests/unit/services/error_monitor/test_config.py new file mode 100644 index 00000000..223b26df --- /dev/null +++ b/tests/unit/services/error_monitor/test_config.py @@ -0,0 +1,269 @@ +"""Tests for error_monitor.config module.""" + +import os +from unittest import mock + +import pytest + + +class TestMonitoredServiceLists: + """Tests that all monitored service lists have the expected entries.""" + + def test_cloud_run_services_count(self): + from backend.services.error_monitor import config + + assert len(config.MONITORED_CLOUD_RUN_SERVICES) == 3 + + def test_cloud_run_services_contains_expected(self): + from backend.services.error_monitor import config + + assert "karaoke-backend" in config.MONITORED_CLOUD_RUN_SERVICES + assert "karaoke-decide" in config.MONITORED_CLOUD_RUN_SERVICES + assert "audio-separator" in config.MONITORED_CLOUD_RUN_SERVICES + + def test_cloud_run_jobs_count(self): + from backend.services.error_monitor import config + + assert len(config.MONITORED_CLOUD_RUN_JOBS) == 4 + + def test_cloud_run_jobs_contains_expected(self): + from backend.services.error_monitor import config + + assert "video-encoding-job" in config.MONITORED_CLOUD_RUN_JOBS + assert "lyrics-transcription-job" in config.MONITORED_CLOUD_RUN_JOBS + assert "audio-separation-job" in config.MONITORED_CLOUD_RUN_JOBS + assert "audio-download-job" in config.MONITORED_CLOUD_RUN_JOBS + + def test_cloud_functions_count_at_least_7(self): + from backend.services.error_monitor import config + + assert len(config.MONITORED_CLOUD_FUNCTIONS) >= 7 + + def test_cloud_functions_contains_expected(self): + from backend.services.error_monitor import config + + expected = [ + "gdrive-validator", + "runner_manager", + "backup_to_aws", + "divebar_mirror", + "kn_data_sync", + "divebar_lookup", + "encoding_worker_idle", + ] + for fn in expected: + assert fn in config.MONITORED_CLOUD_FUNCTIONS, f"Expected '{fn}' in MONITORED_CLOUD_FUNCTIONS" + + def test_gce_instances_count_at_least_4(self): + from backend.services.error_monitor import config + + assert len(config.MONITORED_GCE_INSTANCES) >= 4 + + def test_gce_instances_contains_expected(self): + from backend.services.error_monitor import config + + expected = [ + "encoding-worker-a", + "encoding-worker-b", + "flacfetch-vm", + "divebar-sync-vm", + ] + for instance in expected: + assert instance in config.MONITORED_GCE_INSTANCES, f"Expected '{instance}' in MONITORED_GCE_INSTANCES" + + +class TestDefaultValues: + """Tests that config module exposes correct default constants.""" + + def test_gcp_project_default(self): + with mock.patch.dict(os.environ, {}, clear=False): + # Remove GCP_PROJECT if set, then re-import via reload + env = {k: v for k, v in os.environ.items() if k != "GCP_PROJECT"} + with mock.patch.dict(os.environ, env, clear=True): + import importlib + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.GCP_PROJECT == "nomadkaraoke" + + def test_lookback_minutes_default(self): + from backend.services.error_monitor import config + + assert config.LOOKBACK_MINUTES == 15 + + def test_max_log_entries_default(self): + from backend.services.error_monitor import config + + assert config.MAX_LOG_ENTRIES == 500 + + def test_spike_multiplier_default(self): + from backend.services.error_monitor import config + + assert config.SPIKE_MULTIPLIER == 5.0 + + def test_spike_min_count_default(self): + from backend.services.error_monitor import config + + assert config.SPIKE_MIN_COUNT == 5 + + def test_auto_resolve_multiplier_default(self): + from backend.services.error_monitor import config + + assert config.AUTO_RESOLVE_MULTIPLIER == 8 + + def test_max_discord_messages_per_run_default(self): + from backend.services.error_monitor import config + + assert config.MAX_DISCORD_MESSAGES_PER_RUN == 10 + + def test_discord_max_message_length_default(self): + from backend.services.error_monitor import config + + assert config.DISCORD_MAX_MESSAGE_LENGTH == 2000 + + def test_max_active_patterns_default(self): + from backend.services.error_monitor import config + + assert config.MAX_ACTIVE_PATTERNS == 500 + + def test_rolling_window_days_default(self): + from backend.services.error_monitor import config + + assert config.ROLLING_WINDOW_DAYS == 7 + + def test_max_normalized_message_length_default(self): + from backend.services.error_monitor import config + + assert config.MAX_NORMALIZED_MESSAGE_LENGTH == 200 + + def test_min_patterns_for_analysis_default(self): + from backend.services.error_monitor import config + + assert isinstance(config.MIN_PATTERNS_FOR_ANALYSIS, int) + assert config.MIN_PATTERNS_FOR_ANALYSIS >= 1 + + def test_auto_resolve_min_hours(self): + from backend.services.error_monitor import config + + assert config.AUTO_RESOLVE_MIN_HOURS == 6 + + def test_auto_resolve_max_hours(self): + from backend.services.error_monitor import config + + assert config.AUTO_RESOLVE_MAX_HOURS == 168 # 1 week + + def test_auto_resolve_fallback_hours(self): + from backend.services.error_monitor import config + + assert config.AUTO_RESOLVE_FALLBACK_HOURS == 48 + + def test_service_dependency_map_is_string(self): + from backend.services.error_monitor import config + + assert isinstance(config.SERVICE_DEPENDENCY_MAP, str) + assert len(config.SERVICE_DEPENDENCY_MAP) > 0 + assert "karaoke-backend" in config.SERVICE_DEPENDENCY_MAP + + def test_llm_analysis_model_is_string(self): + from backend.services.error_monitor import config + + assert isinstance(config.LLM_ANALYSIS_MODEL, str) + assert len(config.LLM_ANALYSIS_MODEL) > 0 + + def test_llm_vertex_location_is_string(self): + from backend.services.error_monitor import config + + assert isinstance(config.LLM_VERTEX_LOCATION, str) + assert len(config.LLM_VERTEX_LOCATION) > 0 + + +class TestEnvVarOverrides: + """Tests that environment variable overrides work correctly.""" + + def test_llm_analysis_enabled_default_is_false(self): + """LLM_ANALYSIS_ENABLED should default to False when env var is not set.""" + import importlib + env = {k: v for k, v in os.environ.items() if k != "LLM_ANALYSIS_ENABLED"} + with mock.patch.dict(os.environ, env, clear=True): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.get_llm_enabled() is False + + def test_llm_analysis_enabled_true_when_env_var_set(self): + """get_llm_enabled() should return True when LLM_ANALYSIS_ENABLED=true.""" + import importlib + with mock.patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "true"}): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.get_llm_enabled() is True + + def test_llm_analysis_enabled_true_when_env_var_set_uppercase(self): + """get_llm_enabled() should handle case-insensitive 'TRUE'.""" + import importlib + with mock.patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "TRUE"}): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.get_llm_enabled() is True + + def test_llm_analysis_enabled_false_when_env_var_set_false(self): + """get_llm_enabled() should return False when LLM_ANALYSIS_ENABLED=false.""" + import importlib + with mock.patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "false"}): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.get_llm_enabled() is False + + def test_gcp_project_overridable_via_env(self): + """GCP_PROJECT should pick up the GCP_PROJECT environment variable.""" + import importlib + with mock.patch.dict(os.environ, {"GCP_PROJECT": "my-test-project"}): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.GCP_PROJECT == "my-test-project" + + def test_gcp_region_has_default(self): + from backend.services.error_monitor import config + + assert isinstance(config.GCP_REGION, str) + assert len(config.GCP_REGION) > 0 + + def test_gcp_region_overridable_via_env(self): + """GCP_REGION should pick up the GCP_REGION environment variable.""" + import importlib + with mock.patch.dict(os.environ, {"GCP_REGION": "europe-west1"}): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.GCP_REGION == "europe-west1" + + +class TestHelperFunctions: + """Tests for config helper functions.""" + + def test_get_discord_webhook_secret_name_returns_string(self): + from backend.services.error_monitor import config + + result = config.get_discord_webhook_secret_name() + assert isinstance(result, str) + assert len(result) > 0 + + def test_get_digest_mode_returns_bool(self): + from backend.services.error_monitor import config + + result = config.get_digest_mode() + assert isinstance(result, bool) + + def test_get_digest_mode_default_false(self): + """DIGEST_MODE should default to False.""" + import importlib + env = {k: v for k, v in os.environ.items() if k != "DIGEST_MODE"} + with mock.patch.dict(os.environ, env, clear=True): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.get_digest_mode() is False + + def test_get_digest_mode_true_when_env_set(self): + """get_digest_mode() should return True when DIGEST_MODE=true.""" + import importlib + with mock.patch.dict(os.environ, {"DIGEST_MODE": "true"}): + from backend.services.error_monitor import config as cfg + importlib.reload(cfg) + assert cfg.get_digest_mode() is True diff --git a/tests/unit/services/error_monitor/test_discord.py b/tests/unit/services/error_monitor/test_discord.py new file mode 100644 index 00000000..9033eebc --- /dev/null +++ b/tests/unit/services/error_monitor/test_discord.py @@ -0,0 +1,619 @@ +"""Tests for error_monitor.discord module.""" + +from datetime import datetime +from unittest import mock + +import pytest +import requests + + +class TestFormatNewPatternAlert: + """Tests for format_new_pattern_alert formatting function.""" + + def test_contains_service_name(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused to db at 192.168.1.1", + count=5, + first_seen="2026-04-10T12:00:00Z", + ) + assert "karaoke-backend" in result + + def test_contains_normalized_message(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused to db at 192.168.1.1", + count=5, + first_seen="2026-04-10T12:00:00Z", + ) + assert "Connection refused" in result + + def test_contains_red_circle_emoji(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused to db", + count=3, + first_seen="2026-04-10T12:00:00Z", + ) + assert "🔴" in result + + def test_contains_prod_investigate_prompt(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused to db", + count=3, + first_seen="2026-04-10T12:00:00Z", + ) + assert "/prod-investigate" in result + + def test_truncates_long_message(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + from backend.services.error_monitor.config import DISCORD_MAX_MESSAGE_LENGTH + + very_long_message = "x" * 3000 + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message=very_long_message, + sample_message="sample", + count=1, + first_seen="2026-04-10T12:00:00Z", + ) + assert len(result) <= DISCORD_MAX_MESSAGE_LENGTH + + def test_contains_count(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Timeout error", + sample_message="Timeout after 30s", + count=42, + first_seen="2026-04-10T12:00:00Z", + ) + assert "42" in result + + def test_contains_first_seen(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Timeout error", + sample_message="Timeout after 30s", + count=1, + first_seen="2026-04-10T12:00:00Z", + ) + assert "2026-04-10T12:00:00Z" in result + + def test_contains_sample_message(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Timeout error", + sample_message="Timeout after 30s on job xyz", + count=1, + first_seen="2026-04-10T12:00:00Z", + ) + assert "Timeout after 30s on job xyz" in result + + def test_contains_resource_type_label(self): + from backend.services.error_monitor.discord import format_new_pattern_alert + + result = format_new_pattern_alert( + service="encoding-worker-a", + resource_type="gce_instance", + normalized_message="Disk full", + sample_message="No space left on device", + count=2, + first_seen="2026-04-10T08:00:00Z", + ) + # Resource type should appear somewhere in the message + assert "gce_instance" in result or "GCE" in result or "VM" in result or "Instance" in result + + +class TestFormatIncidentAlert: + """Tests for format_incident_alert formatting function.""" + + def test_p0_uses_siren_emoji(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="Database down", + severity="P0", + root_cause="Firestore quota exceeded", + suggested_fix="Increase quota", + primary_service="karaoke-backend", + patterns=[], + ) + assert "🚨" in result + + def test_p1_uses_red_circle(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="High error rate", + severity="P1", + root_cause="Memory pressure", + suggested_fix="Scale up", + primary_service="karaoke-backend", + patterns=[], + ) + assert "🔴" in result + + def test_p2_uses_warning_emoji(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="Elevated latency", + severity="P2", + root_cause="Slow queries", + suggested_fix="Add index", + primary_service="karaoke-backend", + patterns=[], + ) + assert "⚠️" in result + + def test_p3_uses_yellow_circle(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="Minor degradation", + severity="P3", + root_cause="Cache miss", + suggested_fix="Warm cache", + primary_service="karaoke-backend", + patterns=[], + ) + assert "🟡" in result + + def test_contains_title(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="My Special Incident Title", + severity="P1", + root_cause="Something broke", + suggested_fix="Fix the thing", + primary_service="karaoke-backend", + patterns=[], + ) + assert "My Special Incident Title" in result + + def test_contains_root_cause(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="Incident", + severity="P1", + root_cause="The root cause explanation here", + suggested_fix="Fix it", + primary_service="karaoke-backend", + patterns=[], + ) + assert "The root cause explanation here" in result + + def test_contains_suggested_fix(self): + from backend.services.error_monitor.discord import format_incident_alert + + result = format_incident_alert( + title="Incident", + severity="P1", + root_cause="Root cause", + suggested_fix="The suggested fix steps here", + primary_service="karaoke-backend", + patterns=[], + ) + assert "The suggested fix steps here" in result + + def test_shows_up_to_5_patterns(self): + from backend.services.error_monitor.discord import format_incident_alert + + patterns = [ + {"service": f"service-{i}", "normalized_message": f"Error message {i}", "count": i + 1} + for i in range(7) + ] + result = format_incident_alert( + title="Multi-pattern incident", + severity="P1", + root_cause="Root cause", + suggested_fix="Fix", + primary_service="service-0", + patterns=patterns, + ) + # First 5 should appear + for i in range(5): + assert f"Error message {i}" in result + # 6th and 7th should not appear + assert "Error message 5" not in result + assert "Error message 6" not in result + + def test_pattern_message_truncated_to_80_chars(self): + from backend.services.error_monitor.discord import format_incident_alert + + long_msg = "A" * 100 + patterns = [{"service": "svc", "normalized_message": long_msg, "count": 1}] + result = format_incident_alert( + title="Incident", + severity="P1", + root_cause="Root cause", + suggested_fix="Fix", + primary_service="svc", + patterns=patterns, + ) + # The full 100-char message should not appear; only first 80 + assert long_msg not in result + assert long_msg[:80] in result + + def test_truncates_to_discord_limit(self): + from backend.services.error_monitor.discord import format_incident_alert + from backend.services.error_monitor.config import DISCORD_MAX_MESSAGE_LENGTH + + result = format_incident_alert( + title="T" * 500, + severity="P0", + root_cause="R" * 500, + suggested_fix="F" * 500, + primary_service="svc", + patterns=[], + ) + assert len(result) <= DISCORD_MAX_MESSAGE_LENGTH + + +class TestFormatAutoResolvedAlert: + """Tests for format_auto_resolved_alert formatting function.""" + + def test_contains_checkmark_emoji(self): + from backend.services.error_monitor.discord import format_auto_resolved_alert + + result = format_auto_resolved_alert( + resolved_patterns=[ + {"service": "karaoke-backend", "message": "Connection refused", "hours_silent": 8.5} + ] + ) + assert "✅" in result + + def test_consolidated_list_up_to_8_patterns(self): + from backend.services.error_monitor.discord import format_auto_resolved_alert + + patterns = [ + {"service": f"service-{i}", "message": f"Error {i}", "hours_silent": float(i + 1)} + for i in range(10) + ] + result = format_auto_resolved_alert(resolved_patterns=patterns) + # First 8 should appear + for i in range(8): + assert f"Error {i}" in result + # 9th and 10th should not appear + assert "Error 8" not in result + assert "Error 9" not in result + + def test_contains_service_name(self): + from backend.services.error_monitor.discord import format_auto_resolved_alert + + result = format_auto_resolved_alert( + resolved_patterns=[ + {"service": "audio-separator", "message": "GPU error", "hours_silent": 12.0} + ] + ) + assert "audio-separator" in result + + def test_contains_hours_silent(self): + from backend.services.error_monitor.discord import format_auto_resolved_alert + + result = format_auto_resolved_alert( + resolved_patterns=[ + {"service": "karaoke-backend", "message": "Timeout", "hours_silent": 7.5} + ] + ) + # Should mention 7.5 or the number somewhere + assert "7.5" in result or "7" in result + + def test_message_truncated_to_60_chars(self): + from backend.services.error_monitor.discord import format_auto_resolved_alert + + long_msg = "B" * 80 + result = format_auto_resolved_alert( + resolved_patterns=[ + {"service": "svc", "message": long_msg, "hours_silent": 10.0} + ] + ) + assert long_msg not in result + assert long_msg[:60] in result + + def test_empty_patterns_returns_string(self): + from backend.services.error_monitor.discord import format_auto_resolved_alert + + result = format_auto_resolved_alert(resolved_patterns=[]) + assert isinstance(result, str) + assert "✅" in result + + +class TestFormatSpikeAlert: + """Tests for format_spike_alert formatting function.""" + + def test_contains_warning_emoji(self): + from backend.services.error_monitor.discord import format_spike_alert + + result = format_spike_alert( + service="karaoke-backend", + normalized_message="Connection refused", + current_count=50, + rolling_average=5.0, + ) + assert "⚠️" in result + + def test_contains_service_name(self): + from backend.services.error_monitor.discord import format_spike_alert + + result = format_spike_alert( + service="audio-separator", + normalized_message="OOM error", + current_count=30, + rolling_average=3.0, + ) + assert "audio-separator" in result + + def test_contains_current_count(self): + from backend.services.error_monitor.discord import format_spike_alert + + result = format_spike_alert( + service="karaoke-backend", + normalized_message="Timeout", + current_count=75, + rolling_average=10.0, + ) + assert "75" in result + + def test_contains_rolling_average(self): + from backend.services.error_monitor.discord import format_spike_alert + + result = format_spike_alert( + service="karaoke-backend", + normalized_message="Timeout", + current_count=75, + rolling_average=10.0, + ) + assert "10" in result + + def test_contains_multiplier(self): + from backend.services.error_monitor.discord import format_spike_alert + + result = format_spike_alert( + service="karaoke-backend", + normalized_message="Timeout", + current_count=50, + rolling_average=5.0, + ) + # 50 / 5.0 = 10x + assert "10" in result or "10.0" in result + + def test_contains_normalized_message(self): + from backend.services.error_monitor.discord import format_spike_alert + + result = format_spike_alert( + service="karaoke-backend", + normalized_message="Database connection pool exhausted", + current_count=50, + rolling_average=5.0, + ) + assert "Database connection pool exhausted" in result + + +class TestFormatDailyDigest: + """Tests for format_daily_digest formatting function.""" + + def test_contains_chart_emoji(self): + from backend.services.error_monitor.discord import format_daily_digest + + result = format_daily_digest( + total_errors=100, + new_patterns=5, + resolved_patterns=2, + active_patterns=10, + per_service_counts={"karaoke-backend": 60, "audio-separator": 40}, + ) + assert "📊" in result + + def test_contains_total_errors(self): + from backend.services.error_monitor.discord import format_daily_digest + + result = format_daily_digest( + total_errors=1234, + new_patterns=5, + resolved_patterns=2, + active_patterns=10, + per_service_counts={}, + ) + assert "1234" in result + + def test_contains_new_patterns(self): + from backend.services.error_monitor.discord import format_daily_digest + + result = format_daily_digest( + total_errors=100, + new_patterns=7, + resolved_patterns=2, + active_patterns=10, + per_service_counts={}, + ) + assert "7" in result + + def test_contains_per_service_counts(self): + from backend.services.error_monitor.discord import format_daily_digest + + result = format_daily_digest( + total_errors=100, + new_patterns=5, + resolved_patterns=2, + active_patterns=10, + per_service_counts={"karaoke-backend": 60, "audio-separator": 40}, + ) + assert "karaoke-backend" in result + assert "audio-separator" in result + assert "60" in result + assert "40" in result + + def test_per_service_sorted_by_count_desc(self): + from backend.services.error_monitor.discord import format_daily_digest + + result = format_daily_digest( + total_errors=100, + new_patterns=5, + resolved_patterns=2, + active_patterns=10, + per_service_counts={ + "service-low": 10, + "service-high": 80, + "service-mid": 30, + }, + ) + # service-high should appear before service-low in the output + high_pos = result.index("service-high") + low_pos = result.index("service-low") + assert high_pos < low_pos + + def test_contains_active_patterns(self): + from backend.services.error_monitor.discord import format_daily_digest + + result = format_daily_digest( + total_errors=100, + new_patterns=5, + resolved_patterns=2, + active_patterns=15, + per_service_counts={}, + ) + assert "15" in result + + def test_truncates_to_discord_limit(self): + from backend.services.error_monitor.discord import format_daily_digest + from backend.services.error_monitor.config import DISCORD_MAX_MESSAGE_LENGTH + + # Create lots of services to potentially overflow + per_service = {f"service-{i:03d}": i * 10 for i in range(100)} + result = format_daily_digest( + total_errors=99999, + new_patterns=99, + resolved_patterns=50, + active_patterns=200, + per_service_counts=per_service, + ) + assert len(result) <= DISCORD_MAX_MESSAGE_LENGTH + + +class TestErrorMonitorDiscordSendMessage: + """Tests for ErrorMonitorDiscord.send_message method.""" + + def test_send_message_returns_true_on_success(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + mock_response = mock.MagicMock() + mock_response.status_code = 204 + mock_response.raise_for_status.return_value = None + + with mock.patch("requests.post", return_value=mock_response) as mock_post: + result = client.send_message("Hello Discord!") + assert result is True + mock_post.assert_called_once_with( + "https://discord.com/api/webhooks/123/abc", + json={"content": "Hello Discord!"}, + timeout=30, + ) + + def test_send_message_returns_false_on_http_error(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + mock_response = mock.MagicMock() + mock_response.raise_for_status.side_effect = requests.HTTPError("429 Too Many Requests") + + with mock.patch("requests.post", return_value=mock_response): + result = client.send_message("Hello Discord!") + assert result is False + + def test_send_message_returns_false_on_connection_error(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + + with mock.patch("requests.post", side_effect=requests.ConnectionError("timeout")): + result = client.send_message("Hello Discord!") + assert result is False + + def test_send_message_respects_rate_limit(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + client.messages_sent = MAX_DISCORD_MESSAGES_PER_RUN # Already at limit + + with mock.patch("requests.post") as mock_post: + result = client.send_message("Should be skipped") + assert result is False + mock_post.assert_not_called() + + def test_send_message_increments_counter_on_success(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + assert client.messages_sent == 0 + + mock_response = mock.MagicMock() + mock_response.raise_for_status.return_value = None + + with mock.patch("requests.post", return_value=mock_response): + client.send_message("First message") + assert client.messages_sent == 1 + client.send_message("Second message") + assert client.messages_sent == 2 + + def test_send_message_increments_counter_on_failure(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + assert client.messages_sent == 0 + + mock_response = mock.MagicMock() + mock_response.raise_for_status.side_effect = requests.HTTPError("500") + + with mock.patch("requests.post", return_value=mock_response): + client.send_message("Will fail") + assert client.messages_sent == 1 + + def test_send_message_initial_counter_is_zero(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + assert client.messages_sent == 0 + + def test_send_message_does_not_increment_when_rate_limited(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN + + client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc") + client.messages_sent = MAX_DISCORD_MESSAGES_PER_RUN + + with mock.patch("requests.post") as mock_post: + client.send_message("Skipped") + # Counter should not go beyond limit when rate-limited + assert client.messages_sent == MAX_DISCORD_MESSAGES_PER_RUN diff --git a/tests/unit/services/error_monitor/test_firestore_adapter.py b/tests/unit/services/error_monitor/test_firestore_adapter.py new file mode 100644 index 00000000..78d11021 --- /dev/null +++ b/tests/unit/services/error_monitor/test_firestore_adapter.py @@ -0,0 +1,1112 @@ +"""Tests for error_monitor.firestore_adapter module. + +All tests use a mock Firestore client — no real GCP connections made. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, call, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers / shared fixtures +# --------------------------------------------------------------------------- + + +def _make_doc_snapshot(exists: bool, data: dict | None = None) -> MagicMock: + """Return a mock that mimics a Firestore DocumentSnapshot.""" + snap = MagicMock() + snap.exists = exists + snap.to_dict.return_value = data or {} + return snap + + +def _make_db(): + """Return a mock Firestore client with collection/document chaining.""" + db = MagicMock() + return db + + +# --------------------------------------------------------------------------- +# Import helpers (deferred so import errors are clear in tests) +# --------------------------------------------------------------------------- + + +def _import_adapter(): + from backend.services.error_monitor.firestore_adapter import ErrorPatternsAdapter + + return ErrorPatternsAdapter + + +def _import_pattern_data(): + from backend.services.error_monitor.firestore_adapter import PatternData + + return PatternData + + +def _import_upsert_result(): + from backend.services.error_monitor.firestore_adapter import UpsertResult + + return UpsertResult + + +# --------------------------------------------------------------------------- +# Tests: get_pattern +# --------------------------------------------------------------------------- + + +class TestGetPattern: + """get_pattern should return None for missing docs and dict for existing.""" + + def test_returns_none_for_missing_doc(self): + db = _make_db() + snap = _make_doc_snapshot(exists=False) + db.collection.return_value.document.return_value.get.return_value = snap + + Adapter = _import_adapter() + adapter = Adapter(db=db) + result = adapter.get_pattern("pattern_abc123") + + assert result is None + + def test_returns_dict_for_existing_doc(self): + db = _make_db() + expected = { + "pattern_id": "pattern_abc123", + "service": "karaoke-backend", + "status": "new", + } + snap = _make_doc_snapshot(exists=True, data=expected) + db.collection.return_value.document.return_value.get.return_value = snap + + Adapter = _import_adapter() + adapter = Adapter(db=db) + result = adapter.get_pattern("pattern_abc123") + + assert result == expected + + def test_queries_correct_collection_and_doc(self): + db = _make_db() + snap = _make_doc_snapshot(exists=False) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + col_ref = MagicMock() + col_ref.document.return_value = doc_ref + db.collection.return_value = col_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + adapter.get_pattern("my_pattern_id") + + db.collection.assert_called_with("error_patterns") + col_ref.document.assert_called_with("my_pattern_id") + + +# --------------------------------------------------------------------------- +# Tests: upsert_pattern — new pattern +# --------------------------------------------------------------------------- + + +class TestUpsertPatternNew: + """upsert_pattern should create new patterns with correct initial fields.""" + + def test_creates_new_pattern_when_not_exists(self): + db = _make_db() + # get() returns missing doc — triggers create path + snap = _make_doc_snapshot(exists=False) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_123", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused to 10.0.0.1:5432", + count=3, + timestamp=now, + ) + result = adapter.upsert_pattern(data) + + # The doc_ref.set() should have been called (new pattern) + doc_ref.set.assert_called_once() + created_doc = doc_ref.set.call_args[0][0] + + assert created_doc["pattern_id"] == "pat_123" + assert created_doc["service"] == "karaoke-backend" + assert created_doc["status"] == "new" + assert created_doc["severity"] == "P3" + assert created_doc["total_count"] == 3 + assert created_doc["sample_message"] == "Connection refused to 10.0.0.1:5432" + + def test_new_pattern_result_is_new(self): + db = _make_db() + snap = _make_doc_snapshot(exists=False) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_123", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused", + count=1, + timestamp=now, + ) + result = adapter.upsert_pattern(data) + + UpsertResult = _import_upsert_result() + assert isinstance(result, UpsertResult) + assert result.is_new is True + assert result.pattern_id == "pat_123" + assert result.previous_status is None + + def test_new_pattern_has_rolling_counts(self): + db = _make_db() + snap = _make_doc_snapshot(exists=False) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_123", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="Connection refused", + count=5, + timestamp=now, + ) + adapter.upsert_pattern(data) + created_doc = doc_ref.set.call_args[0][0] + + # rolling_counts should be a list with one entry + assert "rolling_counts" in created_doc + assert isinstance(created_doc["rolling_counts"], list) + assert len(created_doc["rolling_counts"]) == 1 + assert created_doc["rolling_counts"][0]["count"] == 5 + + +# --------------------------------------------------------------------------- +# Tests: upsert_pattern — existing pattern +# --------------------------------------------------------------------------- + + +class TestUpsertPatternExisting: + """upsert_pattern should update existing patterns and increment counts.""" + + def _make_existing_pattern(self, status: str = "new") -> dict: + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + return { + "pattern_id": "pat_existing", + "service": "karaoke-backend", + "resource_type": "cloud_run_service", + "normalized_message": "Connection refused", + "sample_message": "old sample", + "status": status, + "severity": "P3", + "total_count": 10, + "rolling_counts": [ + {"ts": (now - timedelta(hours=2)).isoformat(), "count": 10} + ], + "first_seen": now.isoformat(), + "last_seen": (now - timedelta(hours=2)).isoformat(), + "alerted_at": None, + } + + def test_increments_total_count(self): + db = _make_db() + existing = self._make_existing_pattern() + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 14, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_existing", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="new sample", + count=3, + timestamp=now, + ) + adapter.upsert_pattern(data) + + # update() should have been called + doc_ref.update.assert_called_once() + updated = doc_ref.update.call_args[0][0] + assert updated["total_count"] == 13 # 10 + 3 + + def test_existing_pattern_result_is_not_new(self): + db = _make_db() + existing = self._make_existing_pattern(status="new") + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 14, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_existing", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="new sample", + count=3, + timestamp=now, + ) + result = adapter.upsert_pattern(data) + + assert result.is_new is False + assert result.previous_status == "new" + + def test_updates_sample_message(self): + db = _make_db() + existing = self._make_existing_pattern() + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 14, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_existing", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="updated sample message", + count=1, + timestamp=now, + ) + adapter.upsert_pattern(data) + + updated = doc_ref.update.call_args[0][0] + assert updated["sample_message"] == "updated sample message" + + def test_updates_rolling_counts(self): + db = _make_db() + existing = self._make_existing_pattern() + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 14, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_existing", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="new sample", + count=5, + timestamp=now, + ) + adapter.upsert_pattern(data) + + updated = doc_ref.update.call_args[0][0] + assert "rolling_counts" in updated + # Should have 2 entries now: original + new + assert len(updated["rolling_counts"]) == 2 + assert updated["rolling_counts"][-1]["count"] == 5 + + +# --------------------------------------------------------------------------- +# Tests: upsert_pattern — reactivation +# --------------------------------------------------------------------------- + + +class TestUpsertPatternReactivation: + """upsert_pattern should reactivate auto_resolved and fixed patterns.""" + + def _make_resolved_pattern(self, status: str) -> dict: + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + return { + "pattern_id": "pat_resolved", + "service": "karaoke-backend", + "resource_type": "cloud_run_service", + "normalized_message": "Connection refused", + "sample_message": "old sample", + "status": status, + "severity": "P3", + "total_count": 10, + "rolling_counts": [], + "first_seen": now.isoformat(), + "last_seen": (now - timedelta(days=3)).isoformat(), + "alerted_at": None, + } + + def test_reactivates_auto_resolved_pattern(self): + db = _make_db() + existing = self._make_resolved_pattern(status="auto_resolved") + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 15, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_resolved", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="new occurrence", + count=2, + timestamp=now, + ) + result = adapter.upsert_pattern(data) + + # is_new should be True on reactivation + assert result.is_new is True + assert result.previous_status == "auto_resolved" + + updated = doc_ref.update.call_args[0][0] + assert updated["status"] == "new" + + def test_reactivates_fixed_pattern(self): + db = _make_db() + existing = self._make_resolved_pattern(status="fixed") + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 15, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_resolved", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="regression!", + count=1, + timestamp=now, + ) + result = adapter.upsert_pattern(data) + + assert result.is_new is True + assert result.previous_status == "fixed" + updated = doc_ref.update.call_args[0][0] + assert updated["status"] == "new" + + def test_acknowledged_pattern_not_reactivated(self): + """Acknowledged patterns should NOT be flipped back to 'new'.""" + db = _make_db() + existing = self._make_resolved_pattern(status="acknowledged") + snap = _make_doc_snapshot(exists=True, data=existing) + doc_ref = MagicMock() + doc_ref.get.return_value = snap + db.collection.return_value.document.return_value = doc_ref + + PatternData = _import_pattern_data() + Adapter = _import_adapter() + adapter = Adapter(db=db) + + now = datetime(2026, 4, 10, 15, 0, 0, tzinfo=timezone.utc) + data = PatternData( + pattern_id="pat_resolved", + service="karaoke-backend", + resource_type="cloud_run_service", + normalized_message="Connection refused", + sample_message="still happening", + count=2, + timestamp=now, + ) + result = adapter.upsert_pattern(data) + + assert result.is_new is False + updated = doc_ref.update.call_args[0][0] + # Status should NOT be reset to "new" + assert updated.get("status") != "new" + + +# --------------------------------------------------------------------------- +# Tests: get_active_patterns +# --------------------------------------------------------------------------- + + +class TestGetActivePatterns: + """get_active_patterns should return patterns with status in [new, acknowledged, known].""" + + def test_returns_active_pattern_dicts(self): + db = _make_db() + pattern_1 = {"pattern_id": "p1", "status": "new"} + pattern_2 = {"pattern_id": "p2", "status": "acknowledged"} + pattern_3 = {"pattern_id": "p3", "status": "known"} + + snap_1 = _make_doc_snapshot(exists=True, data=pattern_1) + snap_2 = _make_doc_snapshot(exists=True, data=pattern_2) + snap_3 = _make_doc_snapshot(exists=True, data=pattern_3) + + # Query chain: collection.where.stream + query_mock = MagicMock() + query_mock.stream.return_value = [snap_1, snap_2, snap_3] + db.collection.return_value.where.return_value = query_mock + + Adapter = _import_adapter() + adapter = Adapter(db=db) + results = adapter.get_active_patterns() + + assert len(results) == 3 + assert results[0]["pattern_id"] == "p1" + assert results[1]["status"] == "acknowledged" + + def test_queries_correct_collection(self): + db = _make_db() + query_mock = MagicMock() + query_mock.stream.return_value = [] + db.collection.return_value.where.return_value = query_mock + + Adapter = _import_adapter() + adapter = Adapter(db=db) + adapter.get_active_patterns() + + db.collection.assert_called_with("error_patterns") + + +# --------------------------------------------------------------------------- +# Tests: _compute_resolve_threshold +# --------------------------------------------------------------------------- + + +class TestComputeResolveThreshold: + """_compute_resolve_threshold should return frequency-aware threshold.""" + + def test_fewer_than_3_points_returns_fallback(self): + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + # 0 data points + result = adapter._compute_resolve_threshold([]) + assert result == 48 # AUTO_RESOLVE_FALLBACK_HOURS + + def test_one_point_returns_fallback(self): + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [{"ts": now.isoformat(), "count": 5}] + result = adapter._compute_resolve_threshold(rolling) + assert result == 48 + + def test_two_points_returns_fallback(self): + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [ + {"ts": (now - timedelta(hours=4)).isoformat(), "count": 3}, + {"ts": now.isoformat(), "count": 2}, + ] + result = adapter._compute_resolve_threshold(rolling) + assert result == 48 + + def test_sufficient_data_returns_clamped_value(self): + """With 3+ data points, result should be in [6, 168] range.""" + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + # 3 points separated by 1 hour each → mean_interval = 1h + # threshold = 1 * 8 = 8 → clamped to [6, 168] → 8 + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [ + {"ts": (now - timedelta(hours=2)).isoformat(), "count": 5}, + {"ts": (now - timedelta(hours=1)).isoformat(), "count": 5}, + {"ts": now.isoformat(), "count": 5}, + ] + result = adapter._compute_resolve_threshold(rolling) + + assert 6 <= result <= 168 + + def test_very_frequent_errors_clamped_to_min(self): + """Very frequent (sub-minute) errors → threshold clamped to AUTO_RESOLVE_MIN_HOURS.""" + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + # 3 points separated by 1 minute each → mean_interval = 0.0167h + # threshold = 0.0167 * 8 = 0.133 → clamped to 6 + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [ + {"ts": (now - timedelta(minutes=2)).isoformat(), "count": 100}, + {"ts": (now - timedelta(minutes=1)).isoformat(), "count": 100}, + {"ts": now.isoformat(), "count": 100}, + ] + result = adapter._compute_resolve_threshold(rolling) + assert result == 6 # AUTO_RESOLVE_MIN_HOURS + + def test_very_infrequent_errors_clamped_to_max(self): + """Very infrequent errors → threshold clamped to AUTO_RESOLVE_MAX_HOURS.""" + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + # 3 points separated by 30 days each → mean_interval = 720h + # threshold = 720 * 8 → clamped to 168 + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [ + {"ts": (now - timedelta(days=60)).isoformat(), "count": 1}, + {"ts": (now - timedelta(days=30)).isoformat(), "count": 1}, + {"ts": now.isoformat(), "count": 1}, + ] + result = adapter._compute_resolve_threshold(rolling) + assert result == 168 # AUTO_RESOLVE_MAX_HOURS + + +# --------------------------------------------------------------------------- +# Tests: create_incident +# --------------------------------------------------------------------------- + + +class TestCreateIncident: + """create_incident should write an incident doc and link patterns.""" + + def test_creates_incident_doc(self): + db = _make_db() + incident_ref = MagicMock() + db.collection.return_value.document.return_value = incident_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + incident_id = adapter.create_incident( + title="Database connection failures", + root_cause="Postgres is down", + severity="P1", + suggested_fix="Restart DB", + primary_service="karaoke-backend", + pattern_ids=["pat_1", "pat_2"], + used_llm=True, + ) + + # incident_ref.set() should have been called + incident_ref.set.assert_called_once() + doc_data = incident_ref.set.call_args[0][0] + + assert doc_data["title"] == "Database connection failures" + assert doc_data["root_cause"] == "Postgres is down" + assert doc_data["severity"] == "P1" + assert doc_data["suggested_fix"] == "Restart DB" + assert doc_data["primary_service"] == "karaoke-backend" + assert doc_data["pattern_ids"] == ["pat_1", "pat_2"] + assert doc_data["used_llm"] is True + + def test_incident_id_format(self): + """Incident ID should match inc_YYYYMMDD_HHMM_<8hex> format.""" + import re + + db = _make_db() + doc_ref = MagicMock() + db.collection.return_value.document.return_value = doc_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + incident_id = adapter.create_incident( + title="Test incident", + root_cause="Unknown", + severity="P3", + suggested_fix="Wait", + primary_service="karaoke-backend", + pattern_ids=[], + used_llm=False, + ) + + assert re.match(r"^inc_\d{8}_\d{4}_[0-9a-f]{8}$", incident_id), ( + f"Unexpected incident_id format: {incident_id}" + ) + + def test_links_patterns_via_update(self): + """Each pattern in pattern_ids should be linked to the incident.""" + db = _make_db() + + incident_ref = MagicMock() + # We need to differentiate between collection calls for incidents vs patterns + pattern_ref = MagicMock() + + call_count = [0] + + def collection_side_effect(name): + mock = MagicMock() + if name == "error_incidents": + mock.document.return_value = incident_ref + elif name == "error_patterns": + mock.document.return_value = pattern_ref + return mock + + db.collection.side_effect = collection_side_effect + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.create_incident( + title="Test", + root_cause="Unknown", + severity="P3", + suggested_fix="Fix it", + primary_service="svc", + pattern_ids=["pat_1", "pat_2"], + used_llm=False, + ) + + # pattern_ref.update should have been called twice (once per pattern) + assert pattern_ref.update.call_count == 2 + + +# --------------------------------------------------------------------------- +# Tests: log_discord_alert +# --------------------------------------------------------------------------- + + +class TestLogDiscordAlert: + """log_discord_alert should write an audit doc to discord_alerts.""" + + def test_writes_alert_doc(self): + db = _make_db() + alert_ref = MagicMock() + db.collection.return_value.document.return_value = alert_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.log_discord_alert( + alert_type="new_pattern", + content="Error in karaoke-backend", + success=True, + metadata={"pattern_id": "pat_123"}, + ) + + alert_ref.set.assert_called_once() + doc_data = alert_ref.set.call_args[0][0] + + assert doc_data["alert_type"] == "new_pattern" + assert doc_data["content"] == "Error in karaoke-backend" + assert doc_data["success"] is True + assert doc_data["metadata"]["pattern_id"] == "pat_123" + + def test_alert_id_format(self): + """Alert ID should match alert_YYYYMMDD_HHMMSS_<8hex> format.""" + import re + + db = _make_db() + doc_ref = MagicMock() + col_ref = MagicMock() + col_ref.document.return_value = doc_ref + db.collection.return_value = col_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + # Capture the document ID used + adapter.log_discord_alert( + alert_type="digest", + content="All clear", + success=True, + ) + + called_with = col_ref.document.call_args[0][0] + assert re.match(r"^alert_\d{8}_\d{6}_[0-9a-f]{8}$", called_with), ( + f"Unexpected alert_id format: {called_with}" + ) + + def test_metadata_defaults_to_empty_dict(self): + db = _make_db() + alert_ref = MagicMock() + db.collection.return_value.document.return_value = alert_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.log_discord_alert( + alert_type="test", + content="hello", + success=False, + ) + + doc_data = alert_ref.set.call_args[0][0] + assert doc_data["metadata"] == {} + + +# --------------------------------------------------------------------------- +# Tests: merge_pattern +# --------------------------------------------------------------------------- + + +class TestMergePattern: + """merge_pattern should set source status to 'muted' and record merged_into.""" + + def test_sets_muted_status(self): + db = _make_db() + source_ref = MagicMock() + db.collection.return_value.document.return_value = source_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.merge_pattern( + source_id="pat_source", + target_id="pat_target", + reason="Duplicate error patterns", + ) + + source_ref.update.assert_called_once() + updated = source_ref.update.call_args[0][0] + assert updated["status"] == "muted" + + def test_records_merged_into(self): + db = _make_db() + source_ref = MagicMock() + db.collection.return_value.document.return_value = source_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.merge_pattern( + source_id="pat_source", + target_id="pat_target", + reason="Duplicate", + ) + + updated = source_ref.update.call_args[0][0] + assert updated["merged_into"] == "pat_target" + assert updated["merge_reason"] == "Duplicate" + + +# --------------------------------------------------------------------------- +# Tests: update_pattern_alerted +# --------------------------------------------------------------------------- + + +class TestUpdatePatternAlerted: + """update_pattern_alerted should set alerted_at timestamp.""" + + def test_sets_alerted_at(self): + db = _make_db() + pattern_ref = MagicMock() + db.collection.return_value.document.return_value = pattern_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.update_pattern_alerted("pat_123") + + pattern_ref.update.assert_called_once() + updated = pattern_ref.update.call_args[0][0] + assert "alerted_at" in updated + assert updated["alerted_at"] is not None + + def test_queries_correct_pattern(self): + db = _make_db() + col_ref = MagicMock() + pattern_ref = MagicMock() + col_ref.document.return_value = pattern_ref + db.collection.return_value = col_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.update_pattern_alerted("my_pattern_id") + + db.collection.assert_called_with("error_patterns") + col_ref.document.assert_called_with("my_pattern_id") + + +# --------------------------------------------------------------------------- +# Tests: auto_resolve_pattern +# --------------------------------------------------------------------------- + + +class TestAutoResolvePattern: + """auto_resolve_pattern should set status to 'auto_resolved'.""" + + def test_sets_auto_resolved_status(self): + db = _make_db() + pattern_ref = MagicMock() + db.collection.return_value.document.return_value = pattern_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.auto_resolve_pattern("pat_123", hours_silent=24.5) + + pattern_ref.update.assert_called_once() + updated = pattern_ref.update.call_args[0][0] + assert updated["status"] == "auto_resolved" + + def test_records_hours_silent(self): + db = _make_db() + pattern_ref = MagicMock() + db.collection.return_value.document.return_value = pattern_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.auto_resolve_pattern("pat_123", hours_silent=36.0) + + updated = pattern_ref.update.call_args[0][0] + assert updated["auto_resolve_hours_silent"] == 36.0 + + +# --------------------------------------------------------------------------- +# Tests: resolve_pattern +# --------------------------------------------------------------------------- + + +class TestResolvePattern: + """resolve_pattern should set status to 'fixed' with fixed_by map.""" + + def test_sets_fixed_status(self): + db = _make_db() + pattern_ref = MagicMock() + db.collection.return_value.document.return_value = pattern_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.resolve_pattern("pat_123", pr_url="https://github.com/pr/1", note="Fixed DB") + + pattern_ref.update.assert_called_once() + updated = pattern_ref.update.call_args[0][0] + assert updated["status"] == "fixed" + + def test_includes_fixed_by_map(self): + db = _make_db() + pattern_ref = MagicMock() + db.collection.return_value.document.return_value = pattern_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + adapter.resolve_pattern("pat_123", pr_url="https://github.com/pr/42", note="Hotfix") + + updated = pattern_ref.update.call_args[0][0] + assert "fixed_by" in updated + assert updated["fixed_by"]["pr_url"] == "https://github.com/pr/42" + assert updated["fixed_by"]["note"] == "Hotfix" + + def test_works_without_pr_url(self): + db = _make_db() + pattern_ref = MagicMock() + db.collection.return_value.document.return_value = pattern_ref + + Adapter = _import_adapter() + adapter = Adapter(db=db) + + # Should not raise + adapter.resolve_pattern("pat_123") + + updated = pattern_ref.update.call_args[0][0] + assert updated["status"] == "fixed" + + +# --------------------------------------------------------------------------- +# Tests: check_auto_resolve +# --------------------------------------------------------------------------- + + +class TestCheckAutoResolve: + """check_auto_resolve should return hours_silent if threshold exceeded, None otherwise.""" + + def _make_pattern(self, last_seen_hours_ago: float, status: str = "new") -> dict: + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + last_seen = now - timedelta(hours=last_seen_hours_ago) + # Build minimal rolling_counts with 3 data points for non-fallback threshold + rolling = [ + {"ts": (last_seen - timedelta(hours=2)).isoformat(), "count": 5}, + {"ts": (last_seen - timedelta(hours=1)).isoformat(), "count": 5}, + {"ts": last_seen.isoformat(), "count": 5}, + ] + return { + "pattern_id": "pat_test", + "status": status, + "last_seen": last_seen.isoformat(), + "rolling_counts": rolling, + } + + def test_returns_none_when_recently_seen(self): + """Pattern seen 1 hour ago should NOT be auto-resolved.""" + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + pattern = self._make_pattern(last_seen_hours_ago=1.0) + with patch("backend.services.error_monitor.firestore_adapter.datetime") as mock_dt: + mock_dt.now.return_value = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + mock_dt.fromisoformat = datetime.fromisoformat + mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) + result = adapter.check_auto_resolve(pattern) + + assert result is None + + def test_returns_hours_silent_when_threshold_exceeded(self): + """Pattern silent for 100h should be returned for auto-resolve.""" + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + # Use fallback threshold (48h) by providing <3 rolling_counts + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + last_seen = now - timedelta(hours=100) + pattern = { + "pattern_id": "pat_old", + "status": "new", + "last_seen": last_seen.isoformat(), + "rolling_counts": [], # <3 points → fallback = 48h + } + + with patch("backend.services.error_monitor.firestore_adapter.datetime") as mock_dt: + mock_dt.now.return_value = now + mock_dt.fromisoformat = datetime.fromisoformat + mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) + result = adapter.check_auto_resolve(pattern) + + assert result is not None + assert result > 48.0 # hours_silent should be ~100 + + def test_muted_pattern_returns_none(self): + """Muted patterns should never be auto-resolved.""" + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + last_seen = now - timedelta(hours=100) + pattern = { + "pattern_id": "pat_muted", + "status": "muted", + "last_seen": last_seen.isoformat(), + "rolling_counts": [], + } + + with patch("backend.services.error_monitor.firestore_adapter.datetime") as mock_dt: + mock_dt.now.return_value = now + mock_dt.fromisoformat = datetime.fromisoformat + mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) + result = adapter.check_auto_resolve(pattern) + + assert result is None + + +# --------------------------------------------------------------------------- +# Tests: _prune_rolling_counts +# --------------------------------------------------------------------------- + + +class TestPruneRollingCounts: + """_prune_rolling_counts should remove entries older than 7 days.""" + + def test_removes_old_entries(self): + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [ + {"ts": (now - timedelta(days=10)).isoformat(), "count": 5}, # old + {"ts": (now - timedelta(days=8)).isoformat(), "count": 3}, # old + {"ts": (now - timedelta(days=3)).isoformat(), "count": 7}, # recent + {"ts": now.isoformat(), "count": 2}, # recent + ] + + result = adapter._prune_rolling_counts(rolling) + + assert len(result) == 2 + assert result[0]["count"] == 7 + assert result[1]["count"] == 2 + + def test_keeps_all_recent_entries(self): + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + now = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + rolling = [ + {"ts": (now - timedelta(days=1)).isoformat(), "count": 1}, + {"ts": (now - timedelta(days=2)).isoformat(), "count": 2}, + {"ts": now.isoformat(), "count": 3}, + ] + + result = adapter._prune_rolling_counts(rolling) + assert len(result) == 3 + + def test_empty_list_returns_empty(self): + Adapter = _import_adapter() + adapter = Adapter(db=_make_db()) + + result = adapter._prune_rolling_counts([]) + assert result == [] + + +# --------------------------------------------------------------------------- +# Tests: get_patterns_for_auto_resolve +# --------------------------------------------------------------------------- + + +class TestGetPatternsForAutoResolve: + """get_patterns_for_auto_resolve should return new and acknowledged patterns.""" + + def test_queries_correct_statuses(self): + db = _make_db() + query_mock = MagicMock() + query_mock.stream.return_value = [] + db.collection.return_value.where.return_value = query_mock + + Adapter = _import_adapter() + adapter = Adapter(db=db) + adapter.get_patterns_for_auto_resolve() + + # Collection should be error_patterns + db.collection.assert_called_with("error_patterns") + + def test_returns_list_of_dicts(self): + db = _make_db() + snap = _make_doc_snapshot(exists=True, data={"pattern_id": "p1", "status": "new"}) + query_mock = MagicMock() + query_mock.stream.return_value = [snap] + db.collection.return_value.where.return_value = query_mock + + Adapter = _import_adapter() + adapter = Adapter(db=db) + results = adapter.get_patterns_for_auto_resolve() + + assert isinstance(results, list) + assert results[0]["pattern_id"] == "p1" diff --git a/tests/unit/services/error_monitor/test_known_issues.py b/tests/unit/services/error_monitor/test_known_issues.py new file mode 100644 index 00000000..75002df3 --- /dev/null +++ b/tests/unit/services/error_monitor/test_known_issues.py @@ -0,0 +1,445 @@ +"""Tests for error_monitor.known_issues module.""" + +import pytest + + +class TestIgnoreReasonDataclass: + """IgnoreReason dataclass should hold pattern_name and reason fields.""" + + def test_ignore_reason_has_pattern_name(self): + from backend.services.error_monitor.known_issues import IgnoreReason + + ir = IgnoreReason(pattern_name="startup_probe", reason="Cloud Run cold start") + assert ir.pattern_name == "startup_probe" + + def test_ignore_reason_has_reason(self): + from backend.services.error_monitor.known_issues import IgnoreReason + + ir = IgnoreReason(pattern_name="startup_probe", reason="Cloud Run cold start") + assert ir.reason == "Cloud Run cold start" + + def test_ignore_reason_is_dataclass_equality(self): + from backend.services.error_monitor.known_issues import IgnoreReason + + a = IgnoreReason(pattern_name="foo", reason="bar") + b = IgnoreReason(pattern_name="foo", reason="bar") + assert a == b + + def test_ignore_reason_inequality(self): + from backend.services.error_monitor.known_issues import IgnoreReason + + a = IgnoreReason(pattern_name="foo", reason="bar") + b = IgnoreReason(pattern_name="baz", reason="bar") + assert a != b + + +class TestStartupProbePattern: + """startup_probe — 'startup probe failed' should be ignored (Cloud Run cold start).""" + + def test_startup_probe_exact(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "startup probe failed") + assert result is not None + assert result.pattern_name == "startup_probe" + + def test_startup_probe_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "Startup Probe Failed") + assert result is not None + assert result.pattern_name == "startup_probe" + + def test_startup_probe_in_longer_message(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Readiness probe failed: startup probe failed for container api", + ) + assert result is not None + assert result.pattern_name == "startup_probe" + + def test_startup_probe_any_service(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("audio-separator", "startup probe failed") + assert result is not None + assert result.pattern_name == "startup_probe" + + +class TestReadyConditionPattern: + """ready_condition — 'ready condition status changed' should be ignored (deploy event).""" + + def test_ready_condition_exact(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "ready condition status changed") + assert result is not None + assert result.pattern_name == "ready_condition" + + def test_ready_condition_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "Ready Condition Status Changed") + assert result is not None + assert result.pattern_name == "ready_condition" + + def test_ready_condition_with_context(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Service karaoke-backend: ready condition status changed to True", + ) + assert result is not None + assert result.pattern_name == "ready_condition" + + +class TestSpotPreemptionPattern: + """spot_preemption — 'instance was preempted' should be ignored (GitHub runner spot VMs).""" + + def test_spot_preemption_exact(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("encoding-worker-a", "instance was preempted") + assert result is not None + assert result.pattern_name == "spot_preemption" + + def test_spot_preemption_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("encoding-worker-a", "Instance Was Preempted") + assert result is not None + assert result.pattern_name == "spot_preemption" + + def test_spot_preemption_in_sentence(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "encoding-worker-b", + "GCE: instance was preempted, shutting down", + ) + assert result is not None + assert result.pattern_name == "spot_preemption" + + +class TestIdleShutdownPattern: + """idle_shutdown — 'stopping idle encoding worker' should be ignored (expected behavior).""" + + def test_idle_shutdown_exact(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("encoding_worker_idle", "stopping idle encoding worker") + assert result is not None + assert result.pattern_name == "idle_shutdown" + + def test_idle_shutdown_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("encoding_worker_idle", "Stopping Idle Encoding Worker") + assert result is not None + assert result.pattern_name == "idle_shutdown" + + def test_idle_shutdown_with_context(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "encoding_worker_idle", + "stopping idle encoding worker after 30 minutes of inactivity", + ) + assert result is not None + assert result.pattern_name == "idle_shutdown" + + +class TestHealthCheck404Pattern: + """health_check_404 — GET/HEAD /health with 404/not found should be ignored (load balancer).""" + + def test_health_check_get_404(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", 'GET /health 404') + assert result is not None + assert result.pattern_name == "health_check_404" + + def test_health_check_head_404(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", 'HEAD /health 404') + assert result is not None + assert result.pattern_name == "health_check_404" + + def test_health_check_get_not_found(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", 'GET /health not found') + assert result is not None + assert result.pattern_name == "health_check_404" + + def test_health_check_head_not_found(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", 'HEAD /health not found') + assert result is not None + assert result.pattern_name == "health_check_404" + + def test_health_check_uppercase_method(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "GET /health 404 Not Found") + assert result is not None + assert result.pattern_name == "health_check_404" + + def test_health_check_in_log_line(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + '10.0.0.1 - - [01/Jan/2024] "GET /health HTTP/1.1" 404 0', + ) + assert result is not None + assert result.pattern_name == "health_check_404" + + def test_non_health_404_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + # A 404 for a real endpoint should NOT be ignored + result = should_ignore("karaoke-backend", "GET /api/jobs/abc 404") + assert result is None + + def test_health_200_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + # /health returning 200 is not an error, but should still not trigger this pattern + result = should_ignore("karaoke-backend", "GET /health 200") + assert result is None + + +class TestSchedulerRetryPattern: + """scheduler_retry — 'cloud scheduler.*retry' should be ignored (transient).""" + + def test_scheduler_retry_basic(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("runner_manager", "cloud scheduler retry attempt 1") + assert result is not None + assert result.pattern_name == "scheduler_retry" + + def test_scheduler_retry_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("runner_manager", "Cloud Scheduler Retry") + assert result is not None + assert result.pattern_name == "scheduler_retry" + + def test_scheduler_retry_with_details(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "runner_manager", + "cloud scheduler job failed, will retry in 60s", + ) + assert result is not None + assert result.pattern_name == "scheduler_retry" + + def test_cloud_scheduler_without_retry_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + # "cloud scheduler" alone (without "retry") should NOT be suppressed + result = should_ignore("runner_manager", "cloud scheduler job executed successfully") + assert result is None + + +class TestRunnerStartupPattern: + """runner_startup — 'github.runner.*(starting|stopping|idle)' should be ignored (lifecycle).""" + + def test_runner_starting(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("runner_manager", "github.runner.001 starting") + assert result is not None + assert result.pattern_name == "runner_startup" + + def test_runner_stopping(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("runner_manager", "github.runner.abc stopping") + assert result is not None + assert result.pattern_name == "runner_startup" + + def test_runner_idle(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("runner_manager", "github.runner.xyz idle") + assert result is not None + assert result.pattern_name == "runner_startup" + + def test_runner_starting_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("runner_manager", "GitHub.Runner.001 Starting") + assert result is not None + assert result.pattern_name == "runner_startup" + + def test_runner_other_state_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + # A GitHub runner in an unexpected state should NOT be ignored + result = should_ignore("runner_manager", "github.runner.001 crashed") + assert result is None + + +class TestContainerShutdownPattern: + """container_shutdown — 'container called exit(0)' should be ignored (clean shutdown).""" + + def test_container_exit_0(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "container called exit(0)") + assert result is not None + assert result.pattern_name == "container_shutdown" + + def test_container_exit_0_uppercase(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "Container Called Exit(0)") + assert result is not None + assert result.pattern_name == "container_shutdown" + + def test_container_exit_0_in_sentence(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Cloud Run instance terminated: container called exit(0)", + ) + assert result is not None + assert result.pattern_name == "container_shutdown" + + def test_container_exit_nonzero_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + # exit(1) = error, should NOT be ignored + result = should_ignore("karaoke-backend", "container called exit(1)") + assert result is None + + def test_container_exit_2_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "container called exit(2)") + assert result is None + + +class TestRealErrorsNotIgnored: + """Real errors should never be silently ignored.""" + + def test_firestore_deadline_exceeded(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Firestore transaction failed: DEADLINE_EXCEEDED", + ) + assert result is None + + def test_audio_separation_oom(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "audio-separator", + "CUDA out of memory. Tried to allocate 2.00 GiB", + ) + assert result is None + + def test_payment_stripe_error(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Stripe charge failed: card_declined", + ) + assert result is None + + def test_gcs_permission_denied(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Permission denied accessing gs://nomadkaraoke-prod/output.mp4", + ) + assert result is None + + def test_worker_timeout(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "video-encoding-job", + "Worker timed out after 3600 seconds", + ) + assert result is None + + def test_unhandled_exception(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore( + "karaoke-backend", + "Unhandled exception: KeyError 'job_id'", + ) + assert result is None + + +class TestUnknownServiceMessages: + """Messages from unlisted/unknown services should use pattern matching, not service-gating.""" + + def test_unknown_service_known_pattern_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + # The patterns apply regardless of service name + result = should_ignore("some-unknown-service", "startup probe failed") + assert result is not None + + def test_unknown_service_real_error_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("some-unknown-service", "unhandled exception in request handler") + assert result is None + + def test_empty_service_name_with_known_pattern(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("", "startup probe failed") + assert result is not None + + def test_empty_message_not_ignored(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "") + assert result is None + + +class TestCaseInsensitiveMatching: + """All patterns should match case-insensitively.""" + + def test_startup_probe_all_caps(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "STARTUP PROBE FAILED") + assert result is not None + + def test_ready_condition_mixed_case(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "Ready CONDITION Status CHANGED") + assert result is not None + + def test_idle_shutdown_mixed_case(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("encoding_worker_idle", "STOPPING idle ENCODING worker") + assert result is not None + + def test_container_shutdown_mixed_case(self): + from backend.services.error_monitor.known_issues import should_ignore + + result = should_ignore("karaoke-backend", "Container Called EXIT(0)") + assert result is not None diff --git a/tests/unit/services/error_monitor/test_llm_analysis.py b/tests/unit/services/error_monitor/test_llm_analysis.py new file mode 100644 index 00000000..410e121b --- /dev/null +++ b/tests/unit/services/error_monitor/test_llm_analysis.py @@ -0,0 +1,507 @@ +"""Tests for error_monitor.llm_analysis module. + +All tests mock _call_llm to avoid real LLM calls. +""" + +import json +from unittest.mock import patch + +import pytest + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +SAMPLE_PATTERNS = [ + { + "service": "karaoke-backend", + "normalized_message": "Connection refused to Firestore", + "count": 15, + "first_seen": "2026-04-10T10:00:00Z", + "last_seen": "2026-04-10T10:30:00Z", + }, + { + "service": "audio-separation-job", + "normalized_message": "CUDA out of memory error", + "count": 8, + "first_seen": "2026-04-10T10:05:00Z", + "last_seen": "2026-04-10T10:25:00Z", + }, + { + "service": "audio-download-job", + "normalized_message": "flacfetch-vm connection timeout", + "count": 5, + "first_seen": "2026-04-10T10:10:00Z", + "last_seen": "2026-04-10T10:20:00Z", + }, +] + +VALID_INCIDENTS_JSON = json.dumps( + { + "incidents": [ + { + "title": "Firestore connectivity failure", + "root_cause": "Firestore is unavailable or misconfigured", + "severity": "P1", + "suggested_fix": "Check Firestore service health and credentials", + "primary_service": "karaoke-backend", + "pattern_indices": [0], + }, + { + "title": "GPU audio separation failure", + "root_cause": "GPU out of memory on audio-separation-job", + "severity": "P2", + "suggested_fix": "Check GPU memory limits and job queue depth", + "primary_service": "audio-separation-job", + "pattern_indices": [1, 2], + }, + ] + } +) + +VALID_DUPLICATES_JSON = json.dumps( + { + "duplicates": [ + { + "canonical_index": 0, + "duplicate_indices": [1], + "reason": "Both errors indicate Firestore connection issues", + } + ] + } +) + + +# --------------------------------------------------------------------------- +# Tests for _parse_llm_response +# --------------------------------------------------------------------------- + + +class TestParseLlmResponse: + """Unit tests for the _parse_llm_response helper.""" + + def test_valid_json_returns_incident_analysis(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + result = _parse_llm_response(VALID_INCIDENTS_JSON) + assert result is not None + assert len(result.incidents) == 2 + assert result.used_llm is True + + def test_valid_json_incident_fields(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + result = _parse_llm_response(VALID_INCIDENTS_JSON) + assert result is not None + first = result.incidents[0] + assert first.title == "Firestore connectivity failure" + assert first.root_cause == "Firestore is unavailable or misconfigured" + assert first.severity == "P1" + assert first.primary_service == "karaoke-backend" + assert first.pattern_indices == [0] + + def test_invalid_json_returns_none(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + result = _parse_llm_response("not valid json {{{") + assert result is None + + def test_missing_incidents_key_returns_none(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + result = _parse_llm_response(json.dumps({"data": []})) + assert result is None + + def test_markdown_code_block_stripped(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + wrapped = f"```json\n{VALID_INCIDENTS_JSON}\n```" + result = _parse_llm_response(wrapped) + assert result is not None + assert len(result.incidents) == 2 + + def test_markdown_code_block_without_language_stripped(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + wrapped = f"```\n{VALID_INCIDENTS_JSON}\n```" + result = _parse_llm_response(wrapped) + assert result is not None + assert len(result.incidents) == 2 + + def test_empty_string_returns_none(self): + from backend.services.error_monitor.llm_analysis import _parse_llm_response + + result = _parse_llm_response("") + assert result is None + + def test_incidents_is_list_of_incident_objects(self): + from backend.services.error_monitor.llm_analysis import ( + _parse_llm_response, + Incident, + ) + + result = _parse_llm_response(VALID_INCIDENTS_JSON) + assert result is not None + for incident in result.incidents: + assert isinstance(incident, Incident) + + +# --------------------------------------------------------------------------- +# Tests for analyze_patterns +# --------------------------------------------------------------------------- + + +class TestAnalyzePatterns: + """Unit tests for analyze_patterns.""" + + def test_returns_none_for_fewer_than_min_patterns(self): + from backend.services.error_monitor.llm_analysis import analyze_patterns + from backend.services.error_monitor.config import MIN_PATTERNS_FOR_ANALYSIS + + # Build a list with fewer patterns than the minimum + few_patterns = SAMPLE_PATTERNS[: MIN_PATTERNS_FOR_ANALYSIS - 1] + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm" + ) as mock_call: + result = analyze_patterns(few_patterns) + + assert result is None + mock_call.assert_not_called() + + def test_groups_patterns_into_incidents(self): + from backend.services.error_monitor.llm_analysis import analyze_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=VALID_INCIDENTS_JSON, + ): + result = analyze_patterns(SAMPLE_PATTERNS) + + assert result is not None + assert len(result.incidents) == 2 + assert result.used_llm is True + + def test_incident_has_correct_fields(self): + from backend.services.error_monitor.llm_analysis import analyze_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=VALID_INCIDENTS_JSON, + ): + result = analyze_patterns(SAMPLE_PATTERNS) + + assert result is not None + incident = result.incidents[0] + assert incident.title == "Firestore connectivity failure" + assert incident.severity == "P1" + assert incident.primary_service == "karaoke-backend" + + def test_falls_back_on_llm_exception(self): + from backend.services.error_monitor.llm_analysis import analyze_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + side_effect=Exception("LLM unavailable"), + ): + result = analyze_patterns(SAMPLE_PATTERNS) + + # Should fall back to service grouping, not None + assert result is not None + assert result.used_llm is False + + def test_fallback_groups_by_service(self): + from backend.services.error_monitor.llm_analysis import analyze_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + side_effect=Exception("LLM unavailable"), + ): + result = analyze_patterns(SAMPLE_PATTERNS) + + assert result is not None + # Each unique service should map to one incident + services = {inc.primary_service for inc in result.incidents} + expected_services = {p["service"] for p in SAMPLE_PATTERNS} + assert services == expected_services + + def test_falls_back_on_invalid_llm_response(self): + from backend.services.error_monitor.llm_analysis import analyze_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value="invalid json", + ): + result = analyze_patterns(SAMPLE_PATTERNS) + + # Should fall back to service grouping on parse failure + assert result is not None + assert result.used_llm is False + + def test_exact_min_patterns_triggers_analysis(self): + """Exactly MIN_PATTERNS_FOR_ANALYSIS patterns should trigger the LLM.""" + from backend.services.error_monitor.llm_analysis import analyze_patterns + from backend.services.error_monitor.config import MIN_PATTERNS_FOR_ANALYSIS + + # Ensure we have exactly MIN_PATTERNS_FOR_ANALYSIS patterns + exactly_min = SAMPLE_PATTERNS[:MIN_PATTERNS_FOR_ANALYSIS] + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=VALID_INCIDENTS_JSON, + ) as mock_call: + result = analyze_patterns(exactly_min) + + mock_call.assert_called_once() + + +# --------------------------------------------------------------------------- +# Tests for _fallback_group_by_service +# --------------------------------------------------------------------------- + + +class TestFallbackGroupByService: + """Unit tests for _fallback_group_by_service.""" + + def test_groups_by_service(self): + from backend.services.error_monitor.llm_analysis import _fallback_group_by_service + + result = _fallback_group_by_service(SAMPLE_PATTERNS) + services = {inc.primary_service for inc in result.incidents} + assert services == {"karaoke-backend", "audio-separation-job", "audio-download-job"} + + def test_used_llm_is_false(self): + from backend.services.error_monitor.llm_analysis import _fallback_group_by_service + + result = _fallback_group_by_service(SAMPLE_PATTERNS) + assert result.used_llm is False + + def test_severity_is_p2(self): + from backend.services.error_monitor.llm_analysis import _fallback_group_by_service + + result = _fallback_group_by_service(SAMPLE_PATTERNS) + for incident in result.incidents: + assert incident.severity == "P2" + + def test_pattern_indices_are_populated(self): + from backend.services.error_monitor.llm_analysis import _fallback_group_by_service + + # Two patterns for the same service + two_patterns = [ + { + "service": "karaoke-backend", + "normalized_message": "Error A", + "count": 3, + }, + { + "service": "karaoke-backend", + "normalized_message": "Error B", + "count": 2, + }, + ] + result = _fallback_group_by_service(two_patterns) + assert len(result.incidents) == 1 + assert result.incidents[0].pattern_indices == [0, 1] + + def test_empty_patterns_returns_empty_incidents(self): + from backend.services.error_monitor.llm_analysis import _fallback_group_by_service + + result = _fallback_group_by_service([]) + assert result.incidents == [] + assert result.used_llm is False + + +# --------------------------------------------------------------------------- +# Tests for find_duplicate_patterns +# --------------------------------------------------------------------------- + + +class TestFindDuplicatePatterns: + """Unit tests for find_duplicate_patterns.""" + + def test_returns_empty_for_empty_new_patterns(self): + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm" + ) as mock_call: + result = find_duplicate_patterns([], SAMPLE_PATTERNS) + + assert result == [] + mock_call.assert_not_called() + + def test_returns_empty_for_empty_existing_patterns(self): + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm" + ) as mock_call: + result = find_duplicate_patterns(SAMPLE_PATTERNS, []) + + assert result == [] + mock_call.assert_not_called() + + def test_identifies_duplicates(self): + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + new_patterns = [ + { + "service": "karaoke-backend", + "normalized_message": "Cannot reach Firestore endpoint", + }, + { + "service": "karaoke-backend", + "normalized_message": "Firestore deadline exceeded", + }, + ] + existing_patterns = [ + { + "service": "karaoke-backend", + "normalized_message": "Connection refused to Firestore", + } + ] + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=VALID_DUPLICATES_JSON, + ): + result = find_duplicate_patterns(new_patterns, existing_patterns) + + assert len(result) == 1 + assert result[0].canonical_index == 0 + assert result[0].duplicate_indices == [1] + assert "Firestore" in result[0].reason + + def test_returns_empty_on_no_duplicates_in_response(self): + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + no_dups_json = json.dumps({"duplicates": []}) + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=no_dups_json, + ): + result = find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:]) + + assert result == [] + + def test_returns_empty_on_llm_failure(self): + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + side_effect=Exception("LLM unavailable"), + ): + result = find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:]) + + assert result == [] + + def test_returns_empty_on_invalid_response(self): + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value="not json", + ): + result = find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:]) + + assert result == [] + + def test_duplicate_group_fields(self): + from backend.services.error_monitor.llm_analysis import ( + find_duplicate_patterns, + DuplicateGroup, + ) + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=VALID_DUPLICATES_JSON, + ): + result = find_duplicate_patterns(SAMPLE_PATTERNS[:2], SAMPLE_PATTERNS[2:]) + + assert len(result) == 1 + assert isinstance(result[0], DuplicateGroup) + + def test_uses_low_temperature(self): + """find_duplicate_patterns should call _call_llm with temperature=0.1.""" + from backend.services.error_monitor.llm_analysis import find_duplicate_patterns + + with patch( + "backend.services.error_monitor.llm_analysis._call_llm", + return_value=VALID_DUPLICATES_JSON, + ) as mock_call: + find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:]) + + # Check the temperature kwarg was 0.1 + call_kwargs = mock_call.call_args + assert call_kwargs is not None + # temperature is the 3rd positional arg or keyword arg + if call_kwargs.kwargs.get("temperature") is not None: + assert call_kwargs.kwargs["temperature"] == 0.1 + else: + # positional: (system_prompt, user_prompt, temperature) + assert call_kwargs.args[2] == 0.1 + + +# --------------------------------------------------------------------------- +# Tests for dataclasses +# --------------------------------------------------------------------------- + + +class TestDataclasses: + """Tests for the exported dataclasses.""" + + def test_incident_dataclass(self): + from backend.services.error_monitor.llm_analysis import Incident + + inc = Incident( + title="Test", + root_cause="Root cause", + severity="P1", + suggested_fix="Fix it", + primary_service="karaoke-backend", + pattern_indices=[0, 1], + ) + assert inc.title == "Test" + assert inc.severity == "P1" + assert inc.pattern_indices == [0, 1] + + def test_incident_nullable_fields(self): + from backend.services.error_monitor.llm_analysis import Incident + + inc = Incident( + title="Test", + root_cause=None, + severity="P3", + suggested_fix=None, + primary_service="audio-separator", + pattern_indices=[2], + ) + assert inc.root_cause is None + assert inc.suggested_fix is None + + def test_incident_analysis_dataclass(self): + from backend.services.error_monitor.llm_analysis import IncidentAnalysis, Incident + + ia = IncidentAnalysis(incidents=[]) + assert ia.incidents == [] + assert ia.used_llm is True # default + + def test_incident_analysis_used_llm_false(self): + from backend.services.error_monitor.llm_analysis import IncidentAnalysis + + ia = IncidentAnalysis(incidents=[], used_llm=False) + assert ia.used_llm is False + + def test_duplicate_group_dataclass(self): + from backend.services.error_monitor.llm_analysis import DuplicateGroup + + dg = DuplicateGroup( + canonical_index=0, + duplicate_indices=[1, 2], + reason="Same root cause", + ) + assert dg.canonical_index == 0 + assert dg.duplicate_indices == [1, 2] + assert dg.reason == "Same root cause" diff --git a/tests/unit/services/error_monitor/test_monitor.py b/tests/unit/services/error_monitor/test_monitor.py new file mode 100644 index 00000000..34962fc3 --- /dev/null +++ b/tests/unit/services/error_monitor/test_monitor.py @@ -0,0 +1,954 @@ +"""Tests for error_monitor.monitor module. + +All external dependencies (Cloud Logging, Firestore, Discord, LLM, Secret Manager) +are mocked — no real GCP connections are made. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from unittest.mock import MagicMock, call, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Import helpers +# --------------------------------------------------------------------------- + + +def _import_monitor_module(): + from backend.services.error_monitor import monitor as m + + return m + + +def _import_error_monitor_class(): + from backend.services.error_monitor.monitor import ErrorMonitor + + return ErrorMonitor + + +# --------------------------------------------------------------------------- +# Helper to build a minimal mock log entry +# --------------------------------------------------------------------------- + + +def _make_entry( + payload=None, + resource_type="cloud_run_revision", + service_name="karaoke-backend", + job_name=None, + function_name=None, + instance_id=None, + resource_name=None, +): + """Return a mock Cloud Logging entry.""" + entry = MagicMock() + entry.payload = payload + entry.resource = MagicMock() + entry.resource.type = resource_type + entry.resource.labels = { + "service_name": service_name, + "job_name": job_name or "", + "function_name": function_name or "", + "instance_id": instance_id or "", + } + entry.labels = {} + if resource_name: + entry.labels["compute.googleapis.com/resource_name"] = resource_name + return entry + + +# --------------------------------------------------------------------------- +# Tests: _build_log_filter +# --------------------------------------------------------------------------- + + +class TestBuildLogFilter: + """_build_log_filter should produce correct Cloud Logging filter strings.""" + + def test_cloud_run_revision_filter(self): + m = _import_monitor_module() + result = m._build_log_filter("cloud_run_revision", ["svc-a", "svc-b"], 15) + assert 'resource.type="cloud_run_revision"' in result + assert "severity>=ERROR" in result + assert "svc-a" in result + assert "svc-b" in result + assert "resource.labels.service_name" in result + + def test_cloud_run_job_filter(self): + m = _import_monitor_module() + result = m._build_log_filter("cloud_run_job", ["job-x"], 15) + assert 'resource.type="cloud_run_job"' in result + assert "severity>=ERROR" in result + assert "job-x" in result + assert "resource.labels.job_name" in result + + def test_gce_instance_filter(self): + m = _import_monitor_module() + result = m._build_log_filter("gce_instance", ["enc-worker-a"], 15) + assert 'resource.type="gce_instance"' in result + assert "severity>=ERROR" in result + assert "enc-worker-a" in result + assert 'compute.googleapis.com/resource_name' in result + + def test_cloud_function_filter(self): + m = _import_monitor_module() + result = m._build_log_filter("cloud_function", ["fn-one", "fn-two"], 10) + assert 'resource.type="cloud_function"' in result + assert "fn-one" in result + assert "fn-two" in result + + def test_filter_contains_timestamp(self): + m = _import_monitor_module() + result = m._build_log_filter("cloud_run_revision", ["svc"], 30) + assert "timestamp>=" in result + + def test_single_resource_no_or_clause_needed(self): + m = _import_monitor_module() + result = m._build_log_filter("cloud_run_revision", ["only-svc"], 15) + assert "only-svc" in result + + +# --------------------------------------------------------------------------- +# Tests: _extract_message +# --------------------------------------------------------------------------- + + +class TestExtractMessage: + """_extract_message should handle all payload shapes.""" + + def test_text_payload_string_returned_directly(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = "Something went wrong" + result = m._extract_message(entry) + assert result == "Something went wrong" + + def test_dict_payload_message_key(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = {"message": "DB connection refused"} + result = m._extract_message(entry) + assert result == "DB connection refused" + + def test_dict_payload_textPayload_key(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = {"textPayload": "File not found: /tmp/foo"} + result = m._extract_message(entry) + assert result == "File not found: /tmp/foo" + + def test_dict_payload_error_key(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = {"error": "timeout after 30s"} + result = m._extract_message(entry) + assert result == "timeout after 30s" + + def test_dict_payload_msg_key(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = {"msg": "panic: nil pointer dereference"} + result = m._extract_message(entry) + assert result == "panic: nil pointer dereference" + + def test_none_payload_returns_none(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = None + result = m._extract_message(entry) + assert result is None + + def test_dict_payload_no_known_key_returns_none(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = {"severity": "ERROR", "timestamp": "2026-04-10T10:00:00Z"} + result = m._extract_message(entry) + assert result is None + + def test_empty_string_payload_treated_as_text(self): + m = _import_monitor_module() + entry = MagicMock() + entry.payload = "" + result = m._extract_message(entry) + # Empty string is falsy — should return the string or None depending on impl + # As long as it doesn't crash; either return value is acceptable + assert result is None or result == "" + + +# --------------------------------------------------------------------------- +# Tests: _extract_service_name +# --------------------------------------------------------------------------- + + +class TestExtractServiceName: + """_extract_service_name should return the correct label for each resource type.""" + + def test_cloud_run_revision_returns_service_name(self): + m = _import_monitor_module() + entry = MagicMock() + entry.resource = MagicMock() + entry.resource.labels = {"service_name": "karaoke-backend"} + result = m._extract_service_name(entry, "cloud_run_revision") + assert result == "karaoke-backend" + + def test_cloud_run_job_returns_job_name(self): + m = _import_monitor_module() + entry = MagicMock() + entry.resource = MagicMock() + entry.resource.labels = {"job_name": "video-encoding-job"} + result = m._extract_service_name(entry, "cloud_run_job") + assert result == "video-encoding-job" + + def test_cloud_function_returns_function_name(self): + m = _import_monitor_module() + entry = MagicMock() + entry.resource = MagicMock() + entry.resource.labels = {"function_name": "gdrive-validator"} + result = m._extract_service_name(entry, "cloud_function") + assert result == "gdrive-validator" + + def test_gce_instance_returns_resource_name_from_labels(self): + m = _import_monitor_module() + entry = MagicMock() + entry.resource = MagicMock() + entry.resource.labels = {} + entry.labels = {"compute.googleapis.com/resource_name": "encoding-worker-a"} + result = m._extract_service_name(entry, "gce_instance") + assert result == "encoding-worker-a" + + def test_gce_instance_falls_back_to_instance_id(self): + m = _import_monitor_module() + entry = MagicMock() + entry.resource = MagicMock() + entry.resource.labels = {"instance_id": "1234567890"} + entry.labels = {} + result = m._extract_service_name(entry, "gce_instance") + assert result == "1234567890" + + def test_missing_labels_returns_unknown(self): + m = _import_monitor_module() + entry = MagicMock() + entry.resource = MagicMock() + entry.resource.labels = {} + entry.labels = {} + # Should not raise — returns some fallback string + result = m._extract_service_name(entry, "cloud_run_revision") + assert isinstance(result, str) + + +# --------------------------------------------------------------------------- +# Tests: _classify_resource_type +# --------------------------------------------------------------------------- + + +class TestClassifyResourceType: + """_classify_resource_type should map log resource types to canonical types.""" + + def test_cloud_run_revision_mapped(self): + m = _import_monitor_module() + assert m._classify_resource_type("cloud_run_revision") == "cloud_run_service" + + def test_cloud_run_job_mapped(self): + m = _import_monitor_module() + assert m._classify_resource_type("cloud_run_job") == "cloud_run_job" + + def test_cloud_function_mapped(self): + m = _import_monitor_module() + assert m._classify_resource_type("cloud_function") == "cloud_function" + + def test_gce_instance_mapped(self): + m = _import_monitor_module() + assert m._classify_resource_type("gce_instance") == "gce_instance" + + def test_unknown_type_returns_original_or_unknown(self): + m = _import_monitor_module() + result = m._classify_resource_type("unknown_type") + assert isinstance(result, str) + + +# --------------------------------------------------------------------------- +# Tests: _group_by_pattern (via _run_monitor_cycle logic) +# --------------------------------------------------------------------------- + + +class TestGroupByPattern: + """_group_by_pattern should correctly group entries by hash and count duplicates.""" + + def _make_monitor(self): + """Create an ErrorMonitor instance without calling __init__ (no GCP calls).""" + ErrorMonitor = _import_error_monitor_class() + monitor = ErrorMonitor.__new__(ErrorMonitor) + monitor.logging_client = MagicMock() + monitor.firestore_adapter = MagicMock() + monitor.discord = MagicMock() + return monitor + + def test_groups_entries_by_hash(self): + monitor = self._make_monitor() + m = _import_monitor_module() + + entries = [ + {"service": "svc-a", "normalized": "Connection refused", "message": "msg1"}, + {"service": "svc-a", "normalized": "Connection refused", "message": "msg2"}, + {"service": "svc-b", "normalized": "Timeout error", "message": "msg3"}, + ] + result = m._group_by_pattern(entries) + # 2 unique patterns + assert len(result) == 2 + + def test_counts_duplicates_correctly(self): + monitor = self._make_monitor() + m = _import_monitor_module() + + entries = [ + {"service": "svc-a", "normalized": "Connection refused", "message": "msg1"}, + {"service": "svc-a", "normalized": "Connection refused", "message": "msg2"}, + {"service": "svc-a", "normalized": "Connection refused", "message": "msg3"}, + ] + result = m._group_by_pattern(entries) + assert len(result) == 1 + group = list(result.values())[0] + assert group["count"] == 3 + + def test_each_group_has_sample_message(self): + m = _import_monitor_module() + + entries = [ + {"service": "svc-a", "normalized": "DB error", "message": "DB error at line 42"}, + ] + result = m._group_by_pattern(entries) + group = list(result.values())[0] + assert "sample_message" in group + assert group["sample_message"] == "DB error at line 42" + + +# --------------------------------------------------------------------------- +# Tests: _is_spike +# --------------------------------------------------------------------------- + + +class TestIsSpike: + """_is_spike should detect spikes based on multiplier and minimum count.""" + + def _make_monitor(self): + ErrorMonitor = _import_error_monitor_class() + monitor = ErrorMonitor.__new__(ErrorMonitor) + monitor.logging_client = MagicMock() + monitor.firestore_adapter = MagicMock() + monitor.discord = MagicMock() + return monitor + + def test_detects_spike_when_count_exceeds_multiplier_and_min(self): + m = _import_monitor_module() + # rolling_counts of 1 each for the past 5 periods, current count = 10 + # avg = 1.0, SPIKE_MULTIPLIER=5.0, so 10 > 5 and 10 >= SPIKE_MIN_COUNT=5 + rolling_counts = [{"count": 1}, {"count": 1}, {"count": 1}, {"count": 1}, {"count": 1}] + assert m._is_spike(current_count=10, rolling_counts=rolling_counts) is True + + def test_rejects_non_spike_count(self): + m = _import_monitor_module() + rolling_counts = [{"count": 4}, {"count": 4}, {"count": 4}, {"count": 4}] + # avg = 4.0, 5x = 20, current=5 → not a spike + assert m._is_spike(current_count=5, rolling_counts=rolling_counts) is False + + def test_rejects_spike_below_min_count(self): + m = _import_monitor_module() + # avg = 0.5, 5x = 2.5, current = 3 → exceeds multiplier but below SPIKE_MIN_COUNT + rolling_counts = [{"count": 1}, {"count": 0}] + assert m._is_spike(current_count=3, rolling_counts=rolling_counts) is False + + def test_empty_rolling_counts_not_a_spike(self): + m = _import_monitor_module() + assert m._is_spike(current_count=100, rolling_counts=[]) is False + + +# --------------------------------------------------------------------------- +# Tests: _rolling_average +# --------------------------------------------------------------------------- + + +class TestRollingAverage: + """_rolling_average should compute mean of rolling_counts.""" + + def test_computes_average_correctly(self): + m = _import_monitor_module() + rolling = [{"count": 2}, {"count": 4}, {"count": 6}] + result = m._rolling_average(rolling) + assert result == pytest.approx(4.0) + + def test_single_entry_returns_that_value(self): + m = _import_monitor_module() + rolling = [{"count": 7}] + result = m._rolling_average(rolling) + assert result == pytest.approx(7.0) + + def test_empty_list_returns_zero(self): + m = _import_monitor_module() + result = m._rolling_average([]) + assert result == 0.0 + + def test_counts_with_zeros(self): + m = _import_monitor_module() + rolling = [{"count": 0}, {"count": 0}, {"count": 10}] + result = m._rolling_average(rolling) + assert result == pytest.approx(10 / 3) + + +# --------------------------------------------------------------------------- +# Tests: _run_monitor_cycle end-to-end +# --------------------------------------------------------------------------- + + +def _make_monitor_with_mocks(): + """Return an ErrorMonitor with all external dependencies mocked.""" + ErrorMonitor = _import_error_monitor_class() + monitor = ErrorMonitor.__new__(ErrorMonitor) + monitor.logging_client = MagicMock() + monitor.firestore_adapter = MagicMock() + monitor.discord = MagicMock() + return monitor + + +class TestRunMonitorCycleNoErrors: + """When no error entries are found, auto-resolve check should still run.""" + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_auto_resolve_called_even_when_no_errors(self, mock_digest, mock_llm): + monitor = _make_monitor_with_mocks() + + # Cloud Logging returns empty results for all resource types + monitor.logging_client.list_entries.return_value = iter([]) + + # Firestore adapter returns no patterns eligible for auto-resolve + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + + monitor._run_monitor_cycle() + + # Auto-resolve check must always run + monitor.firestore_adapter.get_patterns_for_auto_resolve.assert_called_once() + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_no_discord_sent_when_no_errors(self, mock_digest, mock_llm): + monitor = _make_monitor_with_mocks() + + monitor.logging_client.list_entries.return_value = iter([]) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + + monitor._run_monitor_cycle() + + monitor.discord.send_message.assert_not_called() + + +class TestRunMonitorCycleOneNewError: + """When one new error pattern is found, an individual alert should be sent.""" + + def _make_log_entry(self): + entry = MagicMock() + entry.payload = "RuntimeError: connection refused" + entry.resource = MagicMock() + entry.resource.type = "cloud_run_revision" + entry.resource.labels = {"service_name": "karaoke-backend"} + entry.labels = {} + return entry + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_individual_alert_sent_for_new_pattern(self, mock_digest, mock_llm, mock_ignore): + from backend.services.error_monitor.firestore_adapter import UpsertResult + + monitor = _make_monitor_with_mocks() + + log_entry = self._make_log_entry() + monitor.logging_client.list_entries.return_value = iter([log_entry]) + + # Upsert returns is_new=True for first encounter + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="abc123", is_new=True + ) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + # An alert should have been sent + assert monitor.discord.send_message.call_count >= 1 + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_upsert_called_for_each_pattern_group(self, mock_digest, mock_llm, mock_ignore): + from backend.services.error_monitor.firestore_adapter import UpsertResult + + monitor = _make_monitor_with_mocks() + + log_entry = self._make_log_entry() + monitor.logging_client.list_entries.return_value = iter([log_entry]) + + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="abc123", is_new=True + ) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + assert monitor.firestore_adapter.upsert_pattern.call_count >= 1 + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_known_issue_skipped(self, mock_digest, mock_llm, mock_ignore): + """Entries matched by should_ignore should not be upserted or alerted.""" + from backend.services.error_monitor.known_issues import IgnoreReason + + # Override: should_ignore returns a reason (i.e., ignore this entry) + mock_ignore.return_value = IgnoreReason( + pattern_name="startup_probe", + reason="Cloud Run cold start — startup probe failure is transient", + ) + + monitor = _make_monitor_with_mocks() + log_entry = self._make_log_entry() + monitor.logging_client.list_entries.return_value = iter([log_entry]) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + + monitor._run_monitor_cycle() + + monitor.firestore_adapter.upsert_pattern.assert_not_called() + monitor.discord.send_message.assert_not_called() + + +# --------------------------------------------------------------------------- +# Tests: _run_daily_digest (P0) +# --------------------------------------------------------------------------- + + +class TestRunDailyDigest: + """_run_daily_digest should fetch active patterns and send a digest to Discord.""" + + @pytest.fixture + def monitor(self): + ErrorMonitor = _import_error_monitor_class() + m = ErrorMonitor.__new__(ErrorMonitor) + m.logging_client = MagicMock() + m.firestore_adapter = MagicMock() + m.discord = MagicMock() + m.discord.messages_sent = 0 + m.logger = MagicMock() + return m + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + def test_digest_sends_message_with_chart_emoji(self, mock_llm, monitor): + """_run_daily_digest should send a message containing the 📊 emoji.""" + monitor.firestore_adapter.get_active_patterns.return_value = [ + {"service": "karaoke-backend", "total_count": 10, "status": "new"}, + {"service": "video-worker", "total_count": 5, "status": "acknowledged"}, + ] + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_daily_digest() + + assert monitor.discord.send_message.call_count == 1 + sent_content = monitor.discord.send_message.call_args[0][0] + assert "📊" in sent_content + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + def test_digest_includes_per_service_counts(self, mock_llm, monitor): + """_run_daily_digest should aggregate counts per service.""" + monitor.firestore_adapter.get_active_patterns.return_value = [ + {"service": "svc-alpha", "total_count": 7, "status": "new"}, + {"service": "svc-alpha", "total_count": 3, "status": "acknowledged"}, + {"service": "svc-beta", "total_count": 2, "status": "new"}, + ] + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_daily_digest() + + sent_content = monitor.discord.send_message.call_args[0][0] + # Both services should be mentioned in the digest + assert "svc-alpha" in sent_content + assert "svc-beta" in sent_content + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + def test_digest_with_no_active_patterns(self, mock_llm, monitor): + """_run_daily_digest should still send a digest even when there are no patterns.""" + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_daily_digest() + + assert monitor.discord.send_message.call_count == 1 + sent_content = monitor.discord.send_message.call_args[0][0] + assert "📊" in sent_content + + +# --------------------------------------------------------------------------- +# Tests: PatternData construction (P0) +# --------------------------------------------------------------------------- + + +class TestPatternDataConstruction: + """upsert_pattern should be called with a PatternData object with correct field types.""" + + def _make_log_entry(self, message="RuntimeError: disk full", service="karaoke-backend"): + entry = MagicMock() + entry.payload = message + entry.resource = MagicMock() + entry.resource.type = "cloud_run_revision" + entry.resource.labels = {"service_name": service} + entry.labels = {} + return entry + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_pattern_data_fields_have_correct_types(self, mock_digest, mock_llm, mock_ignore): + """upsert_pattern receives a PatternData with correctly typed fields.""" + from backend.services.error_monitor.firestore_adapter import PatternData, UpsertResult + + monitor = _make_monitor_with_mocks() + log_entry = self._make_log_entry() + monitor.logging_client.list_entries.return_value = iter([log_entry]) + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="xyz789", is_new=True + ) + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + assert monitor.firestore_adapter.upsert_pattern.call_count >= 1 + call_arg = monitor.firestore_adapter.upsert_pattern.call_args[0][0] + assert isinstance(call_arg, PatternData) + + # pattern_id should be a non-empty hex string + assert isinstance(call_arg.pattern_id, str) + assert len(call_arg.pattern_id) > 0 + # All characters should be valid hex (the hash function produces hex) + assert all(c in "0123456789abcdef" for c in call_arg.pattern_id) + + # service must match the log entry's service label + assert call_arg.service == "karaoke-backend" + + # resource_type maps to canonical string + assert isinstance(call_arg.resource_type, str) + assert len(call_arg.resource_type) > 0 + + # normalized_message and sample_message must be non-empty strings + assert isinstance(call_arg.normalized_message, str) + assert len(call_arg.normalized_message) > 0 + assert isinstance(call_arg.sample_message, str) + assert len(call_arg.sample_message) > 0 + + # count is a positive int + assert isinstance(call_arg.count, int) + assert call_arg.count >= 1 + + # timestamp is a datetime + assert isinstance(call_arg.timestamp, datetime) + + +# --------------------------------------------------------------------------- +# Tests: Discord alert argument verification (P0) +# --------------------------------------------------------------------------- + + +class TestDiscordAlertContent: + """The Discord alert message should contain the service name and normalized message.""" + + def _make_log_entry(self, message, service): + entry = MagicMock() + entry.payload = message + entry.resource = MagicMock() + entry.resource.type = "cloud_run_revision" + entry.resource.labels = {"service_name": service} + entry.labels = {} + return entry + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_new_pattern_alert_contains_service_name(self, mock_digest, mock_llm, mock_ignore): + """The alert message sent to Discord should contain the originating service name.""" + from backend.services.error_monitor.firestore_adapter import UpsertResult + + service_name = "my-special-service" + monitor = _make_monitor_with_mocks() + log_entry = self._make_log_entry("RuntimeError: boom", service=service_name) + monitor.logging_client.list_entries.return_value = iter([log_entry]) + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="aabbcc", is_new=True + ) + monitor.firestore_adapter.get_pattern.return_value = {"first_seen": "2026-01-01T00:00:00Z"} + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + assert monitor.discord.send_message.call_count >= 1 + sent_content = monitor.discord.send_message.call_args[0][0] + assert service_name in sent_content + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_new_pattern_alert_contains_normalized_message( + self, mock_digest, mock_llm, mock_ignore + ): + """The alert message should contain a recognizable part of the normalized error.""" + from backend.services.error_monitor.firestore_adapter import UpsertResult + + monitor = _make_monitor_with_mocks() + # Use a distinctive error phrase that should survive normalization + log_entry = self._make_log_entry( + "ConnectionError: database connection refused", service="karaoke-backend" + ) + monitor.logging_client.list_entries.return_value = iter([log_entry]) + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="def456", is_new=True + ) + monitor.firestore_adapter.get_pattern.return_value = {"first_seen": "2026-01-01T00:00:00Z"} + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + assert monitor.discord.send_message.call_count >= 1 + sent_content = monitor.discord.send_message.call_args[0][0] + # The normalized message should contain at least "connection" or "refused" + assert "connection" in sent_content.lower() or "refused" in sent_content.lower() + + +# --------------------------------------------------------------------------- +# Tests: analyze_patterns input format (P0) +# --------------------------------------------------------------------------- + + +class TestAnalyzePatternsInputFormat: + """When LLM is enabled, analyze_patterns is called with correctly shaped dicts.""" + + def _make_log_entry(self, message, service): + entry = MagicMock() + entry.payload = message + entry.resource = MagicMock() + entry.resource.type = "cloud_run_revision" + entry.resource.labels = {"service_name": service} + entry.labels = {} + return entry + + @patch("backend.services.error_monitor.monitor.analyze_patterns") + @patch("backend.services.error_monitor.monitor.find_duplicate_patterns") + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=True) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_analyze_patterns_receives_correct_keys( + self, mock_digest, mock_llm, mock_ignore, mock_find_dups, mock_analyze + ): + """analyze_patterns should receive a list of dicts with service, normalized_message, count.""" + from backend.services.error_monitor.firestore_adapter import UpsertResult + + mock_analyze.return_value = MagicMock(incidents=[]) + mock_find_dups.return_value = [] + + monitor = _make_monitor_with_mocks() + + # Create 2 distinct log entries so LLM dedup + analysis paths are triggered + entry1 = self._make_log_entry("RuntimeError: disk full", service="svc-one") + entry2 = self._make_log_entry("ValueError: invalid input", service="svc-two") + monitor.logging_client.list_entries.side_effect = [ + iter([entry1, entry2]), + iter([]), + iter([]), + iter([]), + ] + + call_count = [0] + + def upsert_side_effect(data): + call_count[0] += 1 + return UpsertResult(pattern_id=data.pattern_id, is_new=True) + + monitor.firestore_adapter.upsert_pattern.side_effect = upsert_side_effect + monitor.firestore_adapter.get_pattern.return_value = {"first_seen": "2026-01-01T00:00:00Z"} + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + # analyze_patterns should have been called once both new patterns exist + assert mock_analyze.call_count >= 1 + call_args = mock_analyze.call_args[0][0] + assert isinstance(call_args, list) + assert len(call_args) >= 1 + + for item in call_args: + assert "service" in item, f"Missing 'service' key in {item}" + assert "normalized_message" in item, f"Missing 'normalized_message' key in {item}" + assert "count" in item, f"Missing 'count' key in {item}" + assert isinstance(item["service"], str) + assert isinstance(item["normalized_message"], str) + assert isinstance(item["count"], int) + + +# --------------------------------------------------------------------------- +# Tests: Spike detection flow (P1) +# --------------------------------------------------------------------------- + + +class TestSpikeDetectionFlow: + """_run_monitor_cycle should send a spike alert when counts exceed the threshold.""" + + def _make_log_entry(self, message="RuntimeError: timeout", service="karaoke-backend"): + entry = MagicMock() + entry.payload = message + entry.resource = MagicMock() + entry.resource.type = "cloud_run_revision" + entry.resource.labels = {"service_name": service} + entry.labels = {} + return entry + + @patch("backend.services.error_monitor.monitor.should_ignore", return_value=None) + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + @patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False) + def test_spike_alert_sent_when_count_exceeds_threshold( + self, mock_digest, mock_llm, mock_ignore + ): + """A spike alert (⚠️) is sent when current count is >5x rolling average and >= min.""" + from backend.services.error_monitor.firestore_adapter import UpsertResult + + monitor = _make_monitor_with_mocks() + + # Send 10 identical entries to make count = 10 (> SPIKE_MIN_COUNT=5) + entries = [self._make_log_entry() for _ in range(10)] + monitor.logging_client.list_entries.return_value = iter(entries) + + # Upsert returns is_new=False (existing pattern) so spike check runs + monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult( + pattern_id="spike-pattern-1", is_new=False + ) + + # get_pattern returns a pattern with rolling_counts that average to 1 + # avg=1.0, 5x=5.0, current_count=10 → spike detected + monitor.firestore_adapter.get_pattern.return_value = { + "pattern_id": "spike-pattern-1", + "rolling_counts": [{"count": 1}, {"count": 1}, {"count": 1}, {"count": 1}], + } + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + # At least one Discord message should have been sent + assert monitor.discord.send_message.call_count >= 1 + + # At least one of the messages should be a spike alert (⚠️) + all_calls = monitor.discord.send_message.call_args_list + spike_messages = [c for c in all_calls if "⚠️" in c[0][0]] + assert len(spike_messages) >= 1, ( + "Expected at least one spike alert (⚠️) but none found. " + f"Messages sent: {[c[0][0][:80] for c in all_calls]}" + ) + + +# --------------------------------------------------------------------------- +# Tests: Auto-resolve alert (P1) +# --------------------------------------------------------------------------- + + +class TestAutoResolveAlert: + """_check_auto_resolve should resolve eligible patterns and send a ✅ alert.""" + + @pytest.fixture + def monitor(self): + ErrorMonitor = _import_error_monitor_class() + m = ErrorMonitor.__new__(ErrorMonitor) + m.logging_client = MagicMock() + m.firestore_adapter = MagicMock() + m.discord = MagicMock() + m.discord.messages_sent = 0 + m.logger = MagicMock() + return m + + def test_auto_resolve_calls_resolve_and_sends_alert(self, monitor): + """When a pattern qualifies for auto-resolve, the pattern is resolved and ✅ is sent.""" + pattern = { + "pattern_id": "resolve-me-123", + "service": "karaoke-backend", + "normalized_message": "Connection refused", + "status": "new", + } + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern] + monitor.firestore_adapter.check_auto_resolve.return_value = 25.5 # hours silent + monitor.discord.send_message.return_value = True + + monitor._check_auto_resolve() + + # auto_resolve_pattern must have been called with the correct pattern_id + monitor.firestore_adapter.auto_resolve_pattern.assert_called_once_with( + "resolve-me-123", 25.5 + ) + + # A Discord message must have been sent + assert monitor.discord.send_message.call_count == 1 + sent_content = monitor.discord.send_message.call_args[0][0] + assert "✅" in sent_content + + def test_auto_resolve_no_alert_when_no_patterns_qualify(self, monitor): + """When no patterns qualify for auto-resolve, no Discord alert is sent.""" + pattern = { + "pattern_id": "still-active", + "service": "svc-x", + "normalized_message": "Disk full", + "status": "acknowledged", + } + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern] + # check_auto_resolve returns None → pattern does not qualify yet + monitor.firestore_adapter.check_auto_resolve.return_value = None + + monitor._check_auto_resolve() + + monitor.firestore_adapter.auto_resolve_pattern.assert_not_called() + monitor.discord.send_message.assert_not_called() + + def test_auto_resolve_via_monitor_cycle(self, monitor): + """_run_monitor_cycle with no errors should still trigger the auto-resolve check.""" + with patch( + "backend.services.error_monitor.monitor.get_llm_enabled", return_value=False + ), patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False): + monitor.logging_client.list_entries.return_value = iter([]) + + pattern = { + "pattern_id": "cycle-resolve", + "service": "bg-worker", + "normalized_message": "queue timeout", + "status": "new", + } + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern] + monitor.firestore_adapter.check_auto_resolve.return_value = 12.0 + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.discord.send_message.return_value = True + + monitor._run_monitor_cycle() + + monitor.firestore_adapter.auto_resolve_pattern.assert_called_once() + sent_content = monitor.discord.send_message.call_args[0][0] + assert "✅" in sent_content diff --git a/tests/unit/services/error_monitor/test_normalizer.py b/tests/unit/services/error_monitor/test_normalizer.py new file mode 100644 index 00000000..c7f4c8bc --- /dev/null +++ b/tests/unit/services/error_monitor/test_normalizer.py @@ -0,0 +1,415 @@ +"""Tests for error_monitor.normalizer module.""" + +import pytest + + +class TestUUIDReplacement: + """UUID patterns should be replaced with .""" + + def test_uuid_v4_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Job failed for job_id=550e8400-e29b-41d4-a716-446655440000" + result = normalize_message(msg) + assert "550e8400-e29b-41d4-a716-446655440000" not in result + assert "" in result + + def test_uuid_uppercase_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Error in request 550E8400-E29B-41D4-A716-446655440000" + result = normalize_message(msg) + assert "550E8400-E29B-41D4-A716-446655440000" not in result + assert "" in result + + def test_multiple_uuids_all_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Copy from 550e8400-e29b-41d4-a716-446655440000 to 660f9511-f30c-52e5-b827-557766551111" + result = normalize_message(msg) + assert "550e8400" not in result + assert "660f9511" not in result + + +class TestISOTimestampReplacement: + """ISO 8601 timestamps should be replaced with .""" + + def test_iso_datetime_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Error occurred at 2024-01-15T10:30:45Z" + result = normalize_message(msg) + assert "2024-01-15T10:30:45Z" not in result + assert "" in result + + def test_iso_datetime_with_ms_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Timeout at 2024-03-22T14:55:01.123Z" + result = normalize_message(msg) + assert "2024-03-22T14:55:01.123Z" not in result + assert "" in result + + def test_iso_datetime_with_offset_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Created at 2024-06-01T08:00:00+05:30" + result = normalize_message(msg) + assert "2024-06-01T08:00:00+05:30" not in result + assert "" in result + + +class TestEpochTimestampReplacement: + """Epoch timestamps (10+ digit floats) should be replaced with .""" + + def test_epoch_float_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Received at t=1711234567.891234" + result = normalize_message(msg) + assert "1711234567.891234" not in result + assert "" in result + + def test_epoch_integer_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "timestamp=1711234567890" + result = normalize_message(msg) + assert "1711234567890" not in result + assert "" in result + + def test_small_number_not_epoch(self): + from backend.services.error_monitor.normalizer import normalize_message + + # 4-digit number should become , not + msg = "Status code 404 returned" + result = normalize_message(msg) + assert "" not in result + + +class TestEmailReplacement: + """Email addresses should be replaced with .""" + + def test_simple_email_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "User user@example.com not found" + result = normalize_message(msg) + assert "user@example.com" not in result + assert "" in result + + def test_email_with_subdomain_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Sent to andrew@api.nomadkaraoke.com" + result = normalize_message(msg) + assert "andrew@api.nomadkaraoke.com" not in result + assert "" in result + + def test_email_with_plus_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Contact for test+filter@gmail.com" + result = normalize_message(msg) + assert "test+filter@gmail.com" not in result + assert "" in result + + +class TestIPAddressReplacement: + """IP addresses should be replaced with .""" + + def test_ipv4_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Connection refused from 192.168.1.100" + result = normalize_message(msg) + assert "192.168.1.100" not in result + assert "" in result + + def test_ipv4_with_port_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Failed to connect to 10.0.0.1:8080" + result = normalize_message(msg) + assert "10.0.0.1" not in result + assert "" in result + + def test_multiple_ips_all_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Route from 10.0.0.1 to 172.16.0.254 failed" + result = normalize_message(msg) + assert "10.0.0.1" not in result + assert "172.16.0.254" not in result + + +class TestHexIDReplacement: + """Hex/alphanumeric IDs (8+ chars with both digits and letters) should be replaced with .""" + + def test_hex_id_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Resource abc12345def not found" + result = normalize_message(msg) + assert "abc12345def" not in result + assert "" in result + + def test_pure_alpha_word_not_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + # Pure alphabetic strings should not be replaced + msg = "Authentication failed completely" + result = normalize_message(msg) + assert result == "Authentication failed completely" + + def test_short_id_not_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + # IDs shorter than 8 chars should not be replaced + msg = "Error in abc123 handler" + result = normalize_message(msg) + # Should not replace short mixed alphanum under 8 chars as + assert "abc123" in result + + +class TestLargeNumberReplacement: + """Numbers with 4+ digits should be replaced with .""" + + def test_four_digit_number_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "HTTP error code 5000 encountered" + result = normalize_message(msg) + assert "5000" not in result + assert "" in result + + def test_large_number_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Allocated 12345 bytes" + result = normalize_message(msg) + assert "12345" not in result + assert "" in result + + def test_small_number_preserved(self): + from backend.services.error_monitor.normalizer import normalize_message + + # 3-digit numbers like HTTP status codes should be preserved + msg = "HTTP 404 Not Found" + result = normalize_message(msg) + assert "404" in result + + def test_two_digit_number_preserved(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Retry attempt 42" + result = normalize_message(msg) + assert "42" in result + + +class TestJobIDReplacement: + """Job IDs in paths should be replaced.""" + + def test_job_id_in_path_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Failed processing /jobs/abc123def456 in queue" + result = normalize_message(msg) + assert "abc123def456" not in result + assert "/jobs/" in result + + def test_job_id_alphanumeric_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Job at /jobs/XyZ9a1b2C3d4 timed out" + result = normalize_message(msg) + assert "XyZ9a1b2C3d4" not in result + assert "/jobs/" in result + + +class TestGCSPathReplacement: + """GCS paths should be normalized.""" + + def test_gcs_path_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Could not read gs://my-bucket/path/to/file.mp4" + result = normalize_message(msg) + assert "gs://my-bucket/path/to/file.mp4" not in result + assert "gs:///" in result + + def test_gcs_path_without_object_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Bucket access denied: gs://nomadkaraoke-prod-videos/" + result = normalize_message(msg) + assert "gs://nomadkaraoke-prod-videos/" not in result + assert "gs://" in result + + def test_gcs_path_with_nested_path_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "File gs://nomad-audio/jobs/abc123/output/stem.wav not found" + result = normalize_message(msg) + assert "gs://nomad-audio/jobs/abc123/output/stem.wav" not in result + assert "gs:///" in result + + +class TestFirebaseUIDReplacement: + """Firebase UIDs (20-28 mixed-case alphanumeric) should be replaced with .""" + + def test_firebase_uid_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + # Firebase UIDs are 28 chars: letters and digits, mixed case + msg = "User uid=ABC123defGHI456jklMNO7 not authorized" + result = normalize_message(msg) + assert "ABC123defGHI456jklMNO7" not in result + assert "" in result + + def test_firebase_uid_28_chars_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Auth failed for 8JkLmNoPqRsT1uVwXyZ23456AB" + result = normalize_message(msg) + assert "8JkLmNoPqRsT1uVwXyZ23456AB" not in result + assert "" in result + + +class TestFirestoreDocPathReplacement: + """Firestore document paths should be replaced with .""" + + def test_firestore_doc_path_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Document jobs/abc123def/state_data/current does not exist" + result = normalize_message(msg) + assert "jobs/abc123def/state_data/current" not in result + assert "" in result + + def test_firestore_users_path_replaced(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Permission denied for users/uid12345/profile/settings" + result = normalize_message(msg) + assert "users/uid12345/profile/settings" not in result + assert "" in result + + +class TestURLNormalization: + """URLs should be stripped to domain only.""" + + def test_url_stripped_to_domain(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Failed to fetch https://api.example.com/v1/users/12345/data" + result = normalize_message(msg) + assert "/v1/users/12345/data" not in result + assert "api.example.com" in result + + def test_http_url_stripped(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Redirect from http://old.example.com/path?q=abc123" + result = normalize_message(msg) + assert "/path?q=abc123" not in result + assert "old.example.com" in result + + def test_url_with_port_domain_preserved(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Connection to https://internal.service.com:8080/api/endpoint failed" + result = normalize_message(msg) + assert "internal.service.com" in result + + +class TestTruncation: + """Long messages should be truncated to MAX_NORMALIZED_MESSAGE_LENGTH (200 chars).""" + + def test_long_message_truncated(self): + from backend.services.error_monitor.normalizer import normalize_message + from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH + + long_msg = "Error: " + "x" * 300 + result = normalize_message(long_msg) + assert len(result) <= MAX_NORMALIZED_MESSAGE_LENGTH + + def test_short_message_not_truncated(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Short error message" + result = normalize_message(msg) + assert result == msg + + def test_exactly_200_chars_not_truncated(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "A" * 200 + result = normalize_message(msg) + assert len(result) == 200 + + +class TestEdgeCases: + """Edge cases for normalize_message.""" + + def test_empty_message_returns_empty_string(self): + from backend.services.error_monitor.normalizer import normalize_message + + result = normalize_message("") + assert result == "" + + def test_no_replacements_returns_original(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Connection refused by server" + result = normalize_message(msg) + assert result == msg + + def test_deterministic_same_input_same_output(self): + from backend.services.error_monitor.normalizer import normalize_message + + msg = "Job failed for 550e8400-e29b-41d4-a716-446655440000 at 2024-01-15T10:30:45Z" + result1 = normalize_message(msg) + result2 = normalize_message(msg) + assert result1 == result2 + + +class TestComputePatternHash: + """Tests for the compute_pattern_hash function.""" + + def test_returns_64_char_hex(self): + from backend.services.error_monitor.normalizer import compute_pattern_hash + + result = compute_pattern_hash("karaoke-backend", "Connection refused by server") + assert isinstance(result, str) + assert len(result) == 64 + # SHA-256 hex is lowercase hex + assert all(c in "0123456789abcdef" for c in result) + + def test_same_inputs_same_hash(self): + from backend.services.error_monitor.normalizer import compute_pattern_hash + + h1 = compute_pattern_hash("karaoke-backend", "Connection refused by server") + h2 = compute_pattern_hash("karaoke-backend", "Connection refused by server") + assert h1 == h2 + + def test_different_services_different_hash(self): + from backend.services.error_monitor.normalizer import compute_pattern_hash + + h1 = compute_pattern_hash("karaoke-backend", "Connection refused by server") + h2 = compute_pattern_hash("audio-separator", "Connection refused by server") + assert h1 != h2 + + def test_different_messages_different_hash(self): + from backend.services.error_monitor.normalizer import compute_pattern_hash + + h1 = compute_pattern_hash("karaoke-backend", "Connection refused by server") + h2 = compute_pattern_hash("karaoke-backend", "Timeout waiting for response") + assert h1 != h2 + + def test_empty_service_and_message(self): + from backend.services.error_monitor.normalizer import compute_pattern_hash + + result = compute_pattern_hash("", "") + assert isinstance(result, str) + assert len(result) == 64