mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat: production error monitor — log collection, LLM analysis, Discord alerts (#713)
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b847dba411
commit
9a5c9d497a
28 files changed
+11056
-2
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Production error monitoring service for Nomad Karaoke."""
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Configuration for the production error monitoring service.
|
||||
|
||||
All constants, service lists, and environment-driven configuration
|
||||
functions used across the error monitor modules.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GCP project settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GCP_PROJECT: str = os.environ.get("GCP_PROJECT", "nomadkaraoke")
|
||||
GCP_REGION: str = os.environ.get("GCP_REGION", "us-central1")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Log querying settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: How far back to look for errors on each run (minutes).
|
||||
LOOKBACK_MINUTES: int = 15
|
||||
|
||||
#: Maximum number of log entries to fetch per service per run.
|
||||
MAX_LOG_ENTRIES: int = 500
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Monitored services
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MONITORED_CLOUD_RUN_SERVICES: list[str] = [
|
||||
"karaoke-backend",
|
||||
"karaoke-decide",
|
||||
"audio-separator",
|
||||
]
|
||||
|
||||
MONITORED_CLOUD_RUN_JOBS: list[str] = [
|
||||
"video-encoding-job",
|
||||
"lyrics-transcription-job",
|
||||
"audio-separation-job",
|
||||
"audio-download-job",
|
||||
]
|
||||
|
||||
MONITORED_CLOUD_FUNCTIONS: list[str] = [
|
||||
"gdrive-validator",
|
||||
"runner_manager",
|
||||
"backup_to_aws",
|
||||
"divebar_mirror",
|
||||
"kn_data_sync",
|
||||
"divebar_lookup",
|
||||
"encoding_worker_idle",
|
||||
]
|
||||
|
||||
MONITORED_GCE_INSTANCES: list[str] = [
|
||||
"encoding-worker-a",
|
||||
"encoding-worker-b",
|
||||
"flacfetch-vm",
|
||||
"divebar-sync-vm",
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Service dependency map
|
||||
# Human-readable description of which services depend on which workers.
|
||||
# Used as LLM context when generating root-cause analysis.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SERVICE_DEPENDENCY_MAP: str = """\
|
||||
karaoke-backend (main API)
|
||||
→ video-encoding-job (Cloud Run Job, triggered per order)
|
||||
→ lyrics-transcription-job (Cloud Run Job, triggered after audio-sep)
|
||||
→ audio-separation-job (Cloud Run Job, runs on Cloud Run GPU)
|
||||
→ audio-download-job (Cloud Run Job, uses flacfetch-vm)
|
||||
→ gdrive-validator (Cloud Function, validates output files)
|
||||
→ encoding-worker-a/b (GCE VMs, long-running encoding workers)
|
||||
|
||||
karaoke-decide (catalog / discovery API)
|
||||
→ kn_data_sync (Cloud Function, syncs KaraokeNerds catalog)
|
||||
→ divebar_mirror (Cloud Function, mirrors Divebar catalog)
|
||||
→ divebar_lookup (Cloud Function, single-song lookup)
|
||||
|
||||
audio-separator (Cloud Run service, GPU stem separation)
|
||||
→ audio-separation-job (Cloud Run Job, launched by audio-separator)
|
||||
|
||||
flacfetch-vm (GCE VM)
|
||||
→ audio-download-job (initiated by karaoke-backend)
|
||||
|
||||
divebar-sync-vm (GCE VM)
|
||||
→ divebar_mirror (Cloud Function, reads from this VM)
|
||||
|
||||
Shared infrastructure:
|
||||
→ runner_manager (Cloud Function, manages encoding worker lifecycle)
|
||||
→ encoding_worker_idle (Cloud Function, shuts down idle encoding workers)
|
||||
→ backup_to_aws (Cloud Function, daily GCS → S3 backup)
|
||||
"""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM analysis settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LLM_ANALYSIS_MODEL: str = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-2.0-flash-001")
|
||||
LLM_VERTEX_LOCATION: str = os.environ.get("LLM_VERTEX_LOCATION", "us-central1")
|
||||
|
||||
#: Minimum number of active error patterns required before triggering LLM analysis.
|
||||
MIN_PATTERNS_FOR_ANALYSIS: int = 3
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spike detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: An error count is a spike if it exceeds SPIKE_MULTIPLIER × baseline average.
|
||||
SPIKE_MULTIPLIER: float = 5.0
|
||||
|
||||
#: Minimum absolute error count before spike detection fires (avoids noise).
|
||||
SPIKE_MIN_COUNT: int = 5
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auto-resolution settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Multiply LOOKBACK_MINUTES by this to get auto-resolve window in minutes.
|
||||
AUTO_RESOLVE_MULTIPLIER: int = 8
|
||||
|
||||
#: Minimum hours before a pattern can be auto-resolved.
|
||||
AUTO_RESOLVE_MIN_HOURS: int = 6
|
||||
|
||||
#: Maximum hours before a pattern is force-resolved even if still active.
|
||||
AUTO_RESOLVE_MAX_HOURS: int = 168 # 1 week
|
||||
|
||||
#: Fallback resolve window (hours) if baseline data is unavailable.
|
||||
AUTO_RESOLVE_FALLBACK_HOURS: int = 48
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discord settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Maximum Discord messages to send per monitor run (rate-limit guard).
|
||||
MAX_DISCORD_MESSAGES_PER_RUN: int = 10
|
||||
|
||||
#: Discord API hard limit on message length.
|
||||
DISCORD_MAX_MESSAGE_LENGTH: int = 2000
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern tracking settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Maximum number of active error patterns to track in Firestore.
|
||||
MAX_ACTIVE_PATTERNS: int = 500
|
||||
|
||||
#: Rolling window used when computing error baselines.
|
||||
ROLLING_WINDOW_DAYS: int = 7
|
||||
|
||||
#: Maximum length of a normalized error message stored in Firestore.
|
||||
MAX_NORMALIZED_MESSAGE_LENGTH: int = 200
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_llm_enabled() -> bool:
|
||||
"""Return True if LLM analysis is enabled via the LLM_ANALYSIS_ENABLED env var."""
|
||||
return os.environ.get("LLM_ANALYSIS_ENABLED", "false").strip().lower() == "true"
|
||||
|
||||
|
||||
def get_discord_webhook_secret_name() -> str:
|
||||
"""Return the Secret Manager secret name for the Discord webhook URL."""
|
||||
return os.environ.get("DISCORD_WEBHOOK_SECRET_NAME", "error-monitor-discord-webhook")
|
||||
|
||||
|
||||
def get_digest_mode() -> bool:
|
||||
"""Return True if the monitor should run in digest mode (batch alerts instead of immediate).
|
||||
|
||||
In digest mode all alerts are bundled into a single Discord message sent at
|
||||
the end of the run, rather than sending one message per error pattern.
|
||||
"""
|
||||
return os.environ.get("DIGEST_MODE", "false").strip().lower() == "true"
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Discord alert client for the production error monitoring service.
|
||||
|
||||
Provides formatting functions for each alert type, and an ErrorMonitorDiscord
|
||||
webhook client that rate-limits outbound messages to MAX_DISCORD_MESSAGES_PER_RUN
|
||||
per monitor run.
|
||||
|
||||
Usage:
|
||||
from backend.services.error_monitor.discord import (
|
||||
ErrorMonitorDiscord,
|
||||
format_new_pattern_alert,
|
||||
format_incident_alert,
|
||||
format_auto_resolved_alert,
|
||||
format_spike_alert,
|
||||
format_daily_digest,
|
||||
)
|
||||
|
||||
discord = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/...")
|
||||
msg = format_new_pattern_alert(...)
|
||||
discord.send_message(msg)
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import requests
|
||||
|
||||
from backend.services.error_monitor.config import (
|
||||
DISCORD_MAX_MESSAGE_LENGTH,
|
||||
MAX_DISCORD_MESSAGES_PER_RUN,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SEVERITY_EMOJI: dict[str, str] = {
|
||||
"P0": "🚨",
|
||||
"P1": "🔴",
|
||||
"P2": "⚠️",
|
||||
"P3": "🟡",
|
||||
}
|
||||
|
||||
_EASTERN = ZoneInfo("America/New_York")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _now_eastern() -> str:
|
||||
"""Return the current Eastern time as 'HH:MM AM/PM ET' (DST-aware)."""
|
||||
now = datetime.now(tz=_EASTERN)
|
||||
return now.strftime("%I:%M %p ET").lstrip("0")
|
||||
|
||||
|
||||
def _truncate(msg: str) -> str:
|
||||
"""Truncate *msg* to DISCORD_MAX_MESSAGE_LENGTH characters."""
|
||||
if len(msg) <= DISCORD_MAX_MESSAGE_LENGTH:
|
||||
return msg
|
||||
suffix = "…"
|
||||
return msg[: DISCORD_MAX_MESSAGE_LENGTH - len(suffix)] + suffix
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Formatting functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def format_new_pattern_alert(
|
||||
service: str,
|
||||
resource_type: str,
|
||||
normalized_message: str,
|
||||
sample_message: str,
|
||||
count: int,
|
||||
first_seen: str,
|
||||
) -> str:
|
||||
"""Format a Discord alert for a newly-detected error pattern.
|
||||
|
||||
Args:
|
||||
service: Name of the service emitting the error.
|
||||
resource_type: GCP resource type (e.g. 'cloud_run_service', 'gce_instance').
|
||||
normalized_message: Normalised error pattern string.
|
||||
sample_message: A representative raw log message.
|
||||
count: Number of occurrences seen so far.
|
||||
first_seen: ISO 8601 timestamp when the pattern was first seen.
|
||||
|
||||
Returns:
|
||||
Formatted Discord message string, truncated to 2000 chars.
|
||||
"""
|
||||
lines = [
|
||||
f"🔴 **New Error Pattern Detected**",
|
||||
f"",
|
||||
f"**Service:** {service}",
|
||||
f"**Resource type:** {resource_type}",
|
||||
f"**Pattern:** {normalized_message}",
|
||||
f"**Sample:** {sample_message}",
|
||||
f"**Count:** {count}",
|
||||
f"**First seen:** {first_seen}",
|
||||
f"**Alerted at:** {_now_eastern()}",
|
||||
f"",
|
||||
f"To investigate: `/prod-investigate {service} \"{normalized_message}\"`",
|
||||
]
|
||||
return _truncate("\n".join(lines))
|
||||
|
||||
|
||||
def format_incident_alert(
|
||||
title: str,
|
||||
severity: str,
|
||||
root_cause: str,
|
||||
suggested_fix: str,
|
||||
primary_service: str,
|
||||
patterns: list[dict],
|
||||
) -> str:
|
||||
"""Format a Discord alert for an analysed incident (LLM-generated).
|
||||
|
||||
Args:
|
||||
title: Short incident title.
|
||||
severity: Severity level — one of P0, P1, P2, P3.
|
||||
root_cause: Human-readable root-cause analysis.
|
||||
suggested_fix: Recommended remediation steps.
|
||||
primary_service: The main service involved in the incident.
|
||||
patterns: List of dicts, each with keys: service, normalized_message, count.
|
||||
Only the first 5 are shown; normalized_message is truncated to 80 chars.
|
||||
|
||||
Returns:
|
||||
Formatted Discord message string, truncated to 2000 chars.
|
||||
"""
|
||||
emoji = SEVERITY_EMOJI.get(severity, "⚠️")
|
||||
lines = [
|
||||
f"{emoji} **[{severity}] {title}**",
|
||||
f"",
|
||||
f"**Primary service:** {primary_service}",
|
||||
f"**Root cause:** {root_cause}",
|
||||
f"**Suggested fix:** {suggested_fix}",
|
||||
]
|
||||
|
||||
shown_patterns = patterns[:5]
|
||||
if shown_patterns:
|
||||
lines.append("")
|
||||
lines.append(f"**Patterns ({len(shown_patterns)} of {len(patterns)}):**")
|
||||
for p in shown_patterns:
|
||||
svc = p.get("service", "unknown")
|
||||
msg = p.get("normalized_message", "")[:80]
|
||||
cnt = p.get("count", 0)
|
||||
lines.append(f" • `{svc}` — {msg} ({cnt}x)")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"**Alerted at:** {_now_eastern()}")
|
||||
|
||||
return _truncate("\n".join(lines))
|
||||
|
||||
|
||||
def format_auto_resolved_alert(resolved_patterns: list[dict]) -> str:
|
||||
"""Format a consolidated auto-resolved alert.
|
||||
|
||||
Args:
|
||||
resolved_patterns: List of dicts, each with keys: service, message, hours_silent.
|
||||
Only the first 8 are shown; message is truncated to 60 chars.
|
||||
|
||||
Returns:
|
||||
Formatted Discord message string, truncated to 2000 chars.
|
||||
"""
|
||||
lines = [
|
||||
f"✅ **Patterns Auto-Resolved**",
|
||||
f"",
|
||||
]
|
||||
|
||||
shown = resolved_patterns[:8]
|
||||
if shown:
|
||||
for p in shown:
|
||||
svc = p.get("service", "unknown")
|
||||
msg = p.get("message", "")[:60]
|
||||
hours = p.get("hours_silent", 0)
|
||||
lines.append(f" • `{svc}` — {msg} (silent {hours}h)")
|
||||
else:
|
||||
lines.append(" *(no patterns)*")
|
||||
|
||||
if len(resolved_patterns) > 8:
|
||||
lines.append(f" … and {len(resolved_patterns) - 8} more")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"**Resolved at:** {_now_eastern()}")
|
||||
|
||||
return _truncate("\n".join(lines))
|
||||
|
||||
|
||||
def format_spike_alert(
|
||||
service: str,
|
||||
normalized_message: str,
|
||||
current_count: int,
|
||||
rolling_average: float,
|
||||
) -> str:
|
||||
"""Format a Discord alert for an error rate spike.
|
||||
|
||||
Args:
|
||||
service: Name of the service.
|
||||
normalized_message: Normalised error pattern string.
|
||||
current_count: Error count in the current window.
|
||||
rolling_average: Baseline rolling average count.
|
||||
|
||||
Returns:
|
||||
Formatted Discord message string, truncated to 2000 chars.
|
||||
"""
|
||||
if rolling_average > 0:
|
||||
multiplier = current_count / rolling_average
|
||||
multiplier_str = f"{multiplier:.1f}x"
|
||||
else:
|
||||
multiplier_str = "∞x"
|
||||
|
||||
lines = [
|
||||
f"⚠️ **Error Rate Spike Detected**",
|
||||
f"",
|
||||
f"**Service:** {service}",
|
||||
f"**Pattern:** {normalized_message}",
|
||||
f"**Current count:** {current_count}",
|
||||
f"**Normal (rolling avg):** {rolling_average}",
|
||||
f"**Multiplier:** {multiplier_str} above normal",
|
||||
f"",
|
||||
f"**Alerted at:** {_now_eastern()}",
|
||||
]
|
||||
return _truncate("\n".join(lines))
|
||||
|
||||
|
||||
def format_daily_digest(
|
||||
total_errors: int,
|
||||
new_patterns: int,
|
||||
resolved_patterns: int,
|
||||
active_patterns: int,
|
||||
per_service_counts: dict,
|
||||
) -> str:
|
||||
"""Format the daily 24h error digest.
|
||||
|
||||
Args:
|
||||
total_errors: Total error count in the last 24 hours.
|
||||
new_patterns: Number of new error patterns detected.
|
||||
resolved_patterns: Number of patterns auto-resolved.
|
||||
active_patterns: Total number of currently active patterns.
|
||||
per_service_counts: Mapping of service name → error count.
|
||||
Displayed sorted by count descending.
|
||||
|
||||
Returns:
|
||||
Formatted Discord message string, truncated to 2000 chars.
|
||||
"""
|
||||
lines = [
|
||||
f"📊 **Daily Error Digest (24h)**",
|
||||
f"",
|
||||
f"**Total errors:** {total_errors}",
|
||||
f"**New patterns:** {new_patterns}",
|
||||
f"**Resolved patterns:** {resolved_patterns}",
|
||||
f"**Active patterns:** {active_patterns}",
|
||||
]
|
||||
|
||||
if per_service_counts:
|
||||
lines.append("")
|
||||
lines.append("**Per-service breakdown:**")
|
||||
sorted_services = sorted(per_service_counts.items(), key=lambda kv: kv[1], reverse=True)
|
||||
for svc, cnt in sorted_services:
|
||||
lines.append(f" • `{svc}`: {cnt}")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"**Generated at:** {_now_eastern()}")
|
||||
|
||||
return _truncate("\n".join(lines))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discord webhook client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ErrorMonitorDiscord:
|
||||
"""Webhook client for sending error monitor alerts to Discord.
|
||||
|
||||
Enforces a per-run rate limit of MAX_DISCORD_MESSAGES_PER_RUN to avoid
|
||||
flooding the channel during high-volume incident windows.
|
||||
|
||||
Attributes:
|
||||
webhook_url: The Discord incoming webhook URL.
|
||||
messages_sent: Counter of messages attempted this run (success or failure).
|
||||
"""
|
||||
|
||||
def __init__(self, webhook_url: str) -> None:
|
||||
self.webhook_url = webhook_url
|
||||
self.messages_sent: int = 0
|
||||
|
||||
def send_message(self, content: str) -> bool:
|
||||
"""Send a message to the configured Discord webhook.
|
||||
|
||||
Rate-limits: if messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN, the
|
||||
message is silently skipped and False is returned without incrementing
|
||||
the counter.
|
||||
|
||||
Args:
|
||||
content: The message text to send (should be pre-formatted and
|
||||
pre-truncated to ≤2000 chars).
|
||||
|
||||
Returns:
|
||||
True if the message was sent successfully, False otherwise.
|
||||
"""
|
||||
if self.messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN:
|
||||
logger.warning(
|
||||
"Discord rate limit reached (%d messages sent this run); skipping message.",
|
||||
self.messages_sent,
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
self.webhook_url,
|
||||
json={"content": content},
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.debug("Discord message sent successfully.")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("Failed to send Discord message: %s", exc)
|
||||
return False
|
||||
finally:
|
||||
self.messages_sent += 1
|
||||
@@ -0,0 +1,489 @@
|
||||
"""Firestore adapter for the production error monitoring service.
|
||||
|
||||
Handles all CRUD for three Firestore collections:
|
||||
- error_patterns: normalized error patterns (pattern_id as doc ID)
|
||||
- error_incidents: groups of related patterns (incident_id as doc ID)
|
||||
- discord_alerts: audit trail for Discord messages (alert_id as doc ID)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from backend.services.error_monitor.config import (
|
||||
AUTO_RESOLVE_FALLBACK_HOURS,
|
||||
AUTO_RESOLVE_MAX_HOURS,
|
||||
AUTO_RESOLVE_MIN_HOURS,
|
||||
AUTO_RESOLVE_MULTIPLIER,
|
||||
ROLLING_WINDOW_DAYS,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PatternData:
|
||||
"""Input data for creating or updating an error pattern."""
|
||||
|
||||
pattern_id: str
|
||||
service: str
|
||||
resource_type: str
|
||||
normalized_message: str
|
||||
sample_message: str
|
||||
count: int
|
||||
timestamp: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class UpsertResult:
|
||||
"""Result of an upsert_pattern operation."""
|
||||
|
||||
pattern_id: str
|
||||
is_new: bool
|
||||
previous_status: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collection names
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COL_PATTERNS = "error_patterns"
|
||||
_COL_INCIDENTS = "error_incidents"
|
||||
_COL_ALERTS = "discord_alerts"
|
||||
|
||||
# Statuses that can be queried as "active"
|
||||
_ACTIVE_STATUSES = ["new", "acknowledged", "known"]
|
||||
|
||||
# Statuses from which a pattern is reactivated when the error recurs
|
||||
_REACTIVATION_STATUSES = {"auto_resolved", "fixed"}
|
||||
|
||||
# Statuses eligible for auto-resolve checking
|
||||
_AUTO_RESOLVE_ELIGIBLE = {"new", "acknowledged"}
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
"""Return current UTC datetime."""
|
||||
return datetime.now(tz=timezone.utc)
|
||||
|
||||
|
||||
def _iso(dt: datetime) -> str:
|
||||
"""Serialise a datetime to ISO-8601 string."""
|
||||
return dt.isoformat()
|
||||
|
||||
|
||||
def _generate_id(prefix: str, fmt: str) -> str:
|
||||
"""Generate a time-stamped ID with 8 hex random suffix.
|
||||
|
||||
Args:
|
||||
prefix: e.g. "inc" or "alert"
|
||||
fmt: strftime format string applied to utcnow(), e.g. "%Y%m%d_%H%M"
|
||||
"""
|
||||
ts = _utcnow().strftime(fmt)
|
||||
suffix = secrets.token_hex(4) # 8 hex chars
|
||||
return f"{prefix}_{ts}_{suffix}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adapter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ErrorPatternsAdapter:
|
||||
"""Firestore CRUD adapter for error monitoring collections.
|
||||
|
||||
Args:
|
||||
db: Optional Firestore client. If omitted, a default client for the
|
||||
``nomadkaraoke`` GCP project is constructed on first use.
|
||||
"""
|
||||
|
||||
def __init__(self, db=None) -> None:
|
||||
if db is None:
|
||||
from google.cloud import firestore # type: ignore[import]
|
||||
|
||||
db = firestore.Client(project="nomadkaraoke")
|
||||
self._db = db
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Patterns — read
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_pattern(self, pattern_id: str) -> dict | None:
|
||||
"""Return the pattern document as a dict, or None if it doesn't exist.
|
||||
|
||||
Args:
|
||||
pattern_id: Firestore document ID in the ``error_patterns`` collection.
|
||||
"""
|
||||
snap = self._db.collection(_COL_PATTERNS).document(pattern_id).get()
|
||||
if not snap.exists:
|
||||
return None
|
||||
return snap.to_dict()
|
||||
|
||||
def get_active_patterns(self) -> list[dict]:
|
||||
"""Return all patterns with status in [new, acknowledged, known]."""
|
||||
query = self._db.collection(_COL_PATTERNS).where(
|
||||
"status", "in", _ACTIVE_STATUSES
|
||||
)
|
||||
return [snap.to_dict() for snap in query.stream()]
|
||||
|
||||
def get_patterns_for_auto_resolve(self) -> list[dict]:
|
||||
"""Return active patterns eligible for auto-resolution (new or acknowledged)."""
|
||||
query = self._db.collection(_COL_PATTERNS).where(
|
||||
"status", "in", list(_AUTO_RESOLVE_ELIGIBLE)
|
||||
)
|
||||
return [snap.to_dict() for snap in query.stream()]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Patterns — upsert
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def upsert_pattern(self, data: PatternData) -> UpsertResult:
|
||||
"""Create a new pattern or update an existing one.
|
||||
|
||||
New pattern:
|
||||
- status="new", severity="P3"
|
||||
- Initialize ``rolling_counts`` with one entry for this run.
|
||||
|
||||
Existing pattern:
|
||||
- Increment ``total_count`` and append to ``rolling_counts``.
|
||||
- If status was ``auto_resolved`` or ``fixed`` → reactivate (set status="new").
|
||||
- Prune ``rolling_counts`` entries older than ROLLING_WINDOW_DAYS.
|
||||
|
||||
Returns:
|
||||
UpsertResult with ``is_new=True`` for brand-new patterns *and* for
|
||||
reactivated patterns (auto_resolved → new, fixed → new).
|
||||
"""
|
||||
doc_ref = self._db.collection(_COL_PATTERNS).document(data.pattern_id)
|
||||
snap = doc_ref.get()
|
||||
ts_iso = _iso(data.timestamp)
|
||||
|
||||
new_rolling_entry = {"ts": ts_iso, "count": data.count}
|
||||
|
||||
if not snap.exists:
|
||||
# ── brand new pattern ──────────────────────────────────────────
|
||||
doc_ref.set(
|
||||
{
|
||||
"pattern_id": data.pattern_id,
|
||||
"service": data.service,
|
||||
"resource_type": data.resource_type,
|
||||
"normalized_message": data.normalized_message,
|
||||
"sample_message": data.sample_message,
|
||||
"status": "new",
|
||||
"severity": "P3",
|
||||
"total_count": data.count,
|
||||
"rolling_counts": [new_rolling_entry],
|
||||
"first_seen": ts_iso,
|
||||
"last_seen": ts_iso,
|
||||
"alerted_at": None,
|
||||
"created_at": ts_iso,
|
||||
"updated_at": ts_iso,
|
||||
}
|
||||
)
|
||||
return UpsertResult(pattern_id=data.pattern_id, is_new=True)
|
||||
|
||||
# ── existing pattern ───────────────────────────────────────────────
|
||||
existing = snap.to_dict()
|
||||
prev_status = existing.get("status", "new")
|
||||
|
||||
rolling_counts: list[dict] = list(existing.get("rolling_counts") or [])
|
||||
rolling_counts.append(new_rolling_entry)
|
||||
rolling_counts = self._prune_rolling_counts(rolling_counts)
|
||||
|
||||
update: dict[str, Any] = {
|
||||
"total_count": existing.get("total_count", 0) + data.count,
|
||||
"rolling_counts": rolling_counts,
|
||||
"last_seen": ts_iso,
|
||||
"sample_message": data.sample_message,
|
||||
"updated_at": ts_iso,
|
||||
}
|
||||
|
||||
# Reactivation: pattern was considered resolved/fixed but error recurred
|
||||
is_new = False
|
||||
if prev_status in _REACTIVATION_STATUSES:
|
||||
update["status"] = "new"
|
||||
update["alerted_at"] = None
|
||||
is_new = True
|
||||
|
||||
doc_ref.update(update)
|
||||
return UpsertResult(
|
||||
pattern_id=data.pattern_id,
|
||||
is_new=is_new,
|
||||
previous_status=prev_status,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Patterns — status changes
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def auto_resolve_pattern(self, pattern_id: str, hours_silent: float) -> None:
|
||||
"""Mark a pattern as auto-resolved.
|
||||
|
||||
Args:
|
||||
pattern_id: Firestore document ID.
|
||||
hours_silent: How many hours the pattern has been silent.
|
||||
"""
|
||||
self._db.collection(_COL_PATTERNS).document(pattern_id).update(
|
||||
{
|
||||
"status": "auto_resolved",
|
||||
"auto_resolve_hours_silent": hours_silent,
|
||||
"auto_resolved_at": _iso(_utcnow()),
|
||||
"updated_at": _iso(_utcnow()),
|
||||
}
|
||||
)
|
||||
|
||||
def resolve_pattern(
|
||||
self,
|
||||
pattern_id: str,
|
||||
pr_url: str | None = None,
|
||||
note: str = "",
|
||||
) -> None:
|
||||
"""Mark a pattern as fixed (human-resolved).
|
||||
|
||||
Args:
|
||||
pattern_id: Firestore document ID.
|
||||
pr_url: Optional URL of the PR that fixed this pattern.
|
||||
note: Optional human-readable note about the fix.
|
||||
"""
|
||||
now_iso = _iso(_utcnow())
|
||||
self._db.collection(_COL_PATTERNS).document(pattern_id).update(
|
||||
{
|
||||
"status": "fixed",
|
||||
"fixed_by": {
|
||||
"pr_url": pr_url,
|
||||
"note": note,
|
||||
"fixed_at": now_iso,
|
||||
},
|
||||
"updated_at": now_iso,
|
||||
}
|
||||
)
|
||||
|
||||
def update_pattern_alerted(self, pattern_id: str) -> None:
|
||||
"""Set the ``alerted_at`` timestamp on a pattern.
|
||||
|
||||
Called after a Discord alert has been successfully sent for the pattern.
|
||||
"""
|
||||
self._db.collection(_COL_PATTERNS).document(pattern_id).update(
|
||||
{
|
||||
"alerted_at": _iso(_utcnow()),
|
||||
"updated_at": _iso(_utcnow()),
|
||||
}
|
||||
)
|
||||
|
||||
def merge_pattern(self, source_id: str, target_id: str, reason: str) -> None:
|
||||
"""Merge *source* into *target* by muting the source pattern.
|
||||
|
||||
Args:
|
||||
source_id: Pattern to mute.
|
||||
target_id: Pattern that the source is being merged into.
|
||||
reason: Human-readable rationale for the merge.
|
||||
"""
|
||||
self._db.collection(_COL_PATTERNS).document(source_id).update(
|
||||
{
|
||||
"status": "muted",
|
||||
"merged_into": target_id,
|
||||
"merge_reason": reason,
|
||||
"updated_at": _iso(_utcnow()),
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Auto-resolve logic
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def check_auto_resolve(self, pattern: dict) -> float | None:
|
||||
"""Determine whether *pattern* should be auto-resolved.
|
||||
|
||||
Computes the frequency-aware silence threshold and compares it against
|
||||
how long the pattern has been silent.
|
||||
|
||||
Args:
|
||||
pattern: A pattern document dict (as returned by get_pattern).
|
||||
|
||||
Returns:
|
||||
The number of hours the pattern has been silent if the threshold is
|
||||
exceeded, or ``None`` if the pattern should not yet be resolved.
|
||||
"""
|
||||
status = pattern.get("status", "")
|
||||
if status not in _AUTO_RESOLVE_ELIGIBLE:
|
||||
return None
|
||||
|
||||
last_seen_str = pattern.get("last_seen")
|
||||
if not last_seen_str:
|
||||
return None
|
||||
|
||||
last_seen = datetime.fromisoformat(last_seen_str)
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
|
||||
# Ensure both datetimes are tz-aware for subtraction
|
||||
if last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=timezone.utc)
|
||||
|
||||
hours_silent = (now - last_seen).total_seconds() / 3600.0
|
||||
|
||||
rolling_counts = pattern.get("rolling_counts") or []
|
||||
threshold = self._compute_resolve_threshold(rolling_counts)
|
||||
|
||||
if hours_silent >= threshold:
|
||||
return hours_silent
|
||||
return None
|
||||
|
||||
def _compute_resolve_threshold(self, rolling_counts: list[dict]) -> float:
|
||||
"""Compute the frequency-aware auto-resolve threshold in hours.
|
||||
|
||||
Algorithm:
|
||||
1. If fewer than 3 data points → return AUTO_RESOLVE_FALLBACK_HOURS (48).
|
||||
2. Compute mean interval between consecutive timestamps.
|
||||
3. threshold = mean_interval_hours * AUTO_RESOLVE_MULTIPLIER.
|
||||
4. Clamp result to [AUTO_RESOLVE_MIN_HOURS, AUTO_RESOLVE_MAX_HOURS].
|
||||
|
||||
Args:
|
||||
rolling_counts: List of ``{"ts": ISO-string, "count": int}`` dicts.
|
||||
|
||||
Returns:
|
||||
Threshold in hours (float).
|
||||
"""
|
||||
if len(rolling_counts) < 3:
|
||||
return float(AUTO_RESOLVE_FALLBACK_HOURS)
|
||||
|
||||
# Parse timestamps and sort ascending
|
||||
timestamps: list[datetime] = []
|
||||
for entry in rolling_counts:
|
||||
ts = datetime.fromisoformat(entry["ts"])
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=timezone.utc)
|
||||
timestamps.append(ts)
|
||||
timestamps.sort()
|
||||
|
||||
# Compute pairwise intervals
|
||||
intervals_hours: list[float] = []
|
||||
for i in range(1, len(timestamps)):
|
||||
delta = (timestamps[i] - timestamps[i - 1]).total_seconds() / 3600.0
|
||||
intervals_hours.append(delta)
|
||||
|
||||
if not intervals_hours:
|
||||
return float(AUTO_RESOLVE_FALLBACK_HOURS)
|
||||
|
||||
mean_interval = sum(intervals_hours) / len(intervals_hours)
|
||||
threshold = mean_interval * AUTO_RESOLVE_MULTIPLIER
|
||||
|
||||
# Clamp to configured bounds
|
||||
threshold = max(float(AUTO_RESOLVE_MIN_HOURS), threshold)
|
||||
threshold = min(float(AUTO_RESOLVE_MAX_HOURS), threshold)
|
||||
return threshold
|
||||
|
||||
def _prune_rolling_counts(self, rolling_counts: list[dict]) -> list[dict]:
|
||||
"""Remove rolling_count entries older than ROLLING_WINDOW_DAYS.
|
||||
|
||||
Args:
|
||||
rolling_counts: List of ``{"ts": ISO-string, "count": int}`` dicts.
|
||||
|
||||
Returns:
|
||||
Filtered list with only recent entries.
|
||||
"""
|
||||
if not rolling_counts:
|
||||
return []
|
||||
|
||||
cutoff = _utcnow() - timedelta(days=ROLLING_WINDOW_DAYS)
|
||||
result = []
|
||||
for entry in rolling_counts:
|
||||
ts = datetime.fromisoformat(entry["ts"])
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=timezone.utc)
|
||||
if ts >= cutoff:
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Incidents
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def create_incident(
|
||||
self,
|
||||
title: str,
|
||||
root_cause: str,
|
||||
severity: str,
|
||||
suggested_fix: str,
|
||||
primary_service: str,
|
||||
pattern_ids: list[str],
|
||||
used_llm: bool,
|
||||
) -> str:
|
||||
"""Create a new incident document and link associated patterns.
|
||||
|
||||
Args:
|
||||
title: Short human-readable title.
|
||||
root_cause: LLM-generated or manual root-cause description.
|
||||
severity: Severity string (e.g. "P1", "P2", "P3").
|
||||
suggested_fix: Suggested remediation steps.
|
||||
primary_service: The primary affected service.
|
||||
pattern_ids: List of pattern_ids that belong to this incident.
|
||||
used_llm: Whether LLM analysis was used to generate this incident.
|
||||
|
||||
Returns:
|
||||
The generated incident_id string.
|
||||
"""
|
||||
incident_id = _generate_id("inc", "%Y%m%d_%H%M")
|
||||
now_iso = _iso(_utcnow())
|
||||
|
||||
self._db.collection(_COL_INCIDENTS).document(incident_id).set(
|
||||
{
|
||||
"incident_id": incident_id,
|
||||
"title": title,
|
||||
"root_cause": root_cause,
|
||||
"severity": severity,
|
||||
"suggested_fix": suggested_fix,
|
||||
"primary_service": primary_service,
|
||||
"pattern_ids": pattern_ids,
|
||||
"used_llm": used_llm,
|
||||
"created_at": now_iso,
|
||||
"updated_at": now_iso,
|
||||
"status": "open",
|
||||
}
|
||||
)
|
||||
|
||||
# Link each pattern back to this incident
|
||||
for pattern_id in pattern_ids:
|
||||
self._db.collection(_COL_PATTERNS).document(pattern_id).update(
|
||||
{
|
||||
"incident_id": incident_id,
|
||||
"updated_at": now_iso,
|
||||
}
|
||||
)
|
||||
|
||||
return incident_id
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Discord audit trail
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def log_discord_alert(
|
||||
self,
|
||||
alert_type: str,
|
||||
content: str,
|
||||
success: bool,
|
||||
metadata: dict | None = None,
|
||||
) -> None:
|
||||
"""Write an audit record to the ``discord_alerts`` collection.
|
||||
|
||||
Args:
|
||||
alert_type: Category of alert (e.g. "new_pattern", "digest").
|
||||
content: The message body that was (or would have been) sent.
|
||||
success: Whether the Discord webhook call succeeded.
|
||||
metadata: Optional additional data (pattern_ids, incident_id, etc.).
|
||||
"""
|
||||
alert_id = _generate_id("alert", "%Y%m%d_%H%M%S")
|
||||
now_iso = _iso(_utcnow())
|
||||
|
||||
self._db.collection(_COL_ALERTS).document(alert_id).set(
|
||||
{
|
||||
"alert_id": alert_id,
|
||||
"alert_type": alert_type,
|
||||
"content": content,
|
||||
"success": success,
|
||||
"metadata": metadata or {},
|
||||
"created_at": now_iso,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Known infrastructure noise patterns for the production error monitoring service.
|
||||
|
||||
Filters out transient, expected log messages so the error monitor is not flooded
|
||||
with Cloud Run cold-start probes, deploy events, spot VM preemptions, and similar
|
||||
operational noise.
|
||||
|
||||
Usage:
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
reason = should_ignore(service, message)
|
||||
if reason:
|
||||
# skip this log entry — it's known infrastructure noise
|
||||
...
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class IgnoreReason:
|
||||
"""Describes why a log entry should be suppressed.
|
||||
|
||||
Attributes:
|
||||
pattern_name: Machine-readable name of the matching ignore pattern.
|
||||
reason: Human-readable explanation of why this is infrastructure noise.
|
||||
"""
|
||||
|
||||
pattern_name: str
|
||||
reason: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compiled ignore patterns (applied in order)
|
||||
# All patterns use re.IGNORECASE for case-insensitive matching.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_IGNORE_PATTERNS: list[tuple[str, re.Pattern, str]] = [
|
||||
(
|
||||
"startup_probe",
|
||||
re.compile(r"startup probe failed", re.IGNORECASE),
|
||||
"Cloud Run cold start — startup probe failure is transient and expected",
|
||||
),
|
||||
(
|
||||
"ready_condition",
|
||||
re.compile(r"ready condition status changed", re.IGNORECASE),
|
||||
"Cloud Run deploy event — readiness condition change is expected on deploy",
|
||||
),
|
||||
(
|
||||
"spot_preemption",
|
||||
re.compile(r"instance was preempted", re.IGNORECASE),
|
||||
"Spot/preemptible VM preemption — expected behaviour for spot instances",
|
||||
),
|
||||
(
|
||||
"idle_shutdown",
|
||||
re.compile(r"stopping idle encoding worker", re.IGNORECASE),
|
||||
"Encoding worker idle shutdown — expected behaviour when no jobs are queued",
|
||||
),
|
||||
(
|
||||
"health_check_404",
|
||||
re.compile(
|
||||
r"(?:GET|HEAD)\b[^\n]*?/health[^\n]*?\b(?:404|not found)\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"Load balancer health check returning 404 — transient during cold start",
|
||||
),
|
||||
(
|
||||
"scheduler_retry",
|
||||
re.compile(r"cloud scheduler.*retry", re.IGNORECASE),
|
||||
"Cloud Scheduler transient retry — expected on intermittent job failures",
|
||||
),
|
||||
(
|
||||
"runner_startup",
|
||||
re.compile(r"github\.runner\..*(?:starting|stopping|idle)", re.IGNORECASE),
|
||||
"GitHub Actions runner lifecycle event — expected during runner start/stop",
|
||||
),
|
||||
(
|
||||
"container_shutdown",
|
||||
re.compile(r"container called exit\(0\)", re.IGNORECASE),
|
||||
"Container clean exit — exit code 0 indicates a graceful, expected shutdown",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def should_ignore(service: str, message: str) -> IgnoreReason | None:
|
||||
"""Check whether a log entry matches a known infrastructure noise pattern.
|
||||
|
||||
The check is purely message-based; ``service`` is accepted for API symmetry
|
||||
and future service-specific overrides but does not gate any pattern today.
|
||||
|
||||
Args:
|
||||
service: The originating service name (e.g. ``"karaoke-backend"``).
|
||||
message: The raw log message string to evaluate.
|
||||
|
||||
Returns:
|
||||
An :class:`IgnoreReason` if the message matches a known noise pattern,
|
||||
or ``None`` if the message should be processed normally.
|
||||
"""
|
||||
if not message:
|
||||
return None
|
||||
|
||||
for pattern_name, pattern, reason in _IGNORE_PATTERNS:
|
||||
if pattern.search(message):
|
||||
return IgnoreReason(pattern_name=pattern_name, reason=reason)
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,429 @@
|
||||
"""LLM-powered incident analysis and duplicate detection for the error monitor.
|
||||
|
||||
Uses Gemini Flash via the google-generativeai library to:
|
||||
1. Group related error patterns into incidents with root-cause analysis.
|
||||
2. Identify near-duplicate patterns that the regex normalizer missed.
|
||||
|
||||
Both functions fall back gracefully when the LLM is unavailable or returns
|
||||
unparseable output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from google import genai
|
||||
from google.genai import types as genai_types
|
||||
|
||||
from backend.services.error_monitor.config import (
|
||||
GCP_PROJECT,
|
||||
LLM_ANALYSIS_MODEL,
|
||||
LLM_VERTEX_LOCATION,
|
||||
MIN_PATTERNS_FOR_ANALYSIS,
|
||||
SERVICE_DEPENDENCY_MAP,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Incident:
|
||||
"""A grouped incident produced by LLM analysis or the fallback grouper."""
|
||||
|
||||
title: str
|
||||
root_cause: str | None
|
||||
severity: str # P0 / P1 / P2 / P3
|
||||
suggested_fix: str | None
|
||||
primary_service: str
|
||||
pattern_indices: list[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class IncidentAnalysis:
|
||||
"""The result of an incident grouping pass."""
|
||||
|
||||
incidents: list[Incident]
|
||||
used_llm: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class DuplicateGroup:
|
||||
"""A canonical pattern plus the new patterns that duplicate it."""
|
||||
|
||||
canonical_index: int
|
||||
duplicate_indices: list[int]
|
||||
reason: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM call helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_RETRY_ATTEMPTS = 3
|
||||
_RETRY_BASE_DELAY = 2.0 # seconds
|
||||
|
||||
|
||||
def _call_llm(
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
temperature: float = 0.2,
|
||||
) -> str:
|
||||
"""Call Gemini and return the response text.
|
||||
|
||||
Retries up to _RETRY_ATTEMPTS times with exponential back-off on failure.
|
||||
|
||||
Args:
|
||||
system_prompt: System instruction for the model.
|
||||
user_prompt: User-facing prompt content.
|
||||
temperature: Sampling temperature (lower = more deterministic).
|
||||
|
||||
Returns:
|
||||
The model's response text.
|
||||
|
||||
Raises:
|
||||
Exception: Re-raises the last exception after all retries are exhausted.
|
||||
"""
|
||||
last_exc: Exception | None = None
|
||||
|
||||
for attempt in range(_RETRY_ATTEMPTS):
|
||||
try:
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=GCP_PROJECT,
|
||||
location=LLM_VERTEX_LOCATION,
|
||||
)
|
||||
response = client.models.generate_content(
|
||||
model=LLM_ANALYSIS_MODEL,
|
||||
contents=user_prompt,
|
||||
config=genai_types.GenerateContentConfig(
|
||||
system_instruction=system_prompt,
|
||||
temperature=temperature,
|
||||
),
|
||||
)
|
||||
return response.text
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_exc = exc
|
||||
delay = _RETRY_BASE_DELAY * (2**attempt)
|
||||
logger.warning(
|
||||
"LLM call failed (attempt %d/%d): %s — retrying in %.1fs",
|
||||
attempt + 1,
|
||||
_RETRY_ATTEMPTS,
|
||||
exc,
|
||||
delay,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
raise last_exc # type: ignore[misc]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MD_CODE_BLOCK_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
|
||||
|
||||
def _strip_markdown_code_block(text: str) -> str:
|
||||
"""Remove surrounding ```json ... ``` or ``` ... ``` fences if present."""
|
||||
match = _MD_CODE_BLOCK_RE.search(text)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _parse_llm_response(text: str) -> IncidentAnalysis | None:
|
||||
"""Parse the LLM response text into an IncidentAnalysis.
|
||||
|
||||
Args:
|
||||
text: Raw LLM output, possibly wrapped in a markdown code block.
|
||||
|
||||
Returns:
|
||||
IncidentAnalysis on success, None on any parse failure.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
cleaned = _strip_markdown_code_block(text)
|
||||
|
||||
try:
|
||||
data = json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("LLM returned non-JSON response (first 200 chars): %.200s", text)
|
||||
return None
|
||||
|
||||
if "incidents" not in data:
|
||||
logger.warning("LLM response missing 'incidents' key: %s", list(data.keys()))
|
||||
return None
|
||||
|
||||
incidents: list[Incident] = []
|
||||
for raw in data["incidents"]:
|
||||
try:
|
||||
incidents.append(
|
||||
Incident(
|
||||
title=raw.get("title", "Unknown incident"),
|
||||
root_cause=raw.get("root_cause"),
|
||||
severity=raw.get("severity", "P2"),
|
||||
suggested_fix=raw.get("suggested_fix"),
|
||||
primary_service=raw.get("primary_service", "unknown"),
|
||||
pattern_indices=raw.get("pattern_indices", []),
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Skipping malformed incident entry: %s — %s", raw, exc)
|
||||
|
||||
return IncidentAnalysis(incidents=incidents, used_llm=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fallback grouper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fallback_group_by_service(patterns: list[dict]) -> IncidentAnalysis:
|
||||
"""Group patterns by service name when the LLM is unavailable.
|
||||
|
||||
Each unique service becomes one P2 incident containing all of that
|
||||
service's pattern indices.
|
||||
|
||||
Args:
|
||||
patterns: List of error pattern dicts (must include 'service' key).
|
||||
|
||||
Returns:
|
||||
IncidentAnalysis with used_llm=False.
|
||||
"""
|
||||
service_to_indices: dict[str, list[int]] = defaultdict(list)
|
||||
for idx, pattern in enumerate(patterns):
|
||||
service = pattern.get("service", "unknown")
|
||||
service_to_indices[service].append(idx)
|
||||
|
||||
incidents = [
|
||||
Incident(
|
||||
title=f"Errors in {service}",
|
||||
root_cause=None,
|
||||
severity="P2",
|
||||
suggested_fix=None,
|
||||
primary_service=service,
|
||||
pattern_indices=indices,
|
||||
)
|
||||
for service, indices in service_to_indices.items()
|
||||
]
|
||||
|
||||
return IncidentAnalysis(incidents=incidents, used_llm=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main analysis functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ANALYZE_SYSTEM_PROMPT = f"""\
|
||||
You are an expert site-reliability engineer for Nomad Karaoke, a karaoke video
|
||||
generation platform running on Google Cloud Platform.
|
||||
|
||||
Your task is to analyse a list of active error patterns detected across
|
||||
production services, group them into meaningful incidents, and identify root
|
||||
causes.
|
||||
|
||||
## Service dependency map
|
||||
|
||||
{SERVICE_DEPENDENCY_MAP}
|
||||
|
||||
## Severity definitions
|
||||
|
||||
- P0: Service is completely down / total outage
|
||||
- P1: Major pipeline broken — orders cannot complete
|
||||
- P2: Degraded performance — some orders affected
|
||||
- P3: Minor / cosmetic issue with no direct user impact
|
||||
|
||||
## Output format
|
||||
|
||||
Respond with a single JSON object (no markdown fencing) with this schema:
|
||||
|
||||
{{
|
||||
"incidents": [
|
||||
{{
|
||||
"title": "<short human-readable title>",
|
||||
"root_cause": "<root cause or null>",
|
||||
"severity": "P0|P1|P2|P3",
|
||||
"suggested_fix": "<actionable suggestion or null>",
|
||||
"primary_service": "<service name>",
|
||||
"pattern_indices": [<list of 0-based indices from the input list>]
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
Group related patterns together into a single incident where they share a
|
||||
common root cause (e.g., a downstream dependency failure). Unrelated patterns
|
||||
should be in separate incidents. Every input pattern must appear in exactly
|
||||
one incident.
|
||||
"""
|
||||
|
||||
_DEDUP_SYSTEM_PROMPT = """\
|
||||
You are an expert at identifying near-duplicate error patterns in production
|
||||
monitoring systems.
|
||||
|
||||
Your task is to compare a list of NEW error patterns against a list of
|
||||
EXISTING (canonical) patterns and identify which new patterns are essentially
|
||||
duplicates of existing ones — i.e., they describe the same underlying error
|
||||
but with slightly different wording due to variation in log messages.
|
||||
|
||||
## Output format
|
||||
|
||||
Respond with a single JSON object (no markdown fencing) with this schema:
|
||||
|
||||
{
|
||||
"duplicates": [
|
||||
{
|
||||
"canonical_index": <0-based index into the EXISTING patterns list>,
|
||||
"duplicate_indices": [<0-based indices into the NEW patterns list>],
|
||||
"reason": "<brief explanation>"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
If no duplicates are found, return {"duplicates": []}.
|
||||
"""
|
||||
|
||||
|
||||
def analyze_patterns(patterns: list[dict]) -> IncidentAnalysis | None:
|
||||
"""Group error patterns into incidents using Gemini LLM analysis.
|
||||
|
||||
Returns None if there are fewer than MIN_PATTERNS_FOR_ANALYSIS patterns.
|
||||
Falls back to service-based grouping when the LLM fails or returns an
|
||||
unparseable response.
|
||||
|
||||
Args:
|
||||
patterns: List of error pattern dicts to analyse.
|
||||
|
||||
Returns:
|
||||
IncidentAnalysis, or None if insufficient patterns.
|
||||
"""
|
||||
if len(patterns) < MIN_PATTERNS_FOR_ANALYSIS:
|
||||
return None
|
||||
|
||||
user_prompt = _build_analyze_user_prompt(patterns)
|
||||
|
||||
try:
|
||||
raw_response = _call_llm(_ANALYZE_SYSTEM_PROMPT, user_prompt, temperature=0.2)
|
||||
result = _parse_llm_response(raw_response)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
logger.warning("LLM returned unparseable response; falling back to service grouping")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("LLM analysis failed: %s — falling back to service grouping", exc)
|
||||
|
||||
return _fallback_group_by_service(patterns)
|
||||
|
||||
|
||||
def find_duplicate_patterns(
|
||||
new_patterns: list[dict],
|
||||
existing_patterns: list[dict],
|
||||
) -> list[DuplicateGroup]:
|
||||
"""Identify near-duplicate patterns that the regex normalizer missed.
|
||||
|
||||
Returns an empty list if either input list is empty, or if the LLM call
|
||||
fails for any reason.
|
||||
|
||||
Args:
|
||||
new_patterns: Newly detected error patterns to check.
|
||||
existing_patterns: Currently tracked (canonical) error patterns.
|
||||
|
||||
Returns:
|
||||
List of DuplicateGroup objects (may be empty).
|
||||
"""
|
||||
if not new_patterns or not existing_patterns:
|
||||
return []
|
||||
|
||||
user_prompt = _build_dedup_user_prompt(new_patterns, existing_patterns)
|
||||
|
||||
try:
|
||||
raw_response = _call_llm(_DEDUP_SYSTEM_PROMPT, user_prompt, temperature=0.1)
|
||||
return _parse_duplicate_response(raw_response)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Duplicate detection LLM call failed: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt builders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_analyze_user_prompt(patterns: list[dict]) -> str:
|
||||
"""Build the user prompt listing patterns for incident grouping."""
|
||||
lines = ["Error patterns to analyse:\n"]
|
||||
for idx, pattern in enumerate(patterns):
|
||||
service = pattern.get("service", "unknown")
|
||||
message = pattern.get("normalized_message", "")
|
||||
count = pattern.get("count", 0)
|
||||
lines.append(f"[{idx}] service={service!r} count={count} message={message!r}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _build_dedup_user_prompt(
|
||||
new_patterns: list[dict],
|
||||
existing_patterns: list[dict],
|
||||
) -> str:
|
||||
"""Build the user prompt listing existing and new patterns for dedup."""
|
||||
lines = ["EXISTING (canonical) patterns:\n"]
|
||||
for idx, pattern in enumerate(existing_patterns):
|
||||
service = pattern.get("service", "unknown")
|
||||
message = pattern.get("normalized_message", "")
|
||||
lines.append(f"[{idx}] service={service!r} message={message!r}")
|
||||
|
||||
lines.append("\nNEW patterns to check:\n")
|
||||
for idx, pattern in enumerate(new_patterns):
|
||||
service = pattern.get("service", "unknown")
|
||||
message = pattern.get("normalized_message", "")
|
||||
lines.append(f"[{idx}] service={service!r} message={message!r}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _parse_duplicate_response(text: str) -> list[DuplicateGroup]:
|
||||
"""Parse the LLM response for duplicate detection.
|
||||
|
||||
Args:
|
||||
text: Raw LLM output.
|
||||
|
||||
Returns:
|
||||
List of DuplicateGroup objects, or [] on any failure.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
cleaned = _strip_markdown_code_block(text)
|
||||
|
||||
try:
|
||||
data = json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Dedup LLM returned non-JSON: %.200s", text)
|
||||
return []
|
||||
|
||||
if "duplicates" not in data:
|
||||
logger.warning("Dedup LLM response missing 'duplicates' key")
|
||||
return []
|
||||
|
||||
groups: list[DuplicateGroup] = []
|
||||
for raw in data["duplicates"]:
|
||||
try:
|
||||
groups.append(
|
||||
DuplicateGroup(
|
||||
canonical_index=int(raw["canonical_index"]),
|
||||
duplicate_indices=list(raw.get("duplicate_indices", [])),
|
||||
reason=raw.get("reason", ""),
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Skipping malformed duplicate entry: %s — %s", raw, exc)
|
||||
|
||||
return groups
|
||||
@@ -0,0 +1,641 @@
|
||||
"""Monitor orchestrator for the production error monitoring service.
|
||||
|
||||
This is the main entry point. Run as::
|
||||
|
||||
python -m backend.services.error_monitor.monitor
|
||||
|
||||
The ``ErrorMonitor.run()`` method executes the full pipeline:
|
||||
|
||||
* Monitor cycle (default) — query Cloud Logging, detect new patterns, send
|
||||
Discord alerts, check auto-resolution.
|
||||
* Daily digest mode (``DIGEST_MODE=true``) — bundle 24h stats into one message.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from google.cloud import logging as cloud_logging
|
||||
from google.cloud import secretmanager
|
||||
|
||||
from backend.services.error_monitor.config import (
|
||||
GCP_PROJECT,
|
||||
LOOKBACK_MINUTES,
|
||||
MAX_LOG_ENTRIES,
|
||||
MONITORED_CLOUD_FUNCTIONS,
|
||||
MONITORED_CLOUD_RUN_JOBS,
|
||||
MONITORED_CLOUD_RUN_SERVICES,
|
||||
MONITORED_GCE_INSTANCES,
|
||||
SPIKE_MIN_COUNT,
|
||||
SPIKE_MULTIPLIER,
|
||||
get_digest_mode,
|
||||
get_discord_webhook_secret_name,
|
||||
get_llm_enabled,
|
||||
)
|
||||
from backend.services.error_monitor.discord import (
|
||||
ErrorMonitorDiscord,
|
||||
format_auto_resolved_alert,
|
||||
format_daily_digest,
|
||||
format_incident_alert,
|
||||
format_new_pattern_alert,
|
||||
format_spike_alert,
|
||||
)
|
||||
from backend.services.error_monitor.firestore_adapter import (
|
||||
ErrorPatternsAdapter,
|
||||
PatternData,
|
||||
UpsertResult,
|
||||
)
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns, find_duplicate_patterns
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash, normalize_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Maps log resource type → (label_key, service_name_list)
|
||||
RESOURCE_TYPE_FILTERS: dict[str, tuple[str, list[str]]] = {
|
||||
"cloud_run_revision": ("resource.labels.service_name", MONITORED_CLOUD_RUN_SERVICES),
|
||||
"cloud_run_job": ("resource.labels.job_name", MONITORED_CLOUD_RUN_JOBS),
|
||||
"cloud_function": ("resource.labels.function_name", MONITORED_CLOUD_FUNCTIONS),
|
||||
"gce_instance": ('labels."compute.googleapis.com/resource_name"', MONITORED_GCE_INSTANCES),
|
||||
}
|
||||
|
||||
#: Maps log resource type → canonical resource_type string stored in Firestore.
|
||||
RESOURCE_TYPE_MAP: dict[str, str] = {
|
||||
"cloud_run_revision": "cloud_run_service",
|
||||
"cloud_run_job": "cloud_run_job",
|
||||
"cloud_function": "cloud_function",
|
||||
"gce_instance": "gce_instance",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Free functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_log_filter(
|
||||
resource_type: str,
|
||||
resource_names: list[str],
|
||||
lookback_minutes: int,
|
||||
) -> str:
|
||||
"""Build a Cloud Logging filter string.
|
||||
|
||||
Args:
|
||||
resource_type: GCP resource type (e.g. ``"cloud_run_revision"``).
|
||||
resource_names: List of service/job/function/instance names to include.
|
||||
lookback_minutes: How many minutes back to query.
|
||||
|
||||
Returns:
|
||||
A filter string suitable for ``logging_client.list_entries(filter_=...)``.
|
||||
"""
|
||||
label_key = RESOURCE_TYPE_FILTERS[resource_type][0]
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(minutes=lookback_minutes)
|
||||
cutoff_iso = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
name_clause = " OR ".join(f'{label_key}="{name}"' for name in resource_names)
|
||||
if len(resource_names) > 1:
|
||||
name_clause = f"({name_clause})"
|
||||
|
||||
return (
|
||||
f'resource.type="{resource_type}" '
|
||||
f"severity>=ERROR "
|
||||
f'timestamp>="{cutoff_iso}" '
|
||||
f"{name_clause}"
|
||||
)
|
||||
|
||||
|
||||
def _extract_message(entry: Any) -> str | None:
|
||||
"""Extract the log message text from a Cloud Logging entry.
|
||||
|
||||
Handles several payload shapes:
|
||||
|
||||
* ``str`` payload — returned directly.
|
||||
* ``dict`` payload — tries ``"message"``, ``"textPayload"``, ``"error"``,
|
||||
``"msg"`` keys in that order.
|
||||
* ``None`` payload — returns ``None``.
|
||||
|
||||
Args:
|
||||
entry: A Cloud Logging ``LogEntry`` object.
|
||||
|
||||
Returns:
|
||||
The message string, or ``None`` if not extractable.
|
||||
"""
|
||||
payload = entry.payload
|
||||
if payload is None:
|
||||
return None
|
||||
if isinstance(payload, str):
|
||||
return payload if payload else None
|
||||
if isinstance(payload, dict):
|
||||
for key in ("message", "textPayload", "error", "msg"):
|
||||
value = payload.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
return None
|
||||
# Fallback: convert to string representation
|
||||
text = str(payload)
|
||||
return text if text else None
|
||||
|
||||
|
||||
def _extract_service_name(entry: Any, resource_type: str) -> str:
|
||||
"""Extract the service/job/function/instance name from a log entry.
|
||||
|
||||
Args:
|
||||
entry: A Cloud Logging ``LogEntry`` object.
|
||||
resource_type: GCP log resource type string.
|
||||
|
||||
Returns:
|
||||
The service name string (falls back to ``"unknown"`` if not found).
|
||||
"""
|
||||
labels = getattr(entry.resource, "labels", {}) or {}
|
||||
entry_labels = getattr(entry, "labels", {}) or {}
|
||||
|
||||
if resource_type == "cloud_run_revision":
|
||||
return labels.get("service_name") or "unknown"
|
||||
if resource_type == "cloud_run_job":
|
||||
return labels.get("job_name") or "unknown"
|
||||
if resource_type == "cloud_function":
|
||||
return labels.get("function_name") or "unknown"
|
||||
if resource_type == "gce_instance":
|
||||
# Prefer the human-readable resource name label
|
||||
name = entry_labels.get("compute.googleapis.com/resource_name")
|
||||
if name:
|
||||
return name
|
||||
return labels.get("instance_id") or "unknown"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _classify_resource_type(log_resource_type: str) -> str:
|
||||
"""Map a Cloud Logging resource type to the canonical type stored in Firestore.
|
||||
|
||||
Args:
|
||||
log_resource_type: Raw GCP resource type (e.g. ``"cloud_run_revision"``).
|
||||
|
||||
Returns:
|
||||
Canonical resource type string (falls back to the input if not in map).
|
||||
"""
|
||||
return RESOURCE_TYPE_MAP.get(log_resource_type, log_resource_type)
|
||||
|
||||
|
||||
def _group_by_pattern(entries: list[dict]) -> dict[str, dict]:
|
||||
"""Group normalised log entries by their pattern hash.
|
||||
|
||||
Args:
|
||||
entries: List of dicts with ``service``, ``normalized``, and ``message`` keys.
|
||||
|
||||
Returns:
|
||||
Dict mapping ``pattern_id`` → group dict containing ``service``,
|
||||
``resource_type``, ``normalized``, ``sample_message``, ``count``.
|
||||
"""
|
||||
groups: dict[str, dict] = {}
|
||||
for entry in entries:
|
||||
service = entry["service"]
|
||||
normalized = entry["normalized"]
|
||||
pattern_id = compute_pattern_hash(service, normalized)
|
||||
|
||||
if pattern_id not in groups:
|
||||
groups[pattern_id] = {
|
||||
"pattern_id": pattern_id,
|
||||
"service": service,
|
||||
"resource_type": entry.get("resource_type", "unknown"),
|
||||
"normalized": normalized,
|
||||
"sample_message": entry["message"],
|
||||
"count": 0,
|
||||
}
|
||||
groups[pattern_id]["count"] += 1
|
||||
return groups
|
||||
|
||||
|
||||
def _rolling_average(rolling_counts: list[dict]) -> float:
|
||||
"""Compute the mean count from a list of rolling_count dicts.
|
||||
|
||||
Args:
|
||||
rolling_counts: List of ``{"count": int, ...}`` dicts.
|
||||
|
||||
Returns:
|
||||
The arithmetic mean as a float, or ``0.0`` for an empty list.
|
||||
"""
|
||||
if not rolling_counts:
|
||||
return 0.0
|
||||
total = sum(entry.get("count", 0) for entry in rolling_counts)
|
||||
return total / len(rolling_counts)
|
||||
|
||||
|
||||
def _is_spike(current_count: int, rolling_counts: list[dict]) -> bool:
|
||||
"""Determine whether ``current_count`` represents an error rate spike.
|
||||
|
||||
A spike is detected when:
|
||||
- ``current_count`` >= ``SPIKE_MIN_COUNT`` (avoids noise from rare errors), AND
|
||||
- ``current_count`` > ``rolling_average * SPIKE_MULTIPLIER``
|
||||
|
||||
Args:
|
||||
current_count: Error count in the current monitoring window.
|
||||
rolling_counts: Historical rolling counts from Firestore.
|
||||
|
||||
Returns:
|
||||
``True`` if a spike is detected, ``False`` otherwise.
|
||||
"""
|
||||
if not rolling_counts:
|
||||
return False
|
||||
if current_count < SPIKE_MIN_COUNT:
|
||||
return False
|
||||
avg = _rolling_average(rolling_counts)
|
||||
if avg == 0.0:
|
||||
return False
|
||||
return current_count > avg * SPIKE_MULTIPLIER
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ErrorMonitor class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ErrorMonitor:
|
||||
"""Orchestrates the full production error monitoring pipeline.
|
||||
|
||||
On each run (typically every 15 minutes via Cloud Scheduler), it:
|
||||
|
||||
1. Queries Cloud Logging for recent errors across all monitored services.
|
||||
2. Groups errors into patterns using normalisation + hashing.
|
||||
3. Upserts patterns to Firestore, tracking new vs. recurrent.
|
||||
4. Optionally uses an LLM to deduplicate patterns and create incidents.
|
||||
5. Sends Discord alerts for new patterns, incidents, and spikes.
|
||||
6. Auto-resolves patterns that have been silent long enough.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.logging_client = cloud_logging.Client(project=GCP_PROJECT)
|
||||
self.firestore_adapter = ErrorPatternsAdapter()
|
||||
self.discord = ErrorMonitorDiscord(webhook_url=self._get_discord_webhook())
|
||||
|
||||
def _get_discord_webhook(self) -> str:
|
||||
"""Retrieve the Discord webhook URL.
|
||||
|
||||
Checks ``DISCORD_WEBHOOK_URL`` environment variable first; falls back
|
||||
to Secret Manager.
|
||||
|
||||
Returns:
|
||||
Discord webhook URL string.
|
||||
"""
|
||||
url = os.environ.get("DISCORD_WEBHOOK_URL")
|
||||
if url:
|
||||
return url
|
||||
|
||||
secret_name = get_discord_webhook_secret_name()
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
secret_path = f"projects/{GCP_PROJECT}/secrets/{secret_name}/versions/latest"
|
||||
response = client.access_secret_version(name=secret_path)
|
||||
return response.payload.data.decode("utf-8").strip()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self) -> None:
|
||||
"""Execute the monitor pipeline (digest or cycle mode)."""
|
||||
if get_digest_mode():
|
||||
self._run_daily_digest()
|
||||
else:
|
||||
self._run_monitor_cycle()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Monitor cycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_monitor_cycle(self) -> None:
|
||||
"""Execute one monitor cycle: collect → group → upsert → alert → auto-resolve."""
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
now_iso = now.isoformat()
|
||||
|
||||
# ── Step 1: Collect log entries ────────────────────────────────────
|
||||
raw_entries: list[dict] = []
|
||||
for resource_type, (_, resource_names) in RESOURCE_TYPE_FILTERS.items():
|
||||
log_filter = _build_log_filter(resource_type, resource_names, LOOKBACK_MINUTES)
|
||||
logger.info("Querying %s with filter: %s", resource_type, log_filter)
|
||||
try:
|
||||
entries = self.logging_client.list_entries(
|
||||
filter_=log_filter,
|
||||
max_results=MAX_LOG_ENTRIES,
|
||||
)
|
||||
for entry in entries:
|
||||
message = _extract_message(entry)
|
||||
if not message:
|
||||
continue
|
||||
service = _extract_service_name(entry, resource_type)
|
||||
ignore_reason = should_ignore(service, message)
|
||||
if ignore_reason:
|
||||
logger.debug(
|
||||
"Ignoring entry from %s (%s): %s",
|
||||
service,
|
||||
ignore_reason.pattern_name,
|
||||
ignore_reason.reason,
|
||||
)
|
||||
continue
|
||||
normalized = normalize_message(message)
|
||||
if not normalized:
|
||||
continue
|
||||
raw_entries.append(
|
||||
{
|
||||
"service": service,
|
||||
"resource_type": _classify_resource_type(resource_type),
|
||||
"normalized": normalized,
|
||||
"message": message,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("Failed to query %s logs: %s", resource_type, exc)
|
||||
|
||||
logger.info("Collected %d log entries after filtering", len(raw_entries))
|
||||
|
||||
# ── Step 2: Group by pattern ───────────────────────────────────────
|
||||
groups = _group_by_pattern(raw_entries)
|
||||
logger.info("Grouped into %d distinct patterns", len(groups))
|
||||
|
||||
if not groups:
|
||||
logger.info("No new error entries found this cycle; checking auto-resolve.")
|
||||
self._check_auto_resolve()
|
||||
return
|
||||
|
||||
# ── Step 3: Upsert to Firestore ────────────────────────────────────
|
||||
new_pattern_ids: list[str] = []
|
||||
spike_patterns: list[dict] = []
|
||||
|
||||
for pattern_id, group in groups.items():
|
||||
data = PatternData(
|
||||
pattern_id=pattern_id,
|
||||
service=group["service"],
|
||||
resource_type=group["resource_type"],
|
||||
normalized_message=group["normalized"],
|
||||
sample_message=group["sample_message"],
|
||||
count=group["count"],
|
||||
timestamp=now,
|
||||
)
|
||||
result: UpsertResult = self.firestore_adapter.upsert_pattern(data)
|
||||
if result.is_new:
|
||||
new_pattern_ids.append(pattern_id)
|
||||
logger.info("New pattern detected: %s (%s)", pattern_id, group["service"])
|
||||
|
||||
# Check for spike on existing patterns
|
||||
existing = self.firestore_adapter.get_pattern(pattern_id)
|
||||
if existing and not result.is_new:
|
||||
rolling = existing.get("rolling_counts") or []
|
||||
if _is_spike(group["count"], rolling):
|
||||
spike_patterns.append(
|
||||
{
|
||||
"service": group["service"],
|
||||
"normalized_message": group["normalized"],
|
||||
"current_count": group["count"],
|
||||
"rolling_average": _rolling_average(rolling),
|
||||
}
|
||||
)
|
||||
logger.info("Spike detected for pattern %s", pattern_id)
|
||||
|
||||
# ── Step 4: LLM dedup ─────────────────────────────────────────────
|
||||
if new_pattern_ids and get_llm_enabled():
|
||||
new_patterns_data = [
|
||||
{
|
||||
"service": groups[pid]["service"],
|
||||
"normalized_message": groups[pid]["normalized"],
|
||||
"count": groups[pid]["count"],
|
||||
}
|
||||
for pid in new_pattern_ids
|
||||
]
|
||||
existing_patterns = self.firestore_adapter.get_active_patterns()
|
||||
# Exclude newly added patterns from the canonical list
|
||||
canonical = [p for p in existing_patterns if p.get("pattern_id") not in new_pattern_ids]
|
||||
|
||||
if canonical:
|
||||
duplicate_groups = find_duplicate_patterns(new_patterns_data, canonical)
|
||||
for dup_group in duplicate_groups:
|
||||
# Merge each duplicate new pattern into the canonical
|
||||
canonical_pattern = canonical[dup_group.canonical_index]
|
||||
canonical_id = canonical_pattern.get("pattern_id", "")
|
||||
for dup_idx in dup_group.duplicate_indices:
|
||||
if dup_idx < len(new_pattern_ids):
|
||||
dup_id = new_pattern_ids[dup_idx]
|
||||
logger.info(
|
||||
"Merging duplicate pattern %s into %s: %s",
|
||||
dup_id,
|
||||
canonical_id,
|
||||
dup_group.reason,
|
||||
)
|
||||
self.firestore_adapter.merge_pattern(dup_id, canonical_id, dup_group.reason)
|
||||
# Remove from new_pattern_ids since it was merged
|
||||
new_pattern_ids = [pid for pid in new_pattern_ids if pid != dup_id]
|
||||
|
||||
# ── Step 5: LLM incident analysis ─────────────────────────────────
|
||||
incident_pattern_ids: set[str] = set()
|
||||
|
||||
if len(new_pattern_ids) >= 2 and get_llm_enabled():
|
||||
patterns_for_analysis = [
|
||||
{
|
||||
"service": groups[pid]["service"],
|
||||
"normalized_message": groups[pid]["normalized"],
|
||||
"count": groups[pid]["count"],
|
||||
}
|
||||
for pid in new_pattern_ids
|
||||
if pid in groups
|
||||
]
|
||||
analysis = analyze_patterns(patterns_for_analysis)
|
||||
|
||||
if analysis and analysis.incidents:
|
||||
for incident in analysis.incidents:
|
||||
incident_pids = [
|
||||
new_pattern_ids[idx]
|
||||
for idx in incident.pattern_indices
|
||||
if idx < len(new_pattern_ids)
|
||||
]
|
||||
if not incident_pids:
|
||||
continue
|
||||
|
||||
incident_id = self.firestore_adapter.create_incident(
|
||||
title=incident.title,
|
||||
root_cause=incident.root_cause or "",
|
||||
severity=incident.severity,
|
||||
suggested_fix=incident.suggested_fix or "",
|
||||
primary_service=incident.primary_service,
|
||||
pattern_ids=incident_pids,
|
||||
used_llm=analysis.used_llm,
|
||||
)
|
||||
logger.info(
|
||||
"Created incident %s (%s) with %d patterns",
|
||||
incident_id,
|
||||
incident.severity,
|
||||
len(incident_pids),
|
||||
)
|
||||
|
||||
# Build pattern list for the Discord alert
|
||||
pattern_dicts = [
|
||||
{
|
||||
"service": groups[pid]["service"],
|
||||
"normalized_message": groups[pid]["normalized"],
|
||||
"count": groups[pid]["count"],
|
||||
}
|
||||
for pid in incident_pids
|
||||
if pid in groups
|
||||
]
|
||||
msg = format_incident_alert(
|
||||
title=incident.title,
|
||||
severity=incident.severity,
|
||||
root_cause=incident.root_cause or "Unknown",
|
||||
suggested_fix=incident.suggested_fix or "Investigate logs",
|
||||
primary_service=incident.primary_service,
|
||||
patterns=pattern_dicts,
|
||||
)
|
||||
if self.discord.send_message(msg):
|
||||
for pid in incident_pids:
|
||||
try:
|
||||
self.firestore_adapter.update_pattern_alerted(pid)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Could not update alerted_at for %s: %s", pid, exc)
|
||||
|
||||
incident_pattern_ids.update(incident_pids)
|
||||
|
||||
# ── Step 6: Individual alerts for new patterns not in incidents ────
|
||||
for pattern_id in new_pattern_ids:
|
||||
if pattern_id in incident_pattern_ids:
|
||||
continue
|
||||
if pattern_id not in groups:
|
||||
continue
|
||||
group = groups[pattern_id]
|
||||
|
||||
# Fetch fresh copy to get first_seen timestamp
|
||||
existing = self.firestore_adapter.get_pattern(pattern_id)
|
||||
first_seen = existing.get("first_seen", now_iso) if existing else now_iso
|
||||
|
||||
msg = format_new_pattern_alert(
|
||||
service=group["service"],
|
||||
resource_type=group["resource_type"],
|
||||
normalized_message=group["normalized"],
|
||||
sample_message=group["sample_message"],
|
||||
count=group["count"],
|
||||
first_seen=first_seen,
|
||||
)
|
||||
if self.discord.send_message(msg):
|
||||
try:
|
||||
self.firestore_adapter.update_pattern_alerted(pattern_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Could not update alerted_at for %s: %s", pattern_id, exc)
|
||||
|
||||
# ── Step 7: Spike alerts ───────────────────────────────────────────
|
||||
for spike in spike_patterns:
|
||||
msg = format_spike_alert(
|
||||
service=spike["service"],
|
||||
normalized_message=spike["normalized_message"],
|
||||
current_count=spike["current_count"],
|
||||
rolling_average=spike["rolling_average"],
|
||||
)
|
||||
self.discord.send_message(msg)
|
||||
|
||||
# ── Step 8: Auto-resolve check ─────────────────────────────────────
|
||||
self._check_auto_resolve()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Auto-resolve helper
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _check_auto_resolve(self) -> None:
|
||||
"""Check all active patterns for auto-resolution and send a consolidated alert."""
|
||||
patterns = self.firestore_adapter.get_patterns_for_auto_resolve()
|
||||
resolved: list[dict] = []
|
||||
|
||||
for pattern in patterns:
|
||||
pattern_id = pattern.get("pattern_id", "")
|
||||
hours_silent = self.firestore_adapter.check_auto_resolve(pattern)
|
||||
if hours_silent is not None:
|
||||
logger.info(
|
||||
"Auto-resolving pattern %s (silent %.1f hours)", pattern_id, hours_silent
|
||||
)
|
||||
self.firestore_adapter.auto_resolve_pattern(pattern_id, round(hours_silent, 1))
|
||||
resolved.append(
|
||||
{
|
||||
"service": pattern.get("service", "unknown"),
|
||||
"message": pattern.get("normalized_message", ""),
|
||||
"hours_silent": round(hours_silent, 1),
|
||||
}
|
||||
)
|
||||
|
||||
if resolved:
|
||||
msg = format_auto_resolved_alert(resolved)
|
||||
self.discord.send_message(msg)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Daily digest
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_daily_digest(self) -> None:
|
||||
"""Gather 24h stats and send a daily digest to Discord."""
|
||||
# Optional dedup sweep (best-effort; LLM errors are non-fatal)
|
||||
if get_llm_enabled():
|
||||
try:
|
||||
active = self.firestore_adapter.get_active_patterns()
|
||||
if len(active) >= 2:
|
||||
mid = len(active) // 2
|
||||
duplicates = find_duplicate_patterns(active[:mid], active[mid:])
|
||||
for dup in duplicates:
|
||||
for idx in dup.duplicate_indices:
|
||||
target_half = active[mid:]
|
||||
source_half = active[:mid]
|
||||
if idx < len(target_half) and dup.canonical_index < len(source_half):
|
||||
self.firestore_adapter.merge_pattern(
|
||||
target_half[idx]["pattern_id"],
|
||||
source_half[dup.canonical_index]["pattern_id"],
|
||||
dup.reason,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Digest dedup sweep failed (non-fatal): %s", exc)
|
||||
|
||||
# Gather stats from active patterns
|
||||
active_patterns = self.firestore_adapter.get_active_patterns()
|
||||
per_service_counts: dict[str, int] = {}
|
||||
new_count = 0
|
||||
total_errors = 0
|
||||
|
||||
for pattern in active_patterns:
|
||||
service = pattern.get("service", "unknown")
|
||||
count = pattern.get("total_count", 0)
|
||||
per_service_counts[service] = per_service_counts.get(service, 0) + count
|
||||
total_errors += count
|
||||
if pattern.get("status") == "new":
|
||||
new_count += 1
|
||||
|
||||
# Count recently auto-resolved patterns
|
||||
resolved_patterns = self.firestore_adapter.get_patterns_for_auto_resolve()
|
||||
resolved_count = sum(
|
||||
1
|
||||
for p in resolved_patterns
|
||||
if self.firestore_adapter.check_auto_resolve(p) is not None
|
||||
)
|
||||
|
||||
msg = format_daily_digest(
|
||||
total_errors=total_errors,
|
||||
new_patterns=new_count,
|
||||
resolved_patterns=resolved_count,
|
||||
active_patterns=len(active_patterns),
|
||||
per_service_counts=per_service_counts,
|
||||
)
|
||||
self.discord.send_message(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Configure logging and run the error monitor."""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
monitor = ErrorMonitor()
|
||||
monitor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Message normalizer for the production error monitoring service.
|
||||
|
||||
Replaces variable parts of error messages (IDs, timestamps, paths, etc.)
|
||||
with stable placeholders so the same logical error always produces the same
|
||||
pattern hash, regardless of which specific resource or timestamp was involved.
|
||||
|
||||
Normalizer rules are applied in order — more specific patterns first.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compiled regex patterns (applied in order — more specific first)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_NORMALIZERS: list[tuple[re.Pattern, str]] = [
|
||||
# 1. GCS paths: gs://bucket/path/to/object → gs://<BUCKET>/<PATH>
|
||||
# Must come before Firestore and URL normalization.
|
||||
(
|
||||
re.compile(r"gs://([^/\s]+)/([^\s]+)"),
|
||||
r"gs://<BUCKET>/<PATH>",
|
||||
),
|
||||
# 1b. GCS bucket root (no object path): gs://bucket/ or gs://bucket
|
||||
# Uses a lookahead so the slash/end-of-string is not consumed, and uses
|
||||
# a character class that excludes '<' to avoid re-matching placeholders.
|
||||
(
|
||||
re.compile(r"gs://([^/<\s]+)(?=[/\s]|$)"),
|
||||
r"gs://<BUCKET>",
|
||||
),
|
||||
# 2. Firestore document paths (collection/doc/…/doc — at least 2 segments)
|
||||
# Negative lookbehind for '.' prevents matching inside domain names (URLs).
|
||||
# Negative lookbehind for '/' prevents matching inside already-replaced paths.
|
||||
(
|
||||
re.compile(
|
||||
r"(?<![./])\b([a-zA-Z_][a-zA-Z0-9_-]*/[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_-]+){1,})\b"
|
||||
),
|
||||
r"<DOC_PATH>",
|
||||
),
|
||||
# 3. UUIDs (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)
|
||||
(
|
||||
re.compile(
|
||||
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
|
||||
),
|
||||
"<ID>",
|
||||
),
|
||||
# 4. ISO 8601 timestamps (2024-01-15T10:30:45Z / …+05:30 / ….123Z)
|
||||
(
|
||||
re.compile(
|
||||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})"
|
||||
),
|
||||
"<TS>",
|
||||
),
|
||||
# 5. Epoch timestamps: 10+ digit number optionally followed by fractional seconds
|
||||
(
|
||||
re.compile(r"\b\d{10,}(?:\.\d+)?\b"),
|
||||
"<EPOCH>",
|
||||
),
|
||||
# 6. Email addresses
|
||||
(
|
||||
re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"),
|
||||
"<EMAIL>",
|
||||
),
|
||||
# 7. IP addresses (IPv4) — must come before large-number replacement
|
||||
(
|
||||
re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"),
|
||||
"<IP>",
|
||||
),
|
||||
# 8. URLs — strip to domain only (https://host/path → https://host)
|
||||
# Handles http and https; preserves the scheme+domain for context.
|
||||
(
|
||||
re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"),
|
||||
r"https://\1",
|
||||
),
|
||||
# 9. Job IDs in paths (/jobs/<alphanumeric>)
|
||||
(
|
||||
re.compile(r"/jobs/[A-Za-z0-9]+"),
|
||||
"/jobs/<ID>",
|
||||
),
|
||||
# 10. Firebase UIDs (20–28 mixed-case alphanumeric characters)
|
||||
# Firebase Auth UIDs are typically 28 chars; allow 20-28 for flexibility.
|
||||
# Must contain both letters and digits (to avoid replacing plain words).
|
||||
(
|
||||
re.compile(r"\b(?=[A-Za-z0-9]{20,28}\b)(?=[A-Za-z0-9]*[A-Z])(?=[A-Za-z0-9]*[a-z])(?=[A-Za-z0-9]*\d)[A-Za-z0-9]{20,28}\b"),
|
||||
"<UID>",
|
||||
),
|
||||
# 11. Hex/alphanumeric IDs: 8+ chars containing both letters and digits
|
||||
# (e.g. git SHAs, Cloud Run revision IDs, job IDs)
|
||||
# Uses a lookahead to require at least one digit AND one letter.
|
||||
(
|
||||
re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"),
|
||||
"<ID>",
|
||||
),
|
||||
# 12. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
|
||||
(
|
||||
re.compile(r"\b\d{4,}\b"),
|
||||
"<NUM>",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def normalize_message(message: str) -> str:
|
||||
"""Normalize a raw log message by replacing variable parts with placeholders.
|
||||
|
||||
Applies each normalizer rule in order (more specific patterns first) and
|
||||
truncates the result to MAX_NORMALIZED_MESSAGE_LENGTH characters.
|
||||
|
||||
Args:
|
||||
message: Raw log message string.
|
||||
|
||||
Returns:
|
||||
Normalized message string, at most MAX_NORMALIZED_MESSAGE_LENGTH chars.
|
||||
"""
|
||||
if not message:
|
||||
return ""
|
||||
|
||||
result = message
|
||||
for pattern, replacement in _NORMALIZERS:
|
||||
result = pattern.sub(replacement, result)
|
||||
|
||||
return result[:MAX_NORMALIZED_MESSAGE_LENGTH]
|
||||
|
||||
|
||||
def compute_pattern_hash(service: str, normalized_message: str) -> str:
|
||||
"""Compute a stable SHA-256 fingerprint for an error pattern.
|
||||
|
||||
Args:
|
||||
service: The originating service name (e.g. "karaoke-backend").
|
||||
normalized_message: The output of normalize_message().
|
||||
|
||||
Returns:
|
||||
64-character lowercase hex SHA-256 digest.
|
||||
"""
|
||||
payload = f"{service}:{normalized_message}"
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
@@ -36,6 +36,21 @@
|
||||
Uses immutable deployment pattern - see infrastructure/encoding-worker/README.md.
|
||||
```
|
||||
|
||||
### Error Monitor
|
||||
|
||||
```text
|
||||
Cloud Logging (all services) ──→ Error Monitor (Cloud Run Job, */15 min)
|
||||
│
|
||||
├─ Normalize & deduplicate
|
||||
├─ LLM incident grouping (Gemini Flash)
|
||||
├─ Discord alerts (new patterns, spikes, daily digest)
|
||||
├─ Firestore: error_patterns, error_incidents
|
||||
└─ Auto-resolve silent patterns
|
||||
|
||||
Entry point: python -m backend.services.error_monitor.monitor
|
||||
Design spec: docs/archive/2026-04-12-error-monitor-design.md
|
||||
```
|
||||
|
||||
## Processing Pipeline
|
||||
|
||||
```text
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
| **Review reminder & auto-expiry** | Working (24h reminder, 48h cancel + refund) |
|
||||
| **i18n / Multilingual (frontend)** | Working (next-intl, en/es/de, 1062 strings, [locale] routing) |
|
||||
| **Referral system** | Working (referral links with configurable discounts, 20% cash kickback via Stripe Connect, auto-payout at $20) |
|
||||
| **Error monitor** | New (Cloud Run Job every 15 min — log collection, normalization, LLM dedup/grouping, Discord alerts, auto-resolution, daily digest) |
|
||||
|
||||
## Known Issues
|
||||
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
# Nomad Karaoke Error Monitor — Design Spec
|
||||
|
||||
> **Date:** 2026-04-12
|
||||
> **Status:** Approved
|
||||
> **Scope:** Error Monitor service for production error detection, normalization, dedup, alerting, and resolution tracking across all Nomad Karaoke services.
|
||||
> **Reference:** Based on the Aquarius Error Bot system (`/Users/andrew/Projects/aquarius/docs/archive/error-bot-system-reference.md`)
|
||||
|
||||
---
|
||||
|
||||
## 1. System Overview
|
||||
|
||||
A Cloud Run Job that runs every 15 minutes, queries GCP Cloud Logging for errors across all production services, normalizes and deduplicates them, uses Gemini Flash to group related errors into incidents, sends alerts to Discord, tracks patterns in Firestore, and auto-resolves patterns that go silent.
|
||||
|
||||
### End-to-End Flow
|
||||
|
||||
```
|
||||
Production Errors (Cloud Logging)
|
||||
|
|
||||
v
|
||||
Error Monitor (Cloud Run Job, every 15 min)
|
||||
|-- Query Cloud Logging (ERROR severity, last 15 min)
|
||||
| |-- cloud_run_revision: karaoke-backend, karaoke-decide, audio-separator
|
||||
| |-- cloud_run_job: video-encoding-job, lyrics-transcription-job,
|
||||
| | audio-separation-job, audio-download-job
|
||||
| |-- cloud_function: gdrive-validator, runner_manager, backup_to_aws,
|
||||
| | divebar_mirror, kn_data_sync, divebar_lookup,
|
||||
| | encoding_worker_idle
|
||||
| |-- gce_instance: encoding-worker-a, encoding-worker-b,
|
||||
| flacfetch-vm, divebar-sync-vm
|
||||
|-- Normalize messages (strip IDs, timestamps, emails, job IDs, GCS paths)
|
||||
|-- Group by pattern hash (SHA-256 of service::normalized_message)
|
||||
|-- Check Firestore for existing patterns
|
||||
|-- LLM dedup: check new patterns against existing (Gemini Flash)
|
||||
|-- If 2+ new patterns: LLM incident analysis (group by root cause)
|
||||
|-- Send Discord alerts (per-incident or per-pattern)
|
||||
|-- Auto-resolve stale patterns (frequency-aware thresholds)
|
||||
|
|
||||
v
|
||||
Daily Digest (08:00 UTC, same job with DIGEST_MODE=true)
|
||||
|-- Dedup sweep of all active patterns
|
||||
|-- 24h summary: total errors, new patterns, frequency stats
|
||||
|-- Per-service breakdown
|
||||
|-- Send digest to Discord
|
||||
```
|
||||
|
||||
### Monitored Resources
|
||||
|
||||
| Category | Resources |
|
||||
|----------|-----------|
|
||||
| **Cloud Run Services** | karaoke-backend, karaoke-decide, audio-separator |
|
||||
| **Cloud Run Jobs** | video-encoding-job, lyrics-transcription-job, audio-separation-job, audio-download-job |
|
||||
| **Cloud Functions** | gdrive-validator, runner_manager, backup_to_aws, divebar_mirror, kn_data_sync, divebar_lookup, encoding_worker_idle |
|
||||
| **GCE Instances** | encoding-worker-a, encoding-worker-b, flacfetch-vm, divebar-sync-vm |
|
||||
|
||||
---
|
||||
|
||||
## 2. Log Collection
|
||||
|
||||
Four separate Cloud Logging queries, one per resource type, each scoped to:
|
||||
- Severity: `>=ERROR`
|
||||
- Time window: last 15 minutes (configurable via `LOOKBACK_MINUTES`)
|
||||
- Max entries: 500 total across all queries
|
||||
|
||||
### Query Resource Types
|
||||
|
||||
| Resource Type | Filter Field | Values |
|
||||
|---------------|-------------|--------|
|
||||
| `cloud_run_revision` | `resource.labels.service_name` | karaoke-backend, karaoke-decide, audio-separator |
|
||||
| `cloud_run_job` | `resource.labels.job_name` | video-encoding-job, lyrics-transcription-job, audio-separation-job, audio-download-job |
|
||||
| `cloud_function` | `resource.labels.function_name` | gdrive-validator, runner_manager, backup_to_aws, divebar_mirror, kn_data_sync, divebar_lookup, encoding_worker_idle |
|
||||
| `gce_instance` | `labels."compute.googleapis.com/resource_name"` | encoding-worker-a, encoding-worker-b, flacfetch-vm, divebar-sync-vm |
|
||||
|
||||
### Payload Handling
|
||||
|
||||
Three payload types (same as Aquarius):
|
||||
- Structured JSON dict → extract message from `textPayload` or `jsonPayload.message`
|
||||
- Plain text string → use directly
|
||||
- HTTP request logs (no payload) → skip
|
||||
|
||||
---
|
||||
|
||||
## 3. Message Normalization
|
||||
|
||||
### Standard Normalizers (from Aquarius)
|
||||
|
||||
| Pattern | Placeholder |
|
||||
|---------|-------------|
|
||||
| UUIDs | `<ID>` |
|
||||
| ISO timestamps | `<TS>` |
|
||||
| Epoch timestamps | `<EPOCH>` |
|
||||
| Hex/alphanumeric IDs (8+ chars) | `<ID>` |
|
||||
| Email addresses | `<EMAIL>` |
|
||||
| IP addresses | `<IP>` |
|
||||
| URLs | domain only |
|
||||
| Numbers (4+ digits) | `<NUM>` |
|
||||
|
||||
### NK-Specific Normalizers
|
||||
|
||||
| Pattern | Placeholder |
|
||||
|---------|-------------|
|
||||
| Job IDs (`/jobs/<id>`) | `/jobs/<ID>` |
|
||||
| GCS paths (`gs://nomadkaraoke-*/**`) | `gs://<BUCKET>/<PATH>` |
|
||||
| Firebase UIDs | `<UID>` |
|
||||
| Audio file references (hashes, filenames) | `<FILE>` |
|
||||
| Firestore document paths | `<DOC_PATH>` |
|
||||
|
||||
### Pattern Hash
|
||||
|
||||
`SHA-256(service::normalized_message)` — deterministic, collision-resistant.
|
||||
|
||||
---
|
||||
|
||||
## 4. LLM Analysis
|
||||
|
||||
### Model & Provider
|
||||
|
||||
- **Model:** Gemini 2.0 Flash via Vertex AI
|
||||
- **Auth:** Application Default Credentials (service account)
|
||||
- **Temperature:** 0.2 for incident analysis, 0.1 for dedup
|
||||
- **Retry:** Exponential backoff, 3 attempts, 2s base delay
|
||||
- **Fallback:** If LLM unavailable, group by service name (no root cause analysis)
|
||||
|
||||
### Incident Grouping
|
||||
|
||||
When 2+ new patterns are detected, the LLM analyzes them together. The system prompt provides NK-specific service dependency context:
|
||||
|
||||
```
|
||||
karaoke-backend (API) → orchestrates all workers
|
||||
├── audio-download-job → flacfetch-vm (downloads audio)
|
||||
├── audio-separation-job / audio-separator (GPU stem separation)
|
||||
├── lyrics-transcription-job (AudioShake + AI correction)
|
||||
├── video-encoding-job → encoding-worker-a/b (FFmpeg rendering)
|
||||
└── youtube-upload (distribution)
|
||||
|
||||
karaoke-decide (independent) → BigQuery, Spotify, Last.fm
|
||||
|
||||
Cloud Functions: gdrive-validator, backup_to_aws, divebar_mirror,
|
||||
kn_data_sync, divebar_lookup, runner_manager,
|
||||
encoding_worker_idle
|
||||
```
|
||||
|
||||
### Output Format (JSON)
|
||||
|
||||
```json
|
||||
{
|
||||
"incidents": [
|
||||
{
|
||||
"title": "max 60 chars",
|
||||
"root_cause": "one-liner",
|
||||
"severity": "P0|P1|P2|P3",
|
||||
"suggested_fix": "actionable, 1-2 sentences",
|
||||
"primary_service": "upstream root cause service",
|
||||
"pattern_indices": [1, 3]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Severity Levels
|
||||
|
||||
- **P0:** Service completely down, all jobs failing, payment processing broken
|
||||
- **P1:** Major pipeline stage broken (audio separation, lyrics, encoding), auth failures
|
||||
- **P2:** Degraded but functional — intermittent errors, slow responses, single-job failures
|
||||
- **P3:** Minor — deprecation warnings, cleanup failures, spot VM preemptions
|
||||
|
||||
### Duplicate Detection
|
||||
|
||||
Identifies near-duplicate patterns that regex normalization missed (different phrasing of same error). Only merges new patterns into existing canonical ones to preserve alert history.
|
||||
|
||||
---
|
||||
|
||||
## 5. Firestore Data Model
|
||||
|
||||
### `error_patterns` Collection
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `pattern_id` | string | SHA-256(service::normalized_message) — document ID |
|
||||
| `service` | string | Resource name (e.g., `karaoke-backend`, `video-encoding-job`) |
|
||||
| `resource_type` | string | `cloud_run_service` / `cloud_run_job` / `cloud_function` / `gce_instance` |
|
||||
| `normalized_message` | string | Regex-normalized error (max 200 chars) |
|
||||
| `sample_message` | string | Raw error sample |
|
||||
| `first_seen` | string (ISO) | First occurrence |
|
||||
| `last_seen` | string (ISO) | Most recent occurrence |
|
||||
| `total_count` | int | Cumulative count |
|
||||
| `rolling_counts` | map | `{YYYY-MM-DDTHH:MM → count}` (7-day window) |
|
||||
| `status` | string | `new` / `acknowledged` / `known` / `muted` / `fixed` / `auto_resolved` |
|
||||
| `incident_id` | string? | Link to grouped incident |
|
||||
| `severity` | string | `P0` / `P1` / `P2` / `P3` |
|
||||
| `alerted_at` | string (ISO) | When Discord alert was sent |
|
||||
| `notes` | string? | Free-text notes |
|
||||
| `merged_into` | string? | Pattern ID this was merged into |
|
||||
| `fixed_by` | map? | `{pr_display, pr_url, note, resolved_at}` |
|
||||
|
||||
### `error_incidents` Collection
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `incident_id` | string | `inc_YYYYMMDD_HHMM_<8-hex>` — document ID |
|
||||
| `title` | string | Max 60 chars |
|
||||
| `root_cause` | string? | LLM-identified |
|
||||
| `severity` | string | `P0` / `P1` / `P2` / `P3` |
|
||||
| `suggested_fix` | string? | Actionable next step |
|
||||
| `primary_service` | string | Upstream root cause service |
|
||||
| `pattern_ids` | list | Related pattern IDs |
|
||||
| `used_llm` | bool | Whether LLM analysis was used |
|
||||
| `created_at` | string (ISO) | Creation timestamp |
|
||||
| `status` | string | `open` / `investigating` / `fixed` / `false_positive` |
|
||||
|
||||
### `discord_alerts` Collection
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| Document ID | string | `alert_YYYYMMDD_HHMMSS_<8-hex>` |
|
||||
| `timestamp` | string (ISO) | When sent |
|
||||
| `alert_type` | string | `new_pattern` / `spike` / `incident` / `resolved_batch` / `daily_digest` |
|
||||
| `content` | string | Message text (max 2000 chars) |
|
||||
| `success` | bool | Whether webhook POST succeeded |
|
||||
| + alert-specific metadata | varies | service, pattern, count, severity, etc. |
|
||||
|
||||
### Migration from `known-issues.yaml`
|
||||
|
||||
Existing entries in `known-issues.yaml` are migrated to `error_patterns` documents with status `known` or `muted`. The YAML file is then deprecated.
|
||||
|
||||
---
|
||||
|
||||
## 6. Discord Notifications
|
||||
|
||||
### Webhook
|
||||
|
||||
Reuses the existing `discord-alert-webhook` secret in Secret Manager.
|
||||
|
||||
### Alert Types
|
||||
|
||||
| Alert Type | Trigger | Emoji | Content |
|
||||
|------------|---------|-------|---------|
|
||||
| **New Pattern** | Unrecognized error | `🔴` | Service, resource type, normalized msg, sample, count, timestamps, `/prod-investigate` prompt |
|
||||
| **Incident** | 2+ patterns grouped by LLM | `🚨`/`🔴`/`⚠️`/`🟡` (P0-P3) | Title, root cause, per-pattern details, suggested fix |
|
||||
| **Auto-Resolved** | Pattern silent past threshold | `✅` | Pattern, hours silent, consolidated |
|
||||
| **Spike** | Count > 5x rolling avg (min 5) | `⚠️` | Pattern, current vs. normal count |
|
||||
| **Daily Digest** | 08:00 UTC | `📊` | 24h totals, per-service breakdown, frequency stats |
|
||||
|
||||
### Format Rules
|
||||
|
||||
- Compact inline formatting (no bullet lists)
|
||||
- Show actual error message text, never summarize
|
||||
- Include copy-paste `/prod-investigate` prompts
|
||||
- Rate limited: max 10 messages per run
|
||||
- 2000-char limit with truncation
|
||||
- Timestamps in Eastern time (America/New_York, DST-aware)
|
||||
- Honest when LLM fails: "LLM grouping unavailable"
|
||||
|
||||
---
|
||||
|
||||
## 7. Auto-Resolution
|
||||
|
||||
Frequency-aware thresholds based on historical occurrence rate:
|
||||
|
||||
```
|
||||
threshold_hours = mean_interval_between_occurrences * AUTO_RESOLVE_MULTIPLIER
|
||||
```
|
||||
|
||||
| Setting | Value |
|
||||
|---------|-------|
|
||||
| `AUTO_RESOLVE_MULTIPLIER` | 8 (99.97% confidence, Poisson) |
|
||||
| `AUTO_RESOLVE_MIN_HOURS` | 6 |
|
||||
| `AUTO_RESOLVE_MAX_HOURS` | 168 (1 week) |
|
||||
| `AUTO_RESOLVE_FALLBACK_HOURS` | 48 (when <3 data points) |
|
||||
|
||||
High-frequency errors resolve faster; rare errors wait longer.
|
||||
|
||||
---
|
||||
|
||||
## 8. Known Issues / Ignore Patterns
|
||||
|
||||
Regex patterns for infrastructure noise that never triggers alerts:
|
||||
|
||||
- Cloud Run cold start / startup probe failures
|
||||
- Spot VM preemption events (GitHub runners)
|
||||
- GitHub runner startup/shutdown noise
|
||||
- Encoding worker idle shutdown (expected)
|
||||
- Cloud Scheduler retry transients
|
||||
- Health check 404s from load balancers
|
||||
- Ready condition changes during deploys
|
||||
|
||||
Defined in `known_issues.py`, easy to add to without code changes.
|
||||
|
||||
---
|
||||
|
||||
## 9. Infrastructure (Pulumi)
|
||||
|
||||
### New Resources (`infrastructure/modules/error_monitor.py`)
|
||||
|
||||
1. **Cloud Run Job** (`nomad-error-monitor`)
|
||||
- Image: existing karaoke-backend Docker image
|
||||
- Command: `["python", "-m", "backend.services.error_monitor.monitor"]`
|
||||
- Memory: 512Mi, CPU: 1, Timeout: 300s, Max retries: 0
|
||||
- Env vars: `GCP_PROJECT_ID`, `ENVIRONMENT`, `DISCORD_WEBHOOK_URL` (Secret Manager ref)
|
||||
|
||||
2. **Cloud Scheduler — Monitor** (`error-monitor-trigger`)
|
||||
- Schedule: `*/15 * * * *` (UTC)
|
||||
- Target: Cloud Run Jobs API `:run`
|
||||
- Auth: OAuth token via scheduler service account
|
||||
- Retry: 1 attempt, 60s max
|
||||
|
||||
3. **Cloud Scheduler — Daily Digest** (`error-monitor-daily-digest`)
|
||||
- Schedule: `0 8 * * *` (UTC)
|
||||
- Same job, with `DIGEST_MODE=true` environment override
|
||||
|
||||
4. **Service Account** (`error-monitor-scheduler`)
|
||||
- `roles/run.invoker` on the Cloud Run Job
|
||||
|
||||
### IAM for Error Monitor Job
|
||||
|
||||
The job runs under the existing backend service account (`karaoke-backend-sa`), which already has Firestore and Secret Manager access. Additional role needed:
|
||||
- `roles/logging.viewer` (read Cloud Logging — not currently on the backend SA)
|
||||
|
||||
---
|
||||
|
||||
## 10. Slash Command Updates
|
||||
|
||||
All five `/prod-*` commands updated to use `error_patterns` as primary data source.
|
||||
|
||||
### `/prod-health`
|
||||
Add error pattern summary (active count, new patterns in last hour) alongside existing health endpoint checks.
|
||||
|
||||
### `/prod-errors`
|
||||
**Primary source becomes Firestore `error_patterns`** — fast, pre-deduplicated, pre-categorized. Cloud Logging becomes secondary, used only for stack traces when investigating a specific pattern.
|
||||
|
||||
### `/prod-investigate`
|
||||
Query `error_patterns` first for pattern context (history, frequency, related incidents), then Cloud Logging for stack traces and detailed samples.
|
||||
|
||||
### `/prod-review`
|
||||
Replace raw Cloud Logging categorization with pattern statuses: NEW, REGRESSIONS (previously fixed reoccurred), IN PROGRESS (acknowledged), KNOWN, VERIFIED FIXED. Remove `known-issues.yaml` dependency.
|
||||
|
||||
### `/prod-known-issue`
|
||||
Write to Firestore `error_patterns` (status: `known`/`muted`) instead of YAML file.
|
||||
|
||||
### Helper Scripts
|
||||
|
||||
- `scripts/query-error-patterns.py` — CLI to query patterns from Firestore (used by slash commands)
|
||||
- `scripts/resolve-error-pattern.py` — CLI to mark patterns as fixed with PR reference
|
||||
|
||||
---
|
||||
|
||||
## 11. Code Structure
|
||||
|
||||
```
|
||||
backend/services/error_monitor/
|
||||
├── __init__.py
|
||||
├── monitor.py # Entry point — query, normalize, group, analyze, alert
|
||||
├── config.py # Configuration constants, monitored services list
|
||||
├── normalizer.py # Message normalization (regex replacements)
|
||||
├── known_issues.py # Ignore pattern matching
|
||||
├── llm_analysis.py # Vertex AI Gemini incident grouping + dedup
|
||||
├── discord.py # Discord webhook client
|
||||
└── firestore_adapter.py # CRUD for error_patterns, incidents, alerts
|
||||
```
|
||||
|
||||
### Configuration (`config.py`)
|
||||
|
||||
| Setting | Value |
|
||||
|---------|-------|
|
||||
| `GCP_PROJECT` | `nomadkaraoke` |
|
||||
| `LOOKBACK_MINUTES` | 15 |
|
||||
| `MAX_LOG_ENTRIES` | 500 |
|
||||
| `LLM_ANALYSIS_ENABLED` | true (env var) |
|
||||
| `LLM_ANALYSIS_MODEL` | `gemini-2.0-flash` |
|
||||
| `MIN_PATTERNS_FOR_ANALYSIS` | 2 |
|
||||
| `SPIKE_MULTIPLIER` | 5.0 |
|
||||
| `SPIKE_MIN_COUNT` | 5 |
|
||||
| `MAX_DISCORD_MESSAGES_PER_RUN` | 10 |
|
||||
| `MAX_ACTIVE_PATTERNS` | 500 |
|
||||
| `ROLLING_WINDOW_DAYS` | 7 |
|
||||
| `DISCORD_WEBHOOK_SECRET` | `discord-alert-webhook` |
|
||||
|
||||
---
|
||||
|
||||
## 12. Testing
|
||||
|
||||
### Unit Tests
|
||||
|
||||
```
|
||||
tests/unit/services/error_monitor/
|
||||
├── test_monitor.py # Core grouping, windowing logic
|
||||
├── test_normalizer.py # Message normalization
|
||||
├── test_llm_analysis.py # LLM grouping, dedup
|
||||
├── test_firestore_adapter.py # CRUD, rolling counts, resolution
|
||||
├── test_discord.py # Message formatting
|
||||
├── test_known_issues.py # Ignore pattern matching
|
||||
├── test_dedup.py # Duplicate detection
|
||||
└── test_monitor_orchestration.py # Full pipeline integration
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
|
||||
- Test against Firestore emulator for pattern CRUD
|
||||
- Test Cloud Logging query construction (mock the API client)
|
||||
- Test Discord webhook formatting (mock HTTP)
|
||||
|
||||
---
|
||||
|
||||
## 13. Out of Scope (Future Work)
|
||||
|
||||
- **Auto-fixer (Nomad-fix):** Autonomous Claude Code agent that receives errors via Pub/Sub, creates fix PRs. Designed as a separate follow-up project.
|
||||
- **Frontend admin dashboard:** Visual error monitoring UI. The slash commands serve this purpose for now.
|
||||
- **Cross-repo error correlation:** Correlating errors between karaoke-gen and karaoke-decide Firestore. Both write to the same `error_patterns` collection, so patterns from both are visible together.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -42,6 +42,7 @@ from modules import divebar_mirror, kn_data_sync, divebar_lookup, backup
|
||||
from modules import audio_separator_service
|
||||
from modules import gpu_artifact_registry
|
||||
from modules import encoding_worker_manager
|
||||
from modules import error_monitor as error_monitor_module
|
||||
from modules.iam import backend_sa, github_actions_sa, claude_automation_sa, claude_readonly_sa, worker_sas
|
||||
from compute import encoding_worker_vm, github_runners, divebar_sync_vm
|
||||
|
||||
@@ -181,6 +182,14 @@ lyrics_transcription_job = cloud_run.create_lyrics_transcription_job(bucket, bac
|
||||
audio_separation_job = cloud_run.create_audio_separation_job(bucket, backend_service_account)
|
||||
audio_download_job = cloud_run.create_audio_download_job(bucket, backend_service_account, vpc_connector)
|
||||
|
||||
# ==================== Error Monitor ====================
|
||||
|
||||
# Cloud Run Job + Cloud Scheduler for automated error detection and digest reports
|
||||
error_monitor_resources = error_monitor_module.create_error_monitor(
|
||||
backend_image=f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest",
|
||||
backend_sa_email=backend_service_account.email,
|
||||
)
|
||||
|
||||
# ==================== Monitoring ====================
|
||||
|
||||
# Alert policies
|
||||
@@ -668,3 +677,9 @@ pulumi.export("backup_function_url", backup_resources["function"].url)
|
||||
pulumi.export("backup_staging_bucket", backup_resources["staging_bucket"].name)
|
||||
pulumi.export("backup_scheduler_name", backup_resources["scheduler"].name)
|
||||
pulumi.export("backup_service_account", backup_resources["service_account"].email)
|
||||
|
||||
# Error Monitor
|
||||
pulumi.export("error_monitor_job_name", error_monitor_resources["job"].name)
|
||||
pulumi.export("error_monitor_scheduler_sa", error_monitor_resources["scheduler_sa"].email)
|
||||
pulumi.export("error_monitor_scheduler_name", error_monitor_resources["monitor_scheduler"].name)
|
||||
pulumi.export("error_monitor_digest_scheduler_name", error_monitor_resources["digest_scheduler"].name)
|
||||
@@ -133,3 +133,15 @@ class EncodingWorkerConfig:
|
||||
FUNCTION_NAME = "encoding-worker-idle-shutdown"
|
||||
FUNCTION_MEMORY = "256M"
|
||||
FUNCTION_TIMEOUT = 120 # 2 minutes
|
||||
|
||||
|
||||
class ErrorMonitorConfig:
|
||||
"""Configuration for the error monitor Cloud Run Job."""
|
||||
JOB_NAME = "nomad-error-monitor"
|
||||
MEMORY = "512Mi"
|
||||
CPU = "1"
|
||||
TIMEOUT = "300s"
|
||||
MAX_RETRIES = 0
|
||||
MONITOR_SCHEDULE = "*/15 * * * *" # Every 15 minutes UTC
|
||||
DIGEST_SCHEDULE = "0 8 * * *" # 08:00 UTC daily
|
||||
SCHEDULER_SA_NAME = "error-monitor-scheduler"
|
||||
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
Error Monitor Cloud Run Job and Cloud Scheduler resources.
|
||||
|
||||
Deploys the error monitor as a Cloud Run Job triggered by Cloud Scheduler
|
||||
every 15 minutes (error checks) and daily at 08:00 UTC (digest report).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
||||
import pulumi
|
||||
import pulumi_gcp as gcp
|
||||
from pulumi_gcp import cloudrunv2, cloudscheduler, serviceaccount
|
||||
|
||||
from config import PROJECT_ID, REGION, ErrorMonitorConfig
|
||||
|
||||
# Empty body for the monitor trigger (Cloud Run Jobs API requires POST body)
|
||||
_EMPTY_BODY_B64 = base64.b64encode(b"{}").decode()
|
||||
|
||||
# Digest trigger body — sets DIGEST_MODE=true via Cloud Run Job override
|
||||
_DIGEST_BODY_B64 = base64.b64encode(
|
||||
json.dumps({
|
||||
"overrides": {
|
||||
"containerOverrides": [
|
||||
{
|
||||
"env": [
|
||||
{"name": "DIGEST_MODE", "value": "true"},
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}).encode()
|
||||
).decode()
|
||||
|
||||
|
||||
def create_error_monitor(
|
||||
backend_image: pulumi.Input[str],
|
||||
backend_sa_email: pulumi.Input[str],
|
||||
depends_on: list = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Create the error monitor Cloud Run Job and Cloud Scheduler triggers.
|
||||
|
||||
The error monitor runs as the backend SA (same permissions: Firestore,
|
||||
Secrets, Logging). A dedicated scheduler SA is created for Cloud Scheduler
|
||||
OAuth authentication.
|
||||
|
||||
Args:
|
||||
backend_image: Docker image URL for the backend (Artifact Registry).
|
||||
backend_sa_email: Email of the backend service account (runs the job).
|
||||
depends_on: Optional list of Pulumi resources to depend on.
|
||||
|
||||
Returns:
|
||||
dict with keys: 'job', 'scheduler_sa', 'monitor_scheduler', 'digest_scheduler'.
|
||||
"""
|
||||
job_opts = pulumi.ResourceOptions(depends_on=depends_on) if depends_on else None
|
||||
|
||||
# ---- Service Account for Cloud Scheduler ----
|
||||
# Scheduler SA authenticates when invoking the Cloud Run Job via OAuth token.
|
||||
scheduler_sa = serviceaccount.Account(
|
||||
"error-monitor-scheduler-sa",
|
||||
account_id=ErrorMonitorConfig.SCHEDULER_SA_NAME,
|
||||
display_name="Error Monitor Scheduler Service Account",
|
||||
)
|
||||
|
||||
# ---- Cloud Run Job ----
|
||||
error_monitor_job = cloudrunv2.Job(
|
||||
"error-monitor-job",
|
||||
name=ErrorMonitorConfig.JOB_NAME,
|
||||
location=REGION,
|
||||
template=cloudrunv2.JobTemplateArgs(
|
||||
template=cloudrunv2.JobTemplateTemplateArgs(
|
||||
containers=[
|
||||
cloudrunv2.JobTemplateTemplateContainerArgs(
|
||||
image=backend_image,
|
||||
args=["python", "-m", "backend.services.error_monitor.monitor"],
|
||||
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
|
||||
limits={
|
||||
"cpu": ErrorMonitorConfig.CPU,
|
||||
"memory": ErrorMonitorConfig.MEMORY,
|
||||
},
|
||||
),
|
||||
envs=[
|
||||
# GCP project for Firestore, Logging, etc.
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="GCP_PROJECT_ID",
|
||||
value=PROJECT_ID,
|
||||
),
|
||||
# Environment tag for log filtering
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="ENVIRONMENT",
|
||||
value="production",
|
||||
),
|
||||
# Discord webhook for alert notifications
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="DISCORD_WEBHOOK_URL",
|
||||
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
||||
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
||||
secret=f"projects/{PROJECT_ID}/secrets/discord-alert-webhook",
|
||||
version="latest",
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
],
|
||||
service_account=backend_sa_email,
|
||||
timeout=ErrorMonitorConfig.TIMEOUT,
|
||||
max_retries=ErrorMonitorConfig.MAX_RETRIES,
|
||||
),
|
||||
),
|
||||
opts=job_opts,
|
||||
)
|
||||
|
||||
# ---- IAM: grant scheduler SA permission to invoke (run) the job ----
|
||||
error_monitor_job_invoker = cloudrunv2.JobIamMember(
|
||||
"error-monitor-scheduler-invoker",
|
||||
project=PROJECT_ID,
|
||||
location=REGION,
|
||||
name=error_monitor_job.name,
|
||||
role="roles/run.invoker",
|
||||
member=scheduler_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Cloud Run Jobs API endpoint — Cloud Scheduler calls this to trigger a run.
|
||||
# Format: https://run.googleapis.com/v2/projects/{project}/locations/{region}/jobs/{job}:run
|
||||
job_run_uri = error_monitor_job.name.apply(
|
||||
lambda name: (
|
||||
f"https://run.googleapis.com/v2/projects/{PROJECT_ID}"
|
||||
f"/locations/{REGION}/jobs/{name}:run"
|
||||
)
|
||||
)
|
||||
|
||||
# Shared opts: schedulers depend on IAM binding being in place first
|
||||
scheduler_opts = pulumi.ResourceOptions(depends_on=[error_monitor_job_invoker])
|
||||
|
||||
# ---- Cloud Scheduler: every-15-min monitor ----
|
||||
monitor_scheduler = cloudscheduler.Job(
|
||||
"error-monitor-scheduler",
|
||||
name="error-monitor-trigger",
|
||||
description="Trigger error monitor Cloud Run Job every 15 minutes",
|
||||
region=REGION,
|
||||
schedule=ErrorMonitorConfig.MONITOR_SCHEDULE,
|
||||
time_zone="UTC",
|
||||
http_target=cloudscheduler.JobHttpTargetArgs(
|
||||
uri=job_run_uri,
|
||||
http_method="POST",
|
||||
body=_EMPTY_BODY_B64,
|
||||
oauth_token=cloudscheduler.JobHttpTargetOauthTokenArgs(
|
||||
service_account_email=scheduler_sa.email,
|
||||
),
|
||||
),
|
||||
retry_config=cloudscheduler.JobRetryConfigArgs(
|
||||
retry_count=0,
|
||||
),
|
||||
opts=scheduler_opts,
|
||||
)
|
||||
|
||||
# ---- Cloud Scheduler: daily digest at 08:00 UTC ----
|
||||
# Passes DIGEST_MODE=true in the request body as a Cloud Run Job override.
|
||||
digest_scheduler = cloudscheduler.Job(
|
||||
"error-monitor-digest-scheduler",
|
||||
name="error-monitor-daily-digest",
|
||||
description="Trigger error monitor Cloud Run Job daily at 08:00 UTC for digest report",
|
||||
region=REGION,
|
||||
schedule=ErrorMonitorConfig.DIGEST_SCHEDULE,
|
||||
time_zone="UTC",
|
||||
http_target=cloudscheduler.JobHttpTargetArgs(
|
||||
uri=job_run_uri,
|
||||
http_method="POST",
|
||||
body=_DIGEST_BODY_B64,
|
||||
oauth_token=cloudscheduler.JobHttpTargetOauthTokenArgs(
|
||||
service_account_email=scheduler_sa.email,
|
||||
),
|
||||
),
|
||||
retry_config=cloudscheduler.JobRetryConfigArgs(
|
||||
retry_count=0,
|
||||
),
|
||||
opts=scheduler_opts,
|
||||
)
|
||||
|
||||
return {
|
||||
"job": error_monitor_job,
|
||||
"scheduler_sa": scheduler_sa,
|
||||
"monitor_scheduler": monitor_scheduler,
|
||||
"digest_scheduler": digest_scheduler,
|
||||
}
|
||||
@@ -120,6 +120,14 @@ def grant_backend_permissions(
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Cloud Logging Viewer - error monitor reads Cloud Logging for error detection
|
||||
bindings["logging_viewer"] = gcp.projects.IAMMember(
|
||||
"karaoke-backend-logging-viewer",
|
||||
project=PROJECT_ID,
|
||||
role="roles/logging.viewer",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
return bindings
|
||||
|
||||
|
||||
|
||||
Generated
+66
-1
@@ -2546,6 +2546,44 @@ requests-oauthlib = ">=0.7.0"
|
||||
[package.extras]
|
||||
tool = ["click (>=6.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-appengine-logging"
|
||||
version = "1.9.0"
|
||||
description = "Google Cloud Appengine Logging API client library"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "google_cloud_appengine_logging-1.9.0-py3-none-any.whl", hash = "sha256:bbf3a7e4dc171678f7f481259d1f68c3ae7d337530f1f2361f8a0b214dbcfe36"},
|
||||
{file = "google_cloud_appengine_logging-1.9.0.tar.gz", hash = "sha256:ff397f0bbc1485f979ab45767c38e0f676c9598c97c384f7412216e6ea22f805"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]}
|
||||
google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0"
|
||||
grpcio = ">=1.33.2,<2.0.0"
|
||||
proto-plus = [
|
||||
{version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""},
|
||||
{version = ">=1.22.3,<2.0.0"},
|
||||
]
|
||||
protobuf = ">=4.25.8,<8.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-audit-log"
|
||||
version = "0.5.0"
|
||||
description = "Google Cloud Audit Protos"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "google_cloud_audit_log-0.5.0-py3-none-any.whl", hash = "sha256:3f4632f25bf67446fa9085c52868f3cb42fb1afbab9489ba8978e30991afc79f"},
|
||||
{file = "google_cloud_audit_log-0.5.0.tar.gz", hash = "sha256:3b32d5e77db634c46fbd6c5e01f5bda836f420dfbb21d730501c75e9fab4e4a4"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
googleapis-common-protos = ">=1.56.2,<2.0.0"
|
||||
protobuf = ">=4.25.8,<8.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-compute"
|
||||
version = "1.46.0"
|
||||
@@ -2610,6 +2648,33 @@ proto-plus = [
|
||||
]
|
||||
protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-logging"
|
||||
version = "3.15.0"
|
||||
description = "Google Cloud Logging API client library"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "google_cloud_logging-3.15.0-py3-none-any.whl", hash = "sha256:7dcc67434c4e7181510c133d5ac8fd4ce60c23fa4158661f67e54bf440c32450"},
|
||||
{file = "google_cloud_logging-3.15.0.tar.gz", hash = "sha256:72168a1e98bbfc27c75f0b8f630a7f5d786065f3f1f7e9e53d2d787a03693a4a"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]}
|
||||
google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0"
|
||||
google-cloud-appengine-logging = ">=0.1.3,<2.0.0"
|
||||
google-cloud-audit-log = ">=0.3.1,<1.0.0"
|
||||
google-cloud-core = ">=2.0.0,<3.0.0"
|
||||
grpc-google-iam-v1 = ">=0.12.4,<1.0.0"
|
||||
grpcio = ">=1.33.2,<2.0.0"
|
||||
opentelemetry-api = ">=1.9.0"
|
||||
proto-plus = [
|
||||
{version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""},
|
||||
{version = ">=1.22.3,<2.0.0"},
|
||||
]
|
||||
protobuf = ">=4.25.8,<8.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-monitoring"
|
||||
version = "2.29.1"
|
||||
@@ -10702,4 +10767,4 @@ local-whisper = ["whisper-timestamped"]
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.10,<3.14"
|
||||
content-hash = "6ed6be0dbeef6f31529246a14d2ad4286edc5ab2fe455433a886429a78f8054b"
|
||||
content-hash = "7b0bb0bf110afca09d3d7e7c7b829777091be76916b4a29add8ecbcf50372eef"
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.168.2"
|
||||
version = "0.169.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
@@ -94,6 +94,7 @@ google-cloud-secret-manager = "2.26.0"
|
||||
google-cloud-monitoring = "2.29.1" # Cloud Monitoring for YouTube API quota metrics
|
||||
google-cloud-tasks = "2.21.0" # Cloud Tasks for worker coordination (Phase 1 scalability)
|
||||
google-cloud-run = "0.15.0" # Cloud Run Jobs for long-running video encoding (Phase 2 scalability)
|
||||
google-cloud-logging = ">=3.9.0" # Cloud Logging for error monitoring
|
||||
python-dotenv = ">=1.0.0"
|
||||
pydub = ">=0.25.1"
|
||||
# Payment and email services
|
||||
|
||||
Executable
+193
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Query error patterns from Firestore.
|
||||
|
||||
Usage:
|
||||
python scripts/query-error-patterns.py # All active
|
||||
python scripts/query-error-patterns.py --status new # Only new
|
||||
python scripts/query-error-patterns.py --service karaoke-backend
|
||||
python scripts/query-error-patterns.py --hours 24 # Last 24h
|
||||
python scripts/query-error-patterns.py --json # JSON output
|
||||
|
||||
Requires:
|
||||
- google-cloud-firestore
|
||||
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
from google.cloud import firestore
|
||||
from google.cloud.firestore_v1 import FieldFilter
|
||||
|
||||
|
||||
ACTIVE_STATUSES = ["new", "acknowledged", "known"]
|
||||
|
||||
|
||||
def query_error_patterns(
|
||||
status: Optional[str] = None,
|
||||
service: Optional[str] = None,
|
||||
hours: Optional[int] = None,
|
||||
output_json: bool = False,
|
||||
project: str = "nomadkaraoke",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Query error patterns from Firestore.
|
||||
|
||||
Args:
|
||||
status: Filter by status (new/acknowledged/known/fixed). Default: active (new/acknowledged/known).
|
||||
service: Filter by service name.
|
||||
hours: Only return patterns last seen within this many hours.
|
||||
output_json: Print JSON output instead of a table.
|
||||
project: GCP project ID.
|
||||
|
||||
Returns:
|
||||
List of pattern dicts.
|
||||
"""
|
||||
db = firestore.Client(project=project)
|
||||
query = db.collection("error_patterns")
|
||||
|
||||
# Status filter
|
||||
if status:
|
||||
query = query.where(filter=FieldFilter("status", "==", status))
|
||||
else:
|
||||
query = query.where(filter=FieldFilter("status", "in", ACTIVE_STATUSES))
|
||||
|
||||
# Service filter
|
||||
if service:
|
||||
query = query.where(filter=FieldFilter("service", "==", service))
|
||||
|
||||
# Fetch all matching docs (streaming)
|
||||
patterns = []
|
||||
for doc in query.stream():
|
||||
data = doc.to_dict()
|
||||
data["_id"] = doc.id
|
||||
patterns.append(data)
|
||||
|
||||
# Hours filter (applied in Python — Firestore requires composite index for multi-field filters)
|
||||
if hours is not None:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
filtered = []
|
||||
for p in patterns:
|
||||
last_seen = p.get("last_seen")
|
||||
if last_seen is not None:
|
||||
# Firestore Timestamps have a .timestamp() method
|
||||
ts = last_seen.timestamp() if hasattr(last_seen, "timestamp") else float(last_seen)
|
||||
if ts >= cutoff.timestamp():
|
||||
filtered.append(p)
|
||||
patterns = filtered
|
||||
|
||||
# Sort by last_seen descending (most recent first)
|
||||
def sort_key(p):
|
||||
ls = p.get("last_seen")
|
||||
if ls is None:
|
||||
return 0.0
|
||||
return ls.timestamp() if hasattr(ls, "timestamp") else float(ls)
|
||||
|
||||
patterns.sort(key=sort_key, reverse=True)
|
||||
|
||||
if output_json:
|
||||
print(json.dumps(patterns, indent=2, default=str))
|
||||
else:
|
||||
_print_table(patterns)
|
||||
|
||||
return patterns
|
||||
|
||||
|
||||
def _fmt_last_seen(value) -> str:
|
||||
"""Return first 19 chars of ISO timestamp, or empty string."""
|
||||
if value is None:
|
||||
return ""
|
||||
if hasattr(value, "isoformat"):
|
||||
return value.isoformat()[:19]
|
||||
return str(value)[:19]
|
||||
|
||||
|
||||
def _print_table(patterns: list[dict[str, Any]]) -> None:
|
||||
"""Print patterns as a human-readable table."""
|
||||
if not patterns:
|
||||
print("No error patterns found.")
|
||||
return
|
||||
|
||||
# Column widths
|
||||
col_status = 14
|
||||
col_service = 24
|
||||
col_count = 6
|
||||
col_last_seen = 19
|
||||
col_message = 50
|
||||
|
||||
header = (
|
||||
f"{'Status':<{col_status}} "
|
||||
f"{'Service':<{col_service}} "
|
||||
f"{'Count':>{col_count}} "
|
||||
f"{'Last Seen':<{col_last_seen}} "
|
||||
f"{'Message':<{col_message}}"
|
||||
)
|
||||
separator = "-" * len(header)
|
||||
|
||||
print(separator)
|
||||
print(header)
|
||||
print(separator)
|
||||
|
||||
for p in patterns:
|
||||
status = str(p.get("status", ""))[:col_status]
|
||||
service = str(p.get("service", ""))[:col_service]
|
||||
count = p.get("total_count", p.get("occurrence_count", p.get("count", 0)))
|
||||
last_seen = _fmt_last_seen(p.get("last_seen"))
|
||||
message = str(p.get("normalized_message", p.get("error_message", p.get("message", ""))))[:col_message]
|
||||
|
||||
print(
|
||||
f"{status:<{col_status}} "
|
||||
f"{service:<{col_service}} "
|
||||
f"{count:>{col_count}} "
|
||||
f"{last_seen:<{col_last_seen}} "
|
||||
f"{message:<{col_message}}"
|
||||
)
|
||||
|
||||
print(separator)
|
||||
print(f"Total: {len(patterns)} pattern(s)")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Query error patterns from Firestore"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--status",
|
||||
help="Filter by status (new/acknowledged/known/fixed). Default: active patterns.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--service",
|
||||
help="Filter by service name (e.g. karaoke-backend).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hours",
|
||||
type=int,
|
||||
help="Only show patterns last seen within this many hours.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
dest="output_json",
|
||||
help="Output as JSON.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
default="nomadkaraoke",
|
||||
help="GCP project ID (default: nomadkaraoke).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
query_error_patterns(
|
||||
status=args.status,
|
||||
service=args.service,
|
||||
hours=args.hours,
|
||||
output_json=args.output_json,
|
||||
project=args.project,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+159
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Resolve an error pattern by marking it as fixed.
|
||||
|
||||
Usage:
|
||||
python scripts/resolve-error-pattern.py <pattern_prefix> --pr 42 --note "Fixed timeout"
|
||||
python scripts/resolve-error-pattern.py <pattern_prefix> --note "Added retry logic"
|
||||
|
||||
Requires:
|
||||
- google-cloud-firestore
|
||||
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from google.cloud import firestore
|
||||
|
||||
|
||||
def resolve_error_pattern(
|
||||
pattern_prefix: str,
|
||||
note: str,
|
||||
pr: Optional[int] = None,
|
||||
repo: str = "nomadkaraoke/karaoke-gen",
|
||||
project: str = "nomadkaraoke",
|
||||
) -> bool:
|
||||
"""
|
||||
Find an error pattern by ID prefix and mark it as fixed.
|
||||
|
||||
Args:
|
||||
pattern_prefix: Prefix of the pattern document ID.
|
||||
note: Resolution note explaining how it was fixed.
|
||||
pr: Optional PR number that fixed the issue.
|
||||
repo: GitHub repo slug used to build the PR URL.
|
||||
project: GCP project ID.
|
||||
|
||||
Returns:
|
||||
True if the pattern was resolved, False otherwise.
|
||||
"""
|
||||
try:
|
||||
db = firestore.Client(project=project)
|
||||
collection = db.collection("error_patterns")
|
||||
except Exception as exc:
|
||||
print(f"Error: Could not connect to Firestore (project={project}): {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
# Stream all docs and find those whose ID starts with the prefix
|
||||
matches = []
|
||||
try:
|
||||
for doc in collection.stream():
|
||||
if doc.id.startswith(pattern_prefix):
|
||||
matches.append(doc)
|
||||
except Exception as exc:
|
||||
print(f"Error: Could not query error_patterns: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if len(matches) == 0:
|
||||
print(f"Error: No pattern found with ID prefix '{pattern_prefix}'", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if len(matches) > 1:
|
||||
print(
|
||||
f"Ambiguous prefix '{pattern_prefix}' — matches {len(matches)} pattern(s). "
|
||||
"Please provide a longer prefix.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("\nFirst 5 matches:", file=sys.stderr)
|
||||
for doc in matches[:5]:
|
||||
data = doc.to_dict()
|
||||
service = data.get("service", "")
|
||||
message = str(data.get("error_message", data.get("message", "")))[:60]
|
||||
print(f" {doc.id} [{service}] {message}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
doc = matches[0]
|
||||
data = doc.to_dict()
|
||||
resolved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
fixed_by: dict = {
|
||||
"resolved_at": resolved_at,
|
||||
"note": note,
|
||||
}
|
||||
|
||||
if pr is not None:
|
||||
pr_url = f"https://github.com/{repo}/pull/{pr}"
|
||||
fixed_by["pr_url"] = pr_url
|
||||
fixed_by["pr_display"] = f"#{pr}"
|
||||
|
||||
update_payload = {
|
||||
"status": "fixed",
|
||||
"fixed_by": fixed_by,
|
||||
}
|
||||
|
||||
try:
|
||||
doc.reference.update(update_payload)
|
||||
except Exception as exc:
|
||||
print(f"Error: Could not update pattern {doc.id}: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
# Confirmation output
|
||||
service = data.get("service", "")
|
||||
message = str(data.get("error_message", data.get("message", "")))[:80]
|
||||
print(f"Resolved pattern: {doc.id}")
|
||||
print(f" Service: {service}")
|
||||
print(f" Message: {message}")
|
||||
print(f" Status: fixed")
|
||||
print(f" Note: {note}")
|
||||
if pr is not None:
|
||||
print(f" PR: {fixed_by['pr_url']}")
|
||||
print(f" Resolved: {resolved_at}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Resolve an error pattern by marking it as fixed"
|
||||
)
|
||||
parser.add_argument(
|
||||
"pattern_prefix",
|
||||
help="Prefix of the error pattern document ID.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--note",
|
||||
required=True,
|
||||
help="Resolution note (e.g. 'Fixed timeout in encoding worker').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pr",
|
||||
type=int,
|
||||
help="PR number that fixed the issue (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo",
|
||||
default="nomadkaraoke/karaoke-gen",
|
||||
help="GitHub repo slug for PR URL (default: nomadkaraoke/karaoke-gen).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
default="nomadkaraoke",
|
||||
help="GCP project ID (default: nomadkaraoke).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
success = resolve_error_pattern(
|
||||
pattern_prefix=args.pattern_prefix,
|
||||
note=args.note,
|
||||
pr=args.pr,
|
||||
repo=args.repo,
|
||||
project=args.project,
|
||||
)
|
||||
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Whitespace-only changes.
@@ -0,0 +1,269 @@
|
||||
"""Tests for error_monitor.config module."""
|
||||
|
||||
import os
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestMonitoredServiceLists:
|
||||
"""Tests that all monitored service lists have the expected entries."""
|
||||
|
||||
def test_cloud_run_services_count(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert len(config.MONITORED_CLOUD_RUN_SERVICES) == 3
|
||||
|
||||
def test_cloud_run_services_contains_expected(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert "karaoke-backend" in config.MONITORED_CLOUD_RUN_SERVICES
|
||||
assert "karaoke-decide" in config.MONITORED_CLOUD_RUN_SERVICES
|
||||
assert "audio-separator" in config.MONITORED_CLOUD_RUN_SERVICES
|
||||
|
||||
def test_cloud_run_jobs_count(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert len(config.MONITORED_CLOUD_RUN_JOBS) == 4
|
||||
|
||||
def test_cloud_run_jobs_contains_expected(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert "video-encoding-job" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
assert "lyrics-transcription-job" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
assert "audio-separation-job" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
assert "audio-download-job" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
|
||||
def test_cloud_functions_count_at_least_7(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert len(config.MONITORED_CLOUD_FUNCTIONS) >= 7
|
||||
|
||||
def test_cloud_functions_contains_expected(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
expected = [
|
||||
"gdrive-validator",
|
||||
"runner_manager",
|
||||
"backup_to_aws",
|
||||
"divebar_mirror",
|
||||
"kn_data_sync",
|
||||
"divebar_lookup",
|
||||
"encoding_worker_idle",
|
||||
]
|
||||
for fn in expected:
|
||||
assert fn in config.MONITORED_CLOUD_FUNCTIONS, f"Expected '{fn}' in MONITORED_CLOUD_FUNCTIONS"
|
||||
|
||||
def test_gce_instances_count_at_least_4(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert len(config.MONITORED_GCE_INSTANCES) >= 4
|
||||
|
||||
def test_gce_instances_contains_expected(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
expected = [
|
||||
"encoding-worker-a",
|
||||
"encoding-worker-b",
|
||||
"flacfetch-vm",
|
||||
"divebar-sync-vm",
|
||||
]
|
||||
for instance in expected:
|
||||
assert instance in config.MONITORED_GCE_INSTANCES, f"Expected '{instance}' in MONITORED_GCE_INSTANCES"
|
||||
|
||||
|
||||
class TestDefaultValues:
|
||||
"""Tests that config module exposes correct default constants."""
|
||||
|
||||
def test_gcp_project_default(self):
|
||||
with mock.patch.dict(os.environ, {}, clear=False):
|
||||
# Remove GCP_PROJECT if set, then re-import via reload
|
||||
env = {k: v for k, v in os.environ.items() if k != "GCP_PROJECT"}
|
||||
with mock.patch.dict(os.environ, env, clear=True):
|
||||
import importlib
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.GCP_PROJECT == "nomadkaraoke"
|
||||
|
||||
def test_lookback_minutes_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.LOOKBACK_MINUTES == 15
|
||||
|
||||
def test_max_log_entries_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.MAX_LOG_ENTRIES == 500
|
||||
|
||||
def test_spike_multiplier_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.SPIKE_MULTIPLIER == 5.0
|
||||
|
||||
def test_spike_min_count_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.SPIKE_MIN_COUNT == 5
|
||||
|
||||
def test_auto_resolve_multiplier_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.AUTO_RESOLVE_MULTIPLIER == 8
|
||||
|
||||
def test_max_discord_messages_per_run_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.MAX_DISCORD_MESSAGES_PER_RUN == 10
|
||||
|
||||
def test_discord_max_message_length_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.DISCORD_MAX_MESSAGE_LENGTH == 2000
|
||||
|
||||
def test_max_active_patterns_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.MAX_ACTIVE_PATTERNS == 500
|
||||
|
||||
def test_rolling_window_days_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.ROLLING_WINDOW_DAYS == 7
|
||||
|
||||
def test_max_normalized_message_length_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.MAX_NORMALIZED_MESSAGE_LENGTH == 200
|
||||
|
||||
def test_min_patterns_for_analysis_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert isinstance(config.MIN_PATTERNS_FOR_ANALYSIS, int)
|
||||
assert config.MIN_PATTERNS_FOR_ANALYSIS >= 1
|
||||
|
||||
def test_auto_resolve_min_hours(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.AUTO_RESOLVE_MIN_HOURS == 6
|
||||
|
||||
def test_auto_resolve_max_hours(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.AUTO_RESOLVE_MAX_HOURS == 168 # 1 week
|
||||
|
||||
def test_auto_resolve_fallback_hours(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert config.AUTO_RESOLVE_FALLBACK_HOURS == 48
|
||||
|
||||
def test_service_dependency_map_is_string(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert isinstance(config.SERVICE_DEPENDENCY_MAP, str)
|
||||
assert len(config.SERVICE_DEPENDENCY_MAP) > 0
|
||||
assert "karaoke-backend" in config.SERVICE_DEPENDENCY_MAP
|
||||
|
||||
def test_llm_analysis_model_is_string(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert isinstance(config.LLM_ANALYSIS_MODEL, str)
|
||||
assert len(config.LLM_ANALYSIS_MODEL) > 0
|
||||
|
||||
def test_llm_vertex_location_is_string(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert isinstance(config.LLM_VERTEX_LOCATION, str)
|
||||
assert len(config.LLM_VERTEX_LOCATION) > 0
|
||||
|
||||
|
||||
class TestEnvVarOverrides:
|
||||
"""Tests that environment variable overrides work correctly."""
|
||||
|
||||
def test_llm_analysis_enabled_default_is_false(self):
|
||||
"""LLM_ANALYSIS_ENABLED should default to False when env var is not set."""
|
||||
import importlib
|
||||
env = {k: v for k, v in os.environ.items() if k != "LLM_ANALYSIS_ENABLED"}
|
||||
with mock.patch.dict(os.environ, env, clear=True):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_llm_enabled() is False
|
||||
|
||||
def test_llm_analysis_enabled_true_when_env_var_set(self):
|
||||
"""get_llm_enabled() should return True when LLM_ANALYSIS_ENABLED=true."""
|
||||
import importlib
|
||||
with mock.patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "true"}):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_llm_enabled() is True
|
||||
|
||||
def test_llm_analysis_enabled_true_when_env_var_set_uppercase(self):
|
||||
"""get_llm_enabled() should handle case-insensitive 'TRUE'."""
|
||||
import importlib
|
||||
with mock.patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "TRUE"}):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_llm_enabled() is True
|
||||
|
||||
def test_llm_analysis_enabled_false_when_env_var_set_false(self):
|
||||
"""get_llm_enabled() should return False when LLM_ANALYSIS_ENABLED=false."""
|
||||
import importlib
|
||||
with mock.patch.dict(os.environ, {"LLM_ANALYSIS_ENABLED": "false"}):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_llm_enabled() is False
|
||||
|
||||
def test_gcp_project_overridable_via_env(self):
|
||||
"""GCP_PROJECT should pick up the GCP_PROJECT environment variable."""
|
||||
import importlib
|
||||
with mock.patch.dict(os.environ, {"GCP_PROJECT": "my-test-project"}):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.GCP_PROJECT == "my-test-project"
|
||||
|
||||
def test_gcp_region_has_default(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert isinstance(config.GCP_REGION, str)
|
||||
assert len(config.GCP_REGION) > 0
|
||||
|
||||
def test_gcp_region_overridable_via_env(self):
|
||||
"""GCP_REGION should pick up the GCP_REGION environment variable."""
|
||||
import importlib
|
||||
with mock.patch.dict(os.environ, {"GCP_REGION": "europe-west1"}):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.GCP_REGION == "europe-west1"
|
||||
|
||||
|
||||
class TestHelperFunctions:
|
||||
"""Tests for config helper functions."""
|
||||
|
||||
def test_get_discord_webhook_secret_name_returns_string(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
result = config.get_discord_webhook_secret_name()
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
def test_get_digest_mode_returns_bool(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
result = config.get_digest_mode()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
def test_get_digest_mode_default_false(self):
|
||||
"""DIGEST_MODE should default to False."""
|
||||
import importlib
|
||||
env = {k: v for k, v in os.environ.items() if k != "DIGEST_MODE"}
|
||||
with mock.patch.dict(os.environ, env, clear=True):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_digest_mode() is False
|
||||
|
||||
def test_get_digest_mode_true_when_env_set(self):
|
||||
"""get_digest_mode() should return True when DIGEST_MODE=true."""
|
||||
import importlib
|
||||
with mock.patch.dict(os.environ, {"DIGEST_MODE": "true"}):
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_digest_mode() is True
|
||||
@@ -0,0 +1,619 @@
|
||||
"""Tests for error_monitor.discord module."""
|
||||
|
||||
from datetime import datetime
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
|
||||
class TestFormatNewPatternAlert:
|
||||
"""Tests for format_new_pattern_alert formatting function."""
|
||||
|
||||
def test_contains_service_name(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Connection refused",
|
||||
sample_message="Connection refused to db at 192.168.1.1",
|
||||
count=5,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "karaoke-backend" in result
|
||||
|
||||
def test_contains_normalized_message(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Connection refused",
|
||||
sample_message="Connection refused to db at 192.168.1.1",
|
||||
count=5,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "Connection refused" in result
|
||||
|
||||
def test_contains_red_circle_emoji(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Connection refused",
|
||||
sample_message="Connection refused to db",
|
||||
count=3,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "🔴" in result
|
||||
|
||||
def test_contains_prod_investigate_prompt(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Connection refused",
|
||||
sample_message="Connection refused to db",
|
||||
count=3,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "/prod-investigate" in result
|
||||
|
||||
def test_truncates_long_message(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
from backend.services.error_monitor.config import DISCORD_MAX_MESSAGE_LENGTH
|
||||
|
||||
very_long_message = "x" * 3000
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message=very_long_message,
|
||||
sample_message="sample",
|
||||
count=1,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert len(result) <= DISCORD_MAX_MESSAGE_LENGTH
|
||||
|
||||
def test_contains_count(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Timeout error",
|
||||
sample_message="Timeout after 30s",
|
||||
count=42,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "42" in result
|
||||
|
||||
def test_contains_first_seen(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Timeout error",
|
||||
sample_message="Timeout after 30s",
|
||||
count=1,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "2026-04-10T12:00:00Z" in result
|
||||
|
||||
def test_contains_sample_message(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="karaoke-backend",
|
||||
resource_type="cloud_run_service",
|
||||
normalized_message="Timeout error",
|
||||
sample_message="Timeout after 30s on job xyz",
|
||||
count=1,
|
||||
first_seen="2026-04-10T12:00:00Z",
|
||||
)
|
||||
assert "Timeout after 30s on job xyz" in result
|
||||
|
||||
def test_contains_resource_type_label(self):
|
||||
from backend.services.error_monitor.discord import format_new_pattern_alert
|
||||
|
||||
result = format_new_pattern_alert(
|
||||
service="encoding-worker-a",
|
||||
resource_type="gce_instance",
|
||||
normalized_message="Disk full",
|
||||
sample_message="No space left on device",
|
||||
count=2,
|
||||
first_seen="2026-04-10T08:00:00Z",
|
||||
)
|
||||
# Resource type should appear somewhere in the message
|
||||
assert "gce_instance" in result or "GCE" in result or "VM" in result or "Instance" in result
|
||||
|
||||
|
||||
class TestFormatIncidentAlert:
|
||||
"""Tests for format_incident_alert formatting function."""
|
||||
|
||||
def test_p0_uses_siren_emoji(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="Database down",
|
||||
severity="P0",
|
||||
root_cause="Firestore quota exceeded",
|
||||
suggested_fix="Increase quota",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "🚨" in result
|
||||
|
||||
def test_p1_uses_red_circle(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="High error rate",
|
||||
severity="P1",
|
||||
root_cause="Memory pressure",
|
||||
suggested_fix="Scale up",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "🔴" in result
|
||||
|
||||
def test_p2_uses_warning_emoji(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="Elevated latency",
|
||||
severity="P2",
|
||||
root_cause="Slow queries",
|
||||
suggested_fix="Add index",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "⚠️" in result
|
||||
|
||||
def test_p3_uses_yellow_circle(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="Minor degradation",
|
||||
severity="P3",
|
||||
root_cause="Cache miss",
|
||||
suggested_fix="Warm cache",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "🟡" in result
|
||||
|
||||
def test_contains_title(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="My Special Incident Title",
|
||||
severity="P1",
|
||||
root_cause="Something broke",
|
||||
suggested_fix="Fix the thing",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "My Special Incident Title" in result
|
||||
|
||||
def test_contains_root_cause(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="Incident",
|
||||
severity="P1",
|
||||
root_cause="The root cause explanation here",
|
||||
suggested_fix="Fix it",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "The root cause explanation here" in result
|
||||
|
||||
def test_contains_suggested_fix(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
result = format_incident_alert(
|
||||
title="Incident",
|
||||
severity="P1",
|
||||
root_cause="Root cause",
|
||||
suggested_fix="The suggested fix steps here",
|
||||
primary_service="karaoke-backend",
|
||||
patterns=[],
|
||||
)
|
||||
assert "The suggested fix steps here" in result
|
||||
|
||||
def test_shows_up_to_5_patterns(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
patterns = [
|
||||
{"service": f"service-{i}", "normalized_message": f"Error message {i}", "count": i + 1}
|
||||
for i in range(7)
|
||||
]
|
||||
result = format_incident_alert(
|
||||
title="Multi-pattern incident",
|
||||
severity="P1",
|
||||
root_cause="Root cause",
|
||||
suggested_fix="Fix",
|
||||
primary_service="service-0",
|
||||
patterns=patterns,
|
||||
)
|
||||
# First 5 should appear
|
||||
for i in range(5):
|
||||
assert f"Error message {i}" in result
|
||||
# 6th and 7th should not appear
|
||||
assert "Error message 5" not in result
|
||||
assert "Error message 6" not in result
|
||||
|
||||
def test_pattern_message_truncated_to_80_chars(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
|
||||
long_msg = "A" * 100
|
||||
patterns = [{"service": "svc", "normalized_message": long_msg, "count": 1}]
|
||||
result = format_incident_alert(
|
||||
title="Incident",
|
||||
severity="P1",
|
||||
root_cause="Root cause",
|
||||
suggested_fix="Fix",
|
||||
primary_service="svc",
|
||||
patterns=patterns,
|
||||
)
|
||||
# The full 100-char message should not appear; only first 80
|
||||
assert long_msg not in result
|
||||
assert long_msg[:80] in result
|
||||
|
||||
def test_truncates_to_discord_limit(self):
|
||||
from backend.services.error_monitor.discord import format_incident_alert
|
||||
from backend.services.error_monitor.config import DISCORD_MAX_MESSAGE_LENGTH
|
||||
|
||||
result = format_incident_alert(
|
||||
title="T" * 500,
|
||||
severity="P0",
|
||||
root_cause="R" * 500,
|
||||
suggested_fix="F" * 500,
|
||||
primary_service="svc",
|
||||
patterns=[],
|
||||
)
|
||||
assert len(result) <= DISCORD_MAX_MESSAGE_LENGTH
|
||||
|
||||
|
||||
class TestFormatAutoResolvedAlert:
|
||||
"""Tests for format_auto_resolved_alert formatting function."""
|
||||
|
||||
def test_contains_checkmark_emoji(self):
|
||||
from backend.services.error_monitor.discord import format_auto_resolved_alert
|
||||
|
||||
result = format_auto_resolved_alert(
|
||||
resolved_patterns=[
|
||||
{"service": "karaoke-backend", "message": "Connection refused", "hours_silent": 8.5}
|
||||
]
|
||||
)
|
||||
assert "✅" in result
|
||||
|
||||
def test_consolidated_list_up_to_8_patterns(self):
|
||||
from backend.services.error_monitor.discord import format_auto_resolved_alert
|
||||
|
||||
patterns = [
|
||||
{"service": f"service-{i}", "message": f"Error {i}", "hours_silent": float(i + 1)}
|
||||
for i in range(10)
|
||||
]
|
||||
result = format_auto_resolved_alert(resolved_patterns=patterns)
|
||||
# First 8 should appear
|
||||
for i in range(8):
|
||||
assert f"Error {i}" in result
|
||||
# 9th and 10th should not appear
|
||||
assert "Error 8" not in result
|
||||
assert "Error 9" not in result
|
||||
|
||||
def test_contains_service_name(self):
|
||||
from backend.services.error_monitor.discord import format_auto_resolved_alert
|
||||
|
||||
result = format_auto_resolved_alert(
|
||||
resolved_patterns=[
|
||||
{"service": "audio-separator", "message": "GPU error", "hours_silent": 12.0}
|
||||
]
|
||||
)
|
||||
assert "audio-separator" in result
|
||||
|
||||
def test_contains_hours_silent(self):
|
||||
from backend.services.error_monitor.discord import format_auto_resolved_alert
|
||||
|
||||
result = format_auto_resolved_alert(
|
||||
resolved_patterns=[
|
||||
{"service": "karaoke-backend", "message": "Timeout", "hours_silent": 7.5}
|
||||
]
|
||||
)
|
||||
# Should mention 7.5 or the number somewhere
|
||||
assert "7.5" in result or "7" in result
|
||||
|
||||
def test_message_truncated_to_60_chars(self):
|
||||
from backend.services.error_monitor.discord import format_auto_resolved_alert
|
||||
|
||||
long_msg = "B" * 80
|
||||
result = format_auto_resolved_alert(
|
||||
resolved_patterns=[
|
||||
{"service": "svc", "message": long_msg, "hours_silent": 10.0}
|
||||
]
|
||||
)
|
||||
assert long_msg not in result
|
||||
assert long_msg[:60] in result
|
||||
|
||||
def test_empty_patterns_returns_string(self):
|
||||
from backend.services.error_monitor.discord import format_auto_resolved_alert
|
||||
|
||||
result = format_auto_resolved_alert(resolved_patterns=[])
|
||||
assert isinstance(result, str)
|
||||
assert "✅" in result
|
||||
|
||||
|
||||
class TestFormatSpikeAlert:
|
||||
"""Tests for format_spike_alert formatting function."""
|
||||
|
||||
def test_contains_warning_emoji(self):
|
||||
from backend.services.error_monitor.discord import format_spike_alert
|
||||
|
||||
result = format_spike_alert(
|
||||
service="karaoke-backend",
|
||||
normalized_message="Connection refused",
|
||||
current_count=50,
|
||||
rolling_average=5.0,
|
||||
)
|
||||
assert "⚠️" in result
|
||||
|
||||
def test_contains_service_name(self):
|
||||
from backend.services.error_monitor.discord import format_spike_alert
|
||||
|
||||
result = format_spike_alert(
|
||||
service="audio-separator",
|
||||
normalized_message="OOM error",
|
||||
current_count=30,
|
||||
rolling_average=3.0,
|
||||
)
|
||||
assert "audio-separator" in result
|
||||
|
||||
def test_contains_current_count(self):
|
||||
from backend.services.error_monitor.discord import format_spike_alert
|
||||
|
||||
result = format_spike_alert(
|
||||
service="karaoke-backend",
|
||||
normalized_message="Timeout",
|
||||
current_count=75,
|
||||
rolling_average=10.0,
|
||||
)
|
||||
assert "75" in result
|
||||
|
||||
def test_contains_rolling_average(self):
|
||||
from backend.services.error_monitor.discord import format_spike_alert
|
||||
|
||||
result = format_spike_alert(
|
||||
service="karaoke-backend",
|
||||
normalized_message="Timeout",
|
||||
current_count=75,
|
||||
rolling_average=10.0,
|
||||
)
|
||||
assert "10" in result
|
||||
|
||||
def test_contains_multiplier(self):
|
||||
from backend.services.error_monitor.discord import format_spike_alert
|
||||
|
||||
result = format_spike_alert(
|
||||
service="karaoke-backend",
|
||||
normalized_message="Timeout",
|
||||
current_count=50,
|
||||
rolling_average=5.0,
|
||||
)
|
||||
# 50 / 5.0 = 10x
|
||||
assert "10" in result or "10.0" in result
|
||||
|
||||
def test_contains_normalized_message(self):
|
||||
from backend.services.error_monitor.discord import format_spike_alert
|
||||
|
||||
result = format_spike_alert(
|
||||
service="karaoke-backend",
|
||||
normalized_message="Database connection pool exhausted",
|
||||
current_count=50,
|
||||
rolling_average=5.0,
|
||||
)
|
||||
assert "Database connection pool exhausted" in result
|
||||
|
||||
|
||||
class TestFormatDailyDigest:
|
||||
"""Tests for format_daily_digest formatting function."""
|
||||
|
||||
def test_contains_chart_emoji(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
|
||||
result = format_daily_digest(
|
||||
total_errors=100,
|
||||
new_patterns=5,
|
||||
resolved_patterns=2,
|
||||
active_patterns=10,
|
||||
per_service_counts={"karaoke-backend": 60, "audio-separator": 40},
|
||||
)
|
||||
assert "📊" in result
|
||||
|
||||
def test_contains_total_errors(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
|
||||
result = format_daily_digest(
|
||||
total_errors=1234,
|
||||
new_patterns=5,
|
||||
resolved_patterns=2,
|
||||
active_patterns=10,
|
||||
per_service_counts={},
|
||||
)
|
||||
assert "1234" in result
|
||||
|
||||
def test_contains_new_patterns(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
|
||||
result = format_daily_digest(
|
||||
total_errors=100,
|
||||
new_patterns=7,
|
||||
resolved_patterns=2,
|
||||
active_patterns=10,
|
||||
per_service_counts={},
|
||||
)
|
||||
assert "7" in result
|
||||
|
||||
def test_contains_per_service_counts(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
|
||||
result = format_daily_digest(
|
||||
total_errors=100,
|
||||
new_patterns=5,
|
||||
resolved_patterns=2,
|
||||
active_patterns=10,
|
||||
per_service_counts={"karaoke-backend": 60, "audio-separator": 40},
|
||||
)
|
||||
assert "karaoke-backend" in result
|
||||
assert "audio-separator" in result
|
||||
assert "60" in result
|
||||
assert "40" in result
|
||||
|
||||
def test_per_service_sorted_by_count_desc(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
|
||||
result = format_daily_digest(
|
||||
total_errors=100,
|
||||
new_patterns=5,
|
||||
resolved_patterns=2,
|
||||
active_patterns=10,
|
||||
per_service_counts={
|
||||
"service-low": 10,
|
||||
"service-high": 80,
|
||||
"service-mid": 30,
|
||||
},
|
||||
)
|
||||
# service-high should appear before service-low in the output
|
||||
high_pos = result.index("service-high")
|
||||
low_pos = result.index("service-low")
|
||||
assert high_pos < low_pos
|
||||
|
||||
def test_contains_active_patterns(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
|
||||
result = format_daily_digest(
|
||||
total_errors=100,
|
||||
new_patterns=5,
|
||||
resolved_patterns=2,
|
||||
active_patterns=15,
|
||||
per_service_counts={},
|
||||
)
|
||||
assert "15" in result
|
||||
|
||||
def test_truncates_to_discord_limit(self):
|
||||
from backend.services.error_monitor.discord import format_daily_digest
|
||||
from backend.services.error_monitor.config import DISCORD_MAX_MESSAGE_LENGTH
|
||||
|
||||
# Create lots of services to potentially overflow
|
||||
per_service = {f"service-{i:03d}": i * 10 for i in range(100)}
|
||||
result = format_daily_digest(
|
||||
total_errors=99999,
|
||||
new_patterns=99,
|
||||
resolved_patterns=50,
|
||||
active_patterns=200,
|
||||
per_service_counts=per_service,
|
||||
)
|
||||
assert len(result) <= DISCORD_MAX_MESSAGE_LENGTH
|
||||
|
||||
|
||||
class TestErrorMonitorDiscordSendMessage:
|
||||
"""Tests for ErrorMonitorDiscord.send_message method."""
|
||||
|
||||
def test_send_message_returns_true_on_success(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.status_code = 204
|
||||
mock_response.raise_for_status.return_value = None
|
||||
|
||||
with mock.patch("requests.post", return_value=mock_response) as mock_post:
|
||||
result = client.send_message("Hello Discord!")
|
||||
assert result is True
|
||||
mock_post.assert_called_once_with(
|
||||
"https://discord.com/api/webhooks/123/abc",
|
||||
json={"content": "Hello Discord!"},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
def test_send_message_returns_false_on_http_error(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.raise_for_status.side_effect = requests.HTTPError("429 Too Many Requests")
|
||||
|
||||
with mock.patch("requests.post", return_value=mock_response):
|
||||
result = client.send_message("Hello Discord!")
|
||||
assert result is False
|
||||
|
||||
def test_send_message_returns_false_on_connection_error(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
|
||||
with mock.patch("requests.post", side_effect=requests.ConnectionError("timeout")):
|
||||
result = client.send_message("Hello Discord!")
|
||||
assert result is False
|
||||
|
||||
def test_send_message_respects_rate_limit(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
client.messages_sent = MAX_DISCORD_MESSAGES_PER_RUN # Already at limit
|
||||
|
||||
with mock.patch("requests.post") as mock_post:
|
||||
result = client.send_message("Should be skipped")
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_send_message_increments_counter_on_success(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
assert client.messages_sent == 0
|
||||
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.raise_for_status.return_value = None
|
||||
|
||||
with mock.patch("requests.post", return_value=mock_response):
|
||||
client.send_message("First message")
|
||||
assert client.messages_sent == 1
|
||||
client.send_message("Second message")
|
||||
assert client.messages_sent == 2
|
||||
|
||||
def test_send_message_increments_counter_on_failure(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
assert client.messages_sent == 0
|
||||
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.raise_for_status.side_effect = requests.HTTPError("500")
|
||||
|
||||
with mock.patch("requests.post", return_value=mock_response):
|
||||
client.send_message("Will fail")
|
||||
assert client.messages_sent == 1
|
||||
|
||||
def test_send_message_initial_counter_is_zero(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
assert client.messages_sent == 0
|
||||
|
||||
def test_send_message_does_not_increment_when_rate_limited(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/123/abc")
|
||||
client.messages_sent = MAX_DISCORD_MESSAGES_PER_RUN
|
||||
|
||||
with mock.patch("requests.post") as mock_post:
|
||||
client.send_message("Skipped")
|
||||
# Counter should not go beyond limit when rate-limited
|
||||
assert client.messages_sent == MAX_DISCORD_MESSAGES_PER_RUN
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,445 @@
|
||||
"""Tests for error_monitor.known_issues module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestIgnoreReasonDataclass:
|
||||
"""IgnoreReason dataclass should hold pattern_name and reason fields."""
|
||||
|
||||
def test_ignore_reason_has_pattern_name(self):
|
||||
from backend.services.error_monitor.known_issues import IgnoreReason
|
||||
|
||||
ir = IgnoreReason(pattern_name="startup_probe", reason="Cloud Run cold start")
|
||||
assert ir.pattern_name == "startup_probe"
|
||||
|
||||
def test_ignore_reason_has_reason(self):
|
||||
from backend.services.error_monitor.known_issues import IgnoreReason
|
||||
|
||||
ir = IgnoreReason(pattern_name="startup_probe", reason="Cloud Run cold start")
|
||||
assert ir.reason == "Cloud Run cold start"
|
||||
|
||||
def test_ignore_reason_is_dataclass_equality(self):
|
||||
from backend.services.error_monitor.known_issues import IgnoreReason
|
||||
|
||||
a = IgnoreReason(pattern_name="foo", reason="bar")
|
||||
b = IgnoreReason(pattern_name="foo", reason="bar")
|
||||
assert a == b
|
||||
|
||||
def test_ignore_reason_inequality(self):
|
||||
from backend.services.error_monitor.known_issues import IgnoreReason
|
||||
|
||||
a = IgnoreReason(pattern_name="foo", reason="bar")
|
||||
b = IgnoreReason(pattern_name="baz", reason="bar")
|
||||
assert a != b
|
||||
|
||||
|
||||
class TestStartupProbePattern:
|
||||
"""startup_probe — 'startup probe failed' should be ignored (Cloud Run cold start)."""
|
||||
|
||||
def test_startup_probe_exact(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "startup probe failed")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "startup_probe"
|
||||
|
||||
def test_startup_probe_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "Startup Probe Failed")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "startup_probe"
|
||||
|
||||
def test_startup_probe_in_longer_message(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Readiness probe failed: startup probe failed for container api",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "startup_probe"
|
||||
|
||||
def test_startup_probe_any_service(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("audio-separator", "startup probe failed")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "startup_probe"
|
||||
|
||||
|
||||
class TestReadyConditionPattern:
|
||||
"""ready_condition — 'ready condition status changed' should be ignored (deploy event)."""
|
||||
|
||||
def test_ready_condition_exact(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "ready condition status changed")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "ready_condition"
|
||||
|
||||
def test_ready_condition_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "Ready Condition Status Changed")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "ready_condition"
|
||||
|
||||
def test_ready_condition_with_context(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Service karaoke-backend: ready condition status changed to True",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "ready_condition"
|
||||
|
||||
|
||||
class TestSpotPreemptionPattern:
|
||||
"""spot_preemption — 'instance was preempted' should be ignored (GitHub runner spot VMs)."""
|
||||
|
||||
def test_spot_preemption_exact(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("encoding-worker-a", "instance was preempted")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "spot_preemption"
|
||||
|
||||
def test_spot_preemption_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("encoding-worker-a", "Instance Was Preempted")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "spot_preemption"
|
||||
|
||||
def test_spot_preemption_in_sentence(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"encoding-worker-b",
|
||||
"GCE: instance was preempted, shutting down",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "spot_preemption"
|
||||
|
||||
|
||||
class TestIdleShutdownPattern:
|
||||
"""idle_shutdown — 'stopping idle encoding worker' should be ignored (expected behavior)."""
|
||||
|
||||
def test_idle_shutdown_exact(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("encoding_worker_idle", "stopping idle encoding worker")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "idle_shutdown"
|
||||
|
||||
def test_idle_shutdown_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("encoding_worker_idle", "Stopping Idle Encoding Worker")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "idle_shutdown"
|
||||
|
||||
def test_idle_shutdown_with_context(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"encoding_worker_idle",
|
||||
"stopping idle encoding worker after 30 minutes of inactivity",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "idle_shutdown"
|
||||
|
||||
|
||||
class TestHealthCheck404Pattern:
|
||||
"""health_check_404 — GET/HEAD /health with 404/not found should be ignored (load balancer)."""
|
||||
|
||||
def test_health_check_get_404(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", 'GET /health 404')
|
||||
assert result is not None
|
||||
assert result.pattern_name == "health_check_404"
|
||||
|
||||
def test_health_check_head_404(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", 'HEAD /health 404')
|
||||
assert result is not None
|
||||
assert result.pattern_name == "health_check_404"
|
||||
|
||||
def test_health_check_get_not_found(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", 'GET /health not found')
|
||||
assert result is not None
|
||||
assert result.pattern_name == "health_check_404"
|
||||
|
||||
def test_health_check_head_not_found(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", 'HEAD /health not found')
|
||||
assert result is not None
|
||||
assert result.pattern_name == "health_check_404"
|
||||
|
||||
def test_health_check_uppercase_method(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "GET /health 404 Not Found")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "health_check_404"
|
||||
|
||||
def test_health_check_in_log_line(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
'10.0.0.1 - - [01/Jan/2024] "GET /health HTTP/1.1" 404 0',
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "health_check_404"
|
||||
|
||||
def test_non_health_404_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
# A 404 for a real endpoint should NOT be ignored
|
||||
result = should_ignore("karaoke-backend", "GET /api/jobs/abc 404")
|
||||
assert result is None
|
||||
|
||||
def test_health_200_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
# /health returning 200 is not an error, but should still not trigger this pattern
|
||||
result = should_ignore("karaoke-backend", "GET /health 200")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestSchedulerRetryPattern:
|
||||
"""scheduler_retry — 'cloud scheduler.*retry' should be ignored (transient)."""
|
||||
|
||||
def test_scheduler_retry_basic(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("runner_manager", "cloud scheduler retry attempt 1")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "scheduler_retry"
|
||||
|
||||
def test_scheduler_retry_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("runner_manager", "Cloud Scheduler Retry")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "scheduler_retry"
|
||||
|
||||
def test_scheduler_retry_with_details(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"runner_manager",
|
||||
"cloud scheduler job failed, will retry in 60s",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "scheduler_retry"
|
||||
|
||||
def test_cloud_scheduler_without_retry_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
# "cloud scheduler" alone (without "retry") should NOT be suppressed
|
||||
result = should_ignore("runner_manager", "cloud scheduler job executed successfully")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestRunnerStartupPattern:
|
||||
"""runner_startup — 'github.runner.*(starting|stopping|idle)' should be ignored (lifecycle)."""
|
||||
|
||||
def test_runner_starting(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("runner_manager", "github.runner.001 starting")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "runner_startup"
|
||||
|
||||
def test_runner_stopping(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("runner_manager", "github.runner.abc stopping")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "runner_startup"
|
||||
|
||||
def test_runner_idle(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("runner_manager", "github.runner.xyz idle")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "runner_startup"
|
||||
|
||||
def test_runner_starting_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("runner_manager", "GitHub.Runner.001 Starting")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "runner_startup"
|
||||
|
||||
def test_runner_other_state_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
# A GitHub runner in an unexpected state should NOT be ignored
|
||||
result = should_ignore("runner_manager", "github.runner.001 crashed")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestContainerShutdownPattern:
|
||||
"""container_shutdown — 'container called exit(0)' should be ignored (clean shutdown)."""
|
||||
|
||||
def test_container_exit_0(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "container called exit(0)")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "container_shutdown"
|
||||
|
||||
def test_container_exit_0_uppercase(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "Container Called Exit(0)")
|
||||
assert result is not None
|
||||
assert result.pattern_name == "container_shutdown"
|
||||
|
||||
def test_container_exit_0_in_sentence(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Cloud Run instance terminated: container called exit(0)",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "container_shutdown"
|
||||
|
||||
def test_container_exit_nonzero_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
# exit(1) = error, should NOT be ignored
|
||||
result = should_ignore("karaoke-backend", "container called exit(1)")
|
||||
assert result is None
|
||||
|
||||
def test_container_exit_2_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "container called exit(2)")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestRealErrorsNotIgnored:
|
||||
"""Real errors should never be silently ignored."""
|
||||
|
||||
def test_firestore_deadline_exceeded(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Firestore transaction failed: DEADLINE_EXCEEDED",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_audio_separation_oom(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"audio-separator",
|
||||
"CUDA out of memory. Tried to allocate 2.00 GiB",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_payment_stripe_error(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Stripe charge failed: card_declined",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_gcs_permission_denied(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Permission denied accessing gs://nomadkaraoke-prod/output.mp4",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_worker_timeout(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"video-encoding-job",
|
||||
"Worker timed out after 3600 seconds",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_unhandled_exception(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore(
|
||||
"karaoke-backend",
|
||||
"Unhandled exception: KeyError 'job_id'",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestUnknownServiceMessages:
|
||||
"""Messages from unlisted/unknown services should use pattern matching, not service-gating."""
|
||||
|
||||
def test_unknown_service_known_pattern_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
# The patterns apply regardless of service name
|
||||
result = should_ignore("some-unknown-service", "startup probe failed")
|
||||
assert result is not None
|
||||
|
||||
def test_unknown_service_real_error_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("some-unknown-service", "unhandled exception in request handler")
|
||||
assert result is None
|
||||
|
||||
def test_empty_service_name_with_known_pattern(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("", "startup probe failed")
|
||||
assert result is not None
|
||||
|
||||
def test_empty_message_not_ignored(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestCaseInsensitiveMatching:
|
||||
"""All patterns should match case-insensitively."""
|
||||
|
||||
def test_startup_probe_all_caps(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "STARTUP PROBE FAILED")
|
||||
assert result is not None
|
||||
|
||||
def test_ready_condition_mixed_case(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "Ready CONDITION Status CHANGED")
|
||||
assert result is not None
|
||||
|
||||
def test_idle_shutdown_mixed_case(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("encoding_worker_idle", "STOPPING idle ENCODING worker")
|
||||
assert result is not None
|
||||
|
||||
def test_container_shutdown_mixed_case(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("karaoke-backend", "Container Called EXIT(0)")
|
||||
assert result is not None
|
||||
@@ -0,0 +1,507 @@
|
||||
"""Tests for error_monitor.llm_analysis module.
|
||||
|
||||
All tests mock _call_llm to avoid real LLM calls.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SAMPLE_PATTERNS = [
|
||||
{
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Connection refused to Firestore",
|
||||
"count": 15,
|
||||
"first_seen": "2026-04-10T10:00:00Z",
|
||||
"last_seen": "2026-04-10T10:30:00Z",
|
||||
},
|
||||
{
|
||||
"service": "audio-separation-job",
|
||||
"normalized_message": "CUDA out of memory error",
|
||||
"count": 8,
|
||||
"first_seen": "2026-04-10T10:05:00Z",
|
||||
"last_seen": "2026-04-10T10:25:00Z",
|
||||
},
|
||||
{
|
||||
"service": "audio-download-job",
|
||||
"normalized_message": "flacfetch-vm connection timeout",
|
||||
"count": 5,
|
||||
"first_seen": "2026-04-10T10:10:00Z",
|
||||
"last_seen": "2026-04-10T10:20:00Z",
|
||||
},
|
||||
]
|
||||
|
||||
VALID_INCIDENTS_JSON = json.dumps(
|
||||
{
|
||||
"incidents": [
|
||||
{
|
||||
"title": "Firestore connectivity failure",
|
||||
"root_cause": "Firestore is unavailable or misconfigured",
|
||||
"severity": "P1",
|
||||
"suggested_fix": "Check Firestore service health and credentials",
|
||||
"primary_service": "karaoke-backend",
|
||||
"pattern_indices": [0],
|
||||
},
|
||||
{
|
||||
"title": "GPU audio separation failure",
|
||||
"root_cause": "GPU out of memory on audio-separation-job",
|
||||
"severity": "P2",
|
||||
"suggested_fix": "Check GPU memory limits and job queue depth",
|
||||
"primary_service": "audio-separation-job",
|
||||
"pattern_indices": [1, 2],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
VALID_DUPLICATES_JSON = json.dumps(
|
||||
{
|
||||
"duplicates": [
|
||||
{
|
||||
"canonical_index": 0,
|
||||
"duplicate_indices": [1],
|
||||
"reason": "Both errors indicate Firestore connection issues",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for _parse_llm_response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseLlmResponse:
|
||||
"""Unit tests for the _parse_llm_response helper."""
|
||||
|
||||
def test_valid_json_returns_incident_analysis(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
result = _parse_llm_response(VALID_INCIDENTS_JSON)
|
||||
assert result is not None
|
||||
assert len(result.incidents) == 2
|
||||
assert result.used_llm is True
|
||||
|
||||
def test_valid_json_incident_fields(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
result = _parse_llm_response(VALID_INCIDENTS_JSON)
|
||||
assert result is not None
|
||||
first = result.incidents[0]
|
||||
assert first.title == "Firestore connectivity failure"
|
||||
assert first.root_cause == "Firestore is unavailable or misconfigured"
|
||||
assert first.severity == "P1"
|
||||
assert first.primary_service == "karaoke-backend"
|
||||
assert first.pattern_indices == [0]
|
||||
|
||||
def test_invalid_json_returns_none(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
result = _parse_llm_response("not valid json {{{")
|
||||
assert result is None
|
||||
|
||||
def test_missing_incidents_key_returns_none(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
result = _parse_llm_response(json.dumps({"data": []}))
|
||||
assert result is None
|
||||
|
||||
def test_markdown_code_block_stripped(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
wrapped = f"```json\n{VALID_INCIDENTS_JSON}\n```"
|
||||
result = _parse_llm_response(wrapped)
|
||||
assert result is not None
|
||||
assert len(result.incidents) == 2
|
||||
|
||||
def test_markdown_code_block_without_language_stripped(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
wrapped = f"```\n{VALID_INCIDENTS_JSON}\n```"
|
||||
result = _parse_llm_response(wrapped)
|
||||
assert result is not None
|
||||
assert len(result.incidents) == 2
|
||||
|
||||
def test_empty_string_returns_none(self):
|
||||
from backend.services.error_monitor.llm_analysis import _parse_llm_response
|
||||
|
||||
result = _parse_llm_response("")
|
||||
assert result is None
|
||||
|
||||
def test_incidents_is_list_of_incident_objects(self):
|
||||
from backend.services.error_monitor.llm_analysis import (
|
||||
_parse_llm_response,
|
||||
Incident,
|
||||
)
|
||||
|
||||
result = _parse_llm_response(VALID_INCIDENTS_JSON)
|
||||
assert result is not None
|
||||
for incident in result.incidents:
|
||||
assert isinstance(incident, Incident)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for analyze_patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAnalyzePatterns:
|
||||
"""Unit tests for analyze_patterns."""
|
||||
|
||||
def test_returns_none_for_fewer_than_min_patterns(self):
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
from backend.services.error_monitor.config import MIN_PATTERNS_FOR_ANALYSIS
|
||||
|
||||
# Build a list with fewer patterns than the minimum
|
||||
few_patterns = SAMPLE_PATTERNS[: MIN_PATTERNS_FOR_ANALYSIS - 1]
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm"
|
||||
) as mock_call:
|
||||
result = analyze_patterns(few_patterns)
|
||||
|
||||
assert result is None
|
||||
mock_call.assert_not_called()
|
||||
|
||||
def test_groups_patterns_into_incidents(self):
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=VALID_INCIDENTS_JSON,
|
||||
):
|
||||
result = analyze_patterns(SAMPLE_PATTERNS)
|
||||
|
||||
assert result is not None
|
||||
assert len(result.incidents) == 2
|
||||
assert result.used_llm is True
|
||||
|
||||
def test_incident_has_correct_fields(self):
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=VALID_INCIDENTS_JSON,
|
||||
):
|
||||
result = analyze_patterns(SAMPLE_PATTERNS)
|
||||
|
||||
assert result is not None
|
||||
incident = result.incidents[0]
|
||||
assert incident.title == "Firestore connectivity failure"
|
||||
assert incident.severity == "P1"
|
||||
assert incident.primary_service == "karaoke-backend"
|
||||
|
||||
def test_falls_back_on_llm_exception(self):
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
side_effect=Exception("LLM unavailable"),
|
||||
):
|
||||
result = analyze_patterns(SAMPLE_PATTERNS)
|
||||
|
||||
# Should fall back to service grouping, not None
|
||||
assert result is not None
|
||||
assert result.used_llm is False
|
||||
|
||||
def test_fallback_groups_by_service(self):
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
side_effect=Exception("LLM unavailable"),
|
||||
):
|
||||
result = analyze_patterns(SAMPLE_PATTERNS)
|
||||
|
||||
assert result is not None
|
||||
# Each unique service should map to one incident
|
||||
services = {inc.primary_service for inc in result.incidents}
|
||||
expected_services = {p["service"] for p in SAMPLE_PATTERNS}
|
||||
assert services == expected_services
|
||||
|
||||
def test_falls_back_on_invalid_llm_response(self):
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value="invalid json",
|
||||
):
|
||||
result = analyze_patterns(SAMPLE_PATTERNS)
|
||||
|
||||
# Should fall back to service grouping on parse failure
|
||||
assert result is not None
|
||||
assert result.used_llm is False
|
||||
|
||||
def test_exact_min_patterns_triggers_analysis(self):
|
||||
"""Exactly MIN_PATTERNS_FOR_ANALYSIS patterns should trigger the LLM."""
|
||||
from backend.services.error_monitor.llm_analysis import analyze_patterns
|
||||
from backend.services.error_monitor.config import MIN_PATTERNS_FOR_ANALYSIS
|
||||
|
||||
# Ensure we have exactly MIN_PATTERNS_FOR_ANALYSIS patterns
|
||||
exactly_min = SAMPLE_PATTERNS[:MIN_PATTERNS_FOR_ANALYSIS]
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=VALID_INCIDENTS_JSON,
|
||||
) as mock_call:
|
||||
result = analyze_patterns(exactly_min)
|
||||
|
||||
mock_call.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for _fallback_group_by_service
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFallbackGroupByService:
|
||||
"""Unit tests for _fallback_group_by_service."""
|
||||
|
||||
def test_groups_by_service(self):
|
||||
from backend.services.error_monitor.llm_analysis import _fallback_group_by_service
|
||||
|
||||
result = _fallback_group_by_service(SAMPLE_PATTERNS)
|
||||
services = {inc.primary_service for inc in result.incidents}
|
||||
assert services == {"karaoke-backend", "audio-separation-job", "audio-download-job"}
|
||||
|
||||
def test_used_llm_is_false(self):
|
||||
from backend.services.error_monitor.llm_analysis import _fallback_group_by_service
|
||||
|
||||
result = _fallback_group_by_service(SAMPLE_PATTERNS)
|
||||
assert result.used_llm is False
|
||||
|
||||
def test_severity_is_p2(self):
|
||||
from backend.services.error_monitor.llm_analysis import _fallback_group_by_service
|
||||
|
||||
result = _fallback_group_by_service(SAMPLE_PATTERNS)
|
||||
for incident in result.incidents:
|
||||
assert incident.severity == "P2"
|
||||
|
||||
def test_pattern_indices_are_populated(self):
|
||||
from backend.services.error_monitor.llm_analysis import _fallback_group_by_service
|
||||
|
||||
# Two patterns for the same service
|
||||
two_patterns = [
|
||||
{
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Error A",
|
||||
"count": 3,
|
||||
},
|
||||
{
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Error B",
|
||||
"count": 2,
|
||||
},
|
||||
]
|
||||
result = _fallback_group_by_service(two_patterns)
|
||||
assert len(result.incidents) == 1
|
||||
assert result.incidents[0].pattern_indices == [0, 1]
|
||||
|
||||
def test_empty_patterns_returns_empty_incidents(self):
|
||||
from backend.services.error_monitor.llm_analysis import _fallback_group_by_service
|
||||
|
||||
result = _fallback_group_by_service([])
|
||||
assert result.incidents == []
|
||||
assert result.used_llm is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for find_duplicate_patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFindDuplicatePatterns:
|
||||
"""Unit tests for find_duplicate_patterns."""
|
||||
|
||||
def test_returns_empty_for_empty_new_patterns(self):
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm"
|
||||
) as mock_call:
|
||||
result = find_duplicate_patterns([], SAMPLE_PATTERNS)
|
||||
|
||||
assert result == []
|
||||
mock_call.assert_not_called()
|
||||
|
||||
def test_returns_empty_for_empty_existing_patterns(self):
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm"
|
||||
) as mock_call:
|
||||
result = find_duplicate_patterns(SAMPLE_PATTERNS, [])
|
||||
|
||||
assert result == []
|
||||
mock_call.assert_not_called()
|
||||
|
||||
def test_identifies_duplicates(self):
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
new_patterns = [
|
||||
{
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Cannot reach Firestore endpoint",
|
||||
},
|
||||
{
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Firestore deadline exceeded",
|
||||
},
|
||||
]
|
||||
existing_patterns = [
|
||||
{
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Connection refused to Firestore",
|
||||
}
|
||||
]
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=VALID_DUPLICATES_JSON,
|
||||
):
|
||||
result = find_duplicate_patterns(new_patterns, existing_patterns)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].canonical_index == 0
|
||||
assert result[0].duplicate_indices == [1]
|
||||
assert "Firestore" in result[0].reason
|
||||
|
||||
def test_returns_empty_on_no_duplicates_in_response(self):
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
no_dups_json = json.dumps({"duplicates": []})
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=no_dups_json,
|
||||
):
|
||||
result = find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:])
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_returns_empty_on_llm_failure(self):
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
side_effect=Exception("LLM unavailable"),
|
||||
):
|
||||
result = find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:])
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_returns_empty_on_invalid_response(self):
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value="not json",
|
||||
):
|
||||
result = find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:])
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_duplicate_group_fields(self):
|
||||
from backend.services.error_monitor.llm_analysis import (
|
||||
find_duplicate_patterns,
|
||||
DuplicateGroup,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=VALID_DUPLICATES_JSON,
|
||||
):
|
||||
result = find_duplicate_patterns(SAMPLE_PATTERNS[:2], SAMPLE_PATTERNS[2:])
|
||||
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0], DuplicateGroup)
|
||||
|
||||
def test_uses_low_temperature(self):
|
||||
"""find_duplicate_patterns should call _call_llm with temperature=0.1."""
|
||||
from backend.services.error_monitor.llm_analysis import find_duplicate_patterns
|
||||
|
||||
with patch(
|
||||
"backend.services.error_monitor.llm_analysis._call_llm",
|
||||
return_value=VALID_DUPLICATES_JSON,
|
||||
) as mock_call:
|
||||
find_duplicate_patterns(SAMPLE_PATTERNS[:1], SAMPLE_PATTERNS[1:])
|
||||
|
||||
# Check the temperature kwarg was 0.1
|
||||
call_kwargs = mock_call.call_args
|
||||
assert call_kwargs is not None
|
||||
# temperature is the 3rd positional arg or keyword arg
|
||||
if call_kwargs.kwargs.get("temperature") is not None:
|
||||
assert call_kwargs.kwargs["temperature"] == 0.1
|
||||
else:
|
||||
# positional: (system_prompt, user_prompt, temperature)
|
||||
assert call_kwargs.args[2] == 0.1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDataclasses:
|
||||
"""Tests for the exported dataclasses."""
|
||||
|
||||
def test_incident_dataclass(self):
|
||||
from backend.services.error_monitor.llm_analysis import Incident
|
||||
|
||||
inc = Incident(
|
||||
title="Test",
|
||||
root_cause="Root cause",
|
||||
severity="P1",
|
||||
suggested_fix="Fix it",
|
||||
primary_service="karaoke-backend",
|
||||
pattern_indices=[0, 1],
|
||||
)
|
||||
assert inc.title == "Test"
|
||||
assert inc.severity == "P1"
|
||||
assert inc.pattern_indices == [0, 1]
|
||||
|
||||
def test_incident_nullable_fields(self):
|
||||
from backend.services.error_monitor.llm_analysis import Incident
|
||||
|
||||
inc = Incident(
|
||||
title="Test",
|
||||
root_cause=None,
|
||||
severity="P3",
|
||||
suggested_fix=None,
|
||||
primary_service="audio-separator",
|
||||
pattern_indices=[2],
|
||||
)
|
||||
assert inc.root_cause is None
|
||||
assert inc.suggested_fix is None
|
||||
|
||||
def test_incident_analysis_dataclass(self):
|
||||
from backend.services.error_monitor.llm_analysis import IncidentAnalysis, Incident
|
||||
|
||||
ia = IncidentAnalysis(incidents=[])
|
||||
assert ia.incidents == []
|
||||
assert ia.used_llm is True # default
|
||||
|
||||
def test_incident_analysis_used_llm_false(self):
|
||||
from backend.services.error_monitor.llm_analysis import IncidentAnalysis
|
||||
|
||||
ia = IncidentAnalysis(incidents=[], used_llm=False)
|
||||
assert ia.used_llm is False
|
||||
|
||||
def test_duplicate_group_dataclass(self):
|
||||
from backend.services.error_monitor.llm_analysis import DuplicateGroup
|
||||
|
||||
dg = DuplicateGroup(
|
||||
canonical_index=0,
|
||||
duplicate_indices=[1, 2],
|
||||
reason="Same root cause",
|
||||
)
|
||||
assert dg.canonical_index == 0
|
||||
assert dg.duplicate_indices == [1, 2]
|
||||
assert dg.reason == "Same root cause"
|
||||
@@ -0,0 +1,954 @@
|
||||
"""Tests for error_monitor.monitor module.
|
||||
|
||||
All external dependencies (Cloud Logging, Firestore, Discord, LLM, Secret Manager)
|
||||
are mocked — no real GCP connections are made.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _import_monitor_module():
|
||||
from backend.services.error_monitor import monitor as m
|
||||
|
||||
return m
|
||||
|
||||
|
||||
def _import_error_monitor_class():
|
||||
from backend.services.error_monitor.monitor import ErrorMonitor
|
||||
|
||||
return ErrorMonitor
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper to build a minimal mock log entry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_entry(
|
||||
payload=None,
|
||||
resource_type="cloud_run_revision",
|
||||
service_name="karaoke-backend",
|
||||
job_name=None,
|
||||
function_name=None,
|
||||
instance_id=None,
|
||||
resource_name=None,
|
||||
):
|
||||
"""Return a mock Cloud Logging entry."""
|
||||
entry = MagicMock()
|
||||
entry.payload = payload
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.type = resource_type
|
||||
entry.resource.labels = {
|
||||
"service_name": service_name,
|
||||
"job_name": job_name or "",
|
||||
"function_name": function_name or "",
|
||||
"instance_id": instance_id or "",
|
||||
}
|
||||
entry.labels = {}
|
||||
if resource_name:
|
||||
entry.labels["compute.googleapis.com/resource_name"] = resource_name
|
||||
return entry
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _build_log_filter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildLogFilter:
|
||||
"""_build_log_filter should produce correct Cloud Logging filter strings."""
|
||||
|
||||
def test_cloud_run_revision_filter(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._build_log_filter("cloud_run_revision", ["svc-a", "svc-b"], 15)
|
||||
assert 'resource.type="cloud_run_revision"' in result
|
||||
assert "severity>=ERROR" in result
|
||||
assert "svc-a" in result
|
||||
assert "svc-b" in result
|
||||
assert "resource.labels.service_name" in result
|
||||
|
||||
def test_cloud_run_job_filter(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._build_log_filter("cloud_run_job", ["job-x"], 15)
|
||||
assert 'resource.type="cloud_run_job"' in result
|
||||
assert "severity>=ERROR" in result
|
||||
assert "job-x" in result
|
||||
assert "resource.labels.job_name" in result
|
||||
|
||||
def test_gce_instance_filter(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._build_log_filter("gce_instance", ["enc-worker-a"], 15)
|
||||
assert 'resource.type="gce_instance"' in result
|
||||
assert "severity>=ERROR" in result
|
||||
assert "enc-worker-a" in result
|
||||
assert 'compute.googleapis.com/resource_name' in result
|
||||
|
||||
def test_cloud_function_filter(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._build_log_filter("cloud_function", ["fn-one", "fn-two"], 10)
|
||||
assert 'resource.type="cloud_function"' in result
|
||||
assert "fn-one" in result
|
||||
assert "fn-two" in result
|
||||
|
||||
def test_filter_contains_timestamp(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._build_log_filter("cloud_run_revision", ["svc"], 30)
|
||||
assert "timestamp>=" in result
|
||||
|
||||
def test_single_resource_no_or_clause_needed(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._build_log_filter("cloud_run_revision", ["only-svc"], 15)
|
||||
assert "only-svc" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _extract_message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractMessage:
|
||||
"""_extract_message should handle all payload shapes."""
|
||||
|
||||
def test_text_payload_string_returned_directly(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = "Something went wrong"
|
||||
result = m._extract_message(entry)
|
||||
assert result == "Something went wrong"
|
||||
|
||||
def test_dict_payload_message_key(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = {"message": "DB connection refused"}
|
||||
result = m._extract_message(entry)
|
||||
assert result == "DB connection refused"
|
||||
|
||||
def test_dict_payload_textPayload_key(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = {"textPayload": "File not found: /tmp/foo"}
|
||||
result = m._extract_message(entry)
|
||||
assert result == "File not found: /tmp/foo"
|
||||
|
||||
def test_dict_payload_error_key(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = {"error": "timeout after 30s"}
|
||||
result = m._extract_message(entry)
|
||||
assert result == "timeout after 30s"
|
||||
|
||||
def test_dict_payload_msg_key(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = {"msg": "panic: nil pointer dereference"}
|
||||
result = m._extract_message(entry)
|
||||
assert result == "panic: nil pointer dereference"
|
||||
|
||||
def test_none_payload_returns_none(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = None
|
||||
result = m._extract_message(entry)
|
||||
assert result is None
|
||||
|
||||
def test_dict_payload_no_known_key_returns_none(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = {"severity": "ERROR", "timestamp": "2026-04-10T10:00:00Z"}
|
||||
result = m._extract_message(entry)
|
||||
assert result is None
|
||||
|
||||
def test_empty_string_payload_treated_as_text(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.payload = ""
|
||||
result = m._extract_message(entry)
|
||||
# Empty string is falsy — should return the string or None depending on impl
|
||||
# As long as it doesn't crash; either return value is acceptable
|
||||
assert result is None or result == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _extract_service_name
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractServiceName:
|
||||
"""_extract_service_name should return the correct label for each resource type."""
|
||||
|
||||
def test_cloud_run_revision_returns_service_name(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.labels = {"service_name": "karaoke-backend"}
|
||||
result = m._extract_service_name(entry, "cloud_run_revision")
|
||||
assert result == "karaoke-backend"
|
||||
|
||||
def test_cloud_run_job_returns_job_name(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.labels = {"job_name": "video-encoding-job"}
|
||||
result = m._extract_service_name(entry, "cloud_run_job")
|
||||
assert result == "video-encoding-job"
|
||||
|
||||
def test_cloud_function_returns_function_name(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.labels = {"function_name": "gdrive-validator"}
|
||||
result = m._extract_service_name(entry, "cloud_function")
|
||||
assert result == "gdrive-validator"
|
||||
|
||||
def test_gce_instance_returns_resource_name_from_labels(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.labels = {}
|
||||
entry.labels = {"compute.googleapis.com/resource_name": "encoding-worker-a"}
|
||||
result = m._extract_service_name(entry, "gce_instance")
|
||||
assert result == "encoding-worker-a"
|
||||
|
||||
def test_gce_instance_falls_back_to_instance_id(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.labels = {"instance_id": "1234567890"}
|
||||
entry.labels = {}
|
||||
result = m._extract_service_name(entry, "gce_instance")
|
||||
assert result == "1234567890"
|
||||
|
||||
def test_missing_labels_returns_unknown(self):
|
||||
m = _import_monitor_module()
|
||||
entry = MagicMock()
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.labels = {}
|
||||
entry.labels = {}
|
||||
# Should not raise — returns some fallback string
|
||||
result = m._extract_service_name(entry, "cloud_run_revision")
|
||||
assert isinstance(result, str)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _classify_resource_type
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestClassifyResourceType:
|
||||
"""_classify_resource_type should map log resource types to canonical types."""
|
||||
|
||||
def test_cloud_run_revision_mapped(self):
|
||||
m = _import_monitor_module()
|
||||
assert m._classify_resource_type("cloud_run_revision") == "cloud_run_service"
|
||||
|
||||
def test_cloud_run_job_mapped(self):
|
||||
m = _import_monitor_module()
|
||||
assert m._classify_resource_type("cloud_run_job") == "cloud_run_job"
|
||||
|
||||
def test_cloud_function_mapped(self):
|
||||
m = _import_monitor_module()
|
||||
assert m._classify_resource_type("cloud_function") == "cloud_function"
|
||||
|
||||
def test_gce_instance_mapped(self):
|
||||
m = _import_monitor_module()
|
||||
assert m._classify_resource_type("gce_instance") == "gce_instance"
|
||||
|
||||
def test_unknown_type_returns_original_or_unknown(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._classify_resource_type("unknown_type")
|
||||
assert isinstance(result, str)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _group_by_pattern (via _run_monitor_cycle logic)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGroupByPattern:
|
||||
"""_group_by_pattern should correctly group entries by hash and count duplicates."""
|
||||
|
||||
def _make_monitor(self):
|
||||
"""Create an ErrorMonitor instance without calling __init__ (no GCP calls)."""
|
||||
ErrorMonitor = _import_error_monitor_class()
|
||||
monitor = ErrorMonitor.__new__(ErrorMonitor)
|
||||
monitor.logging_client = MagicMock()
|
||||
monitor.firestore_adapter = MagicMock()
|
||||
monitor.discord = MagicMock()
|
||||
return monitor
|
||||
|
||||
def test_groups_entries_by_hash(self):
|
||||
monitor = self._make_monitor()
|
||||
m = _import_monitor_module()
|
||||
|
||||
entries = [
|
||||
{"service": "svc-a", "normalized": "Connection refused", "message": "msg1"},
|
||||
{"service": "svc-a", "normalized": "Connection refused", "message": "msg2"},
|
||||
{"service": "svc-b", "normalized": "Timeout error", "message": "msg3"},
|
||||
]
|
||||
result = m._group_by_pattern(entries)
|
||||
# 2 unique patterns
|
||||
assert len(result) == 2
|
||||
|
||||
def test_counts_duplicates_correctly(self):
|
||||
monitor = self._make_monitor()
|
||||
m = _import_monitor_module()
|
||||
|
||||
entries = [
|
||||
{"service": "svc-a", "normalized": "Connection refused", "message": "msg1"},
|
||||
{"service": "svc-a", "normalized": "Connection refused", "message": "msg2"},
|
||||
{"service": "svc-a", "normalized": "Connection refused", "message": "msg3"},
|
||||
]
|
||||
result = m._group_by_pattern(entries)
|
||||
assert len(result) == 1
|
||||
group = list(result.values())[0]
|
||||
assert group["count"] == 3
|
||||
|
||||
def test_each_group_has_sample_message(self):
|
||||
m = _import_monitor_module()
|
||||
|
||||
entries = [
|
||||
{"service": "svc-a", "normalized": "DB error", "message": "DB error at line 42"},
|
||||
]
|
||||
result = m._group_by_pattern(entries)
|
||||
group = list(result.values())[0]
|
||||
assert "sample_message" in group
|
||||
assert group["sample_message"] == "DB error at line 42"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _is_spike
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsSpike:
|
||||
"""_is_spike should detect spikes based on multiplier and minimum count."""
|
||||
|
||||
def _make_monitor(self):
|
||||
ErrorMonitor = _import_error_monitor_class()
|
||||
monitor = ErrorMonitor.__new__(ErrorMonitor)
|
||||
monitor.logging_client = MagicMock()
|
||||
monitor.firestore_adapter = MagicMock()
|
||||
monitor.discord = MagicMock()
|
||||
return monitor
|
||||
|
||||
def test_detects_spike_when_count_exceeds_multiplier_and_min(self):
|
||||
m = _import_monitor_module()
|
||||
# rolling_counts of 1 each for the past 5 periods, current count = 10
|
||||
# avg = 1.0, SPIKE_MULTIPLIER=5.0, so 10 > 5 and 10 >= SPIKE_MIN_COUNT=5
|
||||
rolling_counts = [{"count": 1}, {"count": 1}, {"count": 1}, {"count": 1}, {"count": 1}]
|
||||
assert m._is_spike(current_count=10, rolling_counts=rolling_counts) is True
|
||||
|
||||
def test_rejects_non_spike_count(self):
|
||||
m = _import_monitor_module()
|
||||
rolling_counts = [{"count": 4}, {"count": 4}, {"count": 4}, {"count": 4}]
|
||||
# avg = 4.0, 5x = 20, current=5 → not a spike
|
||||
assert m._is_spike(current_count=5, rolling_counts=rolling_counts) is False
|
||||
|
||||
def test_rejects_spike_below_min_count(self):
|
||||
m = _import_monitor_module()
|
||||
# avg = 0.5, 5x = 2.5, current = 3 → exceeds multiplier but below SPIKE_MIN_COUNT
|
||||
rolling_counts = [{"count": 1}, {"count": 0}]
|
||||
assert m._is_spike(current_count=3, rolling_counts=rolling_counts) is False
|
||||
|
||||
def test_empty_rolling_counts_not_a_spike(self):
|
||||
m = _import_monitor_module()
|
||||
assert m._is_spike(current_count=100, rolling_counts=[]) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _rolling_average
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRollingAverage:
|
||||
"""_rolling_average should compute mean of rolling_counts."""
|
||||
|
||||
def test_computes_average_correctly(self):
|
||||
m = _import_monitor_module()
|
||||
rolling = [{"count": 2}, {"count": 4}, {"count": 6}]
|
||||
result = m._rolling_average(rolling)
|
||||
assert result == pytest.approx(4.0)
|
||||
|
||||
def test_single_entry_returns_that_value(self):
|
||||
m = _import_monitor_module()
|
||||
rolling = [{"count": 7}]
|
||||
result = m._rolling_average(rolling)
|
||||
assert result == pytest.approx(7.0)
|
||||
|
||||
def test_empty_list_returns_zero(self):
|
||||
m = _import_monitor_module()
|
||||
result = m._rolling_average([])
|
||||
assert result == 0.0
|
||||
|
||||
def test_counts_with_zeros(self):
|
||||
m = _import_monitor_module()
|
||||
rolling = [{"count": 0}, {"count": 0}, {"count": 10}]
|
||||
result = m._rolling_average(rolling)
|
||||
assert result == pytest.approx(10 / 3)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _run_monitor_cycle end-to-end
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_monitor_with_mocks():
|
||||
"""Return an ErrorMonitor with all external dependencies mocked."""
|
||||
ErrorMonitor = _import_error_monitor_class()
|
||||
monitor = ErrorMonitor.__new__(ErrorMonitor)
|
||||
monitor.logging_client = MagicMock()
|
||||
monitor.firestore_adapter = MagicMock()
|
||||
monitor.discord = MagicMock()
|
||||
return monitor
|
||||
|
||||
|
||||
class TestRunMonitorCycleNoErrors:
|
||||
"""When no error entries are found, auto-resolve check should still run."""
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_auto_resolve_called_even_when_no_errors(self, mock_digest, mock_llm):
|
||||
monitor = _make_monitor_with_mocks()
|
||||
|
||||
# Cloud Logging returns empty results for all resource types
|
||||
monitor.logging_client.list_entries.return_value = iter([])
|
||||
|
||||
# Firestore adapter returns no patterns eligible for auto-resolve
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
# Auto-resolve check must always run
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.assert_called_once()
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_no_discord_sent_when_no_errors(self, mock_digest, mock_llm):
|
||||
monitor = _make_monitor_with_mocks()
|
||||
|
||||
monitor.logging_client.list_entries.return_value = iter([])
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
|
||||
|
||||
class TestRunMonitorCycleOneNewError:
|
||||
"""When one new error pattern is found, an individual alert should be sent."""
|
||||
|
||||
def _make_log_entry(self):
|
||||
entry = MagicMock()
|
||||
entry.payload = "RuntimeError: connection refused"
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.type = "cloud_run_revision"
|
||||
entry.resource.labels = {"service_name": "karaoke-backend"}
|
||||
entry.labels = {}
|
||||
return entry
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_individual_alert_sent_for_new_pattern(self, mock_digest, mock_llm, mock_ignore):
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
|
||||
log_entry = self._make_log_entry()
|
||||
monitor.logging_client.list_entries.return_value = iter([log_entry])
|
||||
|
||||
# Upsert returns is_new=True for first encounter
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(
|
||||
pattern_id="abc123", is_new=True
|
||||
)
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
# An alert should have been sent
|
||||
assert monitor.discord.send_message.call_count >= 1
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_upsert_called_for_each_pattern_group(self, mock_digest, mock_llm, mock_ignore):
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
|
||||
log_entry = self._make_log_entry()
|
||||
monitor.logging_client.list_entries.return_value = iter([log_entry])
|
||||
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(
|
||||
pattern_id="abc123", is_new=True
|
||||
)
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
assert monitor.firestore_adapter.upsert_pattern.call_count >= 1
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_known_issue_skipped(self, mock_digest, mock_llm, mock_ignore):
|
||||
"""Entries matched by should_ignore should not be upserted or alerted."""
|
||||
from backend.services.error_monitor.known_issues import IgnoreReason
|
||||
|
||||
# Override: should_ignore returns a reason (i.e., ignore this entry)
|
||||
mock_ignore.return_value = IgnoreReason(
|
||||
pattern_name="startup_probe",
|
||||
reason="Cloud Run cold start — startup probe failure is transient",
|
||||
)
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
log_entry = self._make_log_entry()
|
||||
monitor.logging_client.list_entries.return_value = iter([log_entry])
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
monitor.firestore_adapter.upsert_pattern.assert_not_called()
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _run_daily_digest (P0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRunDailyDigest:
|
||||
"""_run_daily_digest should fetch active patterns and send a digest to Discord."""
|
||||
|
||||
@pytest.fixture
|
||||
def monitor(self):
|
||||
ErrorMonitor = _import_error_monitor_class()
|
||||
m = ErrorMonitor.__new__(ErrorMonitor)
|
||||
m.logging_client = MagicMock()
|
||||
m.firestore_adapter = MagicMock()
|
||||
m.discord = MagicMock()
|
||||
m.discord.messages_sent = 0
|
||||
m.logger = MagicMock()
|
||||
return m
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
def test_digest_sends_message_with_chart_emoji(self, mock_llm, monitor):
|
||||
"""_run_daily_digest should send a message containing the 📊 emoji."""
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = [
|
||||
{"service": "karaoke-backend", "total_count": 10, "status": "new"},
|
||||
{"service": "video-worker", "total_count": 5, "status": "acknowledged"},
|
||||
]
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert "📊" in sent_content
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
def test_digest_includes_per_service_counts(self, mock_llm, monitor):
|
||||
"""_run_daily_digest should aggregate counts per service."""
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = [
|
||||
{"service": "svc-alpha", "total_count": 7, "status": "new"},
|
||||
{"service": "svc-alpha", "total_count": 3, "status": "acknowledged"},
|
||||
{"service": "svc-beta", "total_count": 2, "status": "new"},
|
||||
]
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
# Both services should be mentioned in the digest
|
||||
assert "svc-alpha" in sent_content
|
||||
assert "svc-beta" in sent_content
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
def test_digest_with_no_active_patterns(self, mock_llm, monitor):
|
||||
"""_run_daily_digest should still send a digest even when there are no patterns."""
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert "📊" in sent_content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: PatternData construction (P0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPatternDataConstruction:
|
||||
"""upsert_pattern should be called with a PatternData object with correct field types."""
|
||||
|
||||
def _make_log_entry(self, message="RuntimeError: disk full", service="karaoke-backend"):
|
||||
entry = MagicMock()
|
||||
entry.payload = message
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.type = "cloud_run_revision"
|
||||
entry.resource.labels = {"service_name": service}
|
||||
entry.labels = {}
|
||||
return entry
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_pattern_data_fields_have_correct_types(self, mock_digest, mock_llm, mock_ignore):
|
||||
"""upsert_pattern receives a PatternData with correctly typed fields."""
|
||||
from backend.services.error_monitor.firestore_adapter import PatternData, UpsertResult
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
log_entry = self._make_log_entry()
|
||||
monitor.logging_client.list_entries.return_value = iter([log_entry])
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(
|
||||
pattern_id="xyz789", is_new=True
|
||||
)
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
assert monitor.firestore_adapter.upsert_pattern.call_count >= 1
|
||||
call_arg = monitor.firestore_adapter.upsert_pattern.call_args[0][0]
|
||||
assert isinstance(call_arg, PatternData)
|
||||
|
||||
# pattern_id should be a non-empty hex string
|
||||
assert isinstance(call_arg.pattern_id, str)
|
||||
assert len(call_arg.pattern_id) > 0
|
||||
# All characters should be valid hex (the hash function produces hex)
|
||||
assert all(c in "0123456789abcdef" for c in call_arg.pattern_id)
|
||||
|
||||
# service must match the log entry's service label
|
||||
assert call_arg.service == "karaoke-backend"
|
||||
|
||||
# resource_type maps to canonical string
|
||||
assert isinstance(call_arg.resource_type, str)
|
||||
assert len(call_arg.resource_type) > 0
|
||||
|
||||
# normalized_message and sample_message must be non-empty strings
|
||||
assert isinstance(call_arg.normalized_message, str)
|
||||
assert len(call_arg.normalized_message) > 0
|
||||
assert isinstance(call_arg.sample_message, str)
|
||||
assert len(call_arg.sample_message) > 0
|
||||
|
||||
# count is a positive int
|
||||
assert isinstance(call_arg.count, int)
|
||||
assert call_arg.count >= 1
|
||||
|
||||
# timestamp is a datetime
|
||||
assert isinstance(call_arg.timestamp, datetime)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Discord alert argument verification (P0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDiscordAlertContent:
|
||||
"""The Discord alert message should contain the service name and normalized message."""
|
||||
|
||||
def _make_log_entry(self, message, service):
|
||||
entry = MagicMock()
|
||||
entry.payload = message
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.type = "cloud_run_revision"
|
||||
entry.resource.labels = {"service_name": service}
|
||||
entry.labels = {}
|
||||
return entry
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_new_pattern_alert_contains_service_name(self, mock_digest, mock_llm, mock_ignore):
|
||||
"""The alert message sent to Discord should contain the originating service name."""
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
service_name = "my-special-service"
|
||||
monitor = _make_monitor_with_mocks()
|
||||
log_entry = self._make_log_entry("RuntimeError: boom", service=service_name)
|
||||
monitor.logging_client.list_entries.return_value = iter([log_entry])
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(
|
||||
pattern_id="aabbcc", is_new=True
|
||||
)
|
||||
monitor.firestore_adapter.get_pattern.return_value = {"first_seen": "2026-01-01T00:00:00Z"}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
assert monitor.discord.send_message.call_count >= 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert service_name in sent_content
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_new_pattern_alert_contains_normalized_message(
|
||||
self, mock_digest, mock_llm, mock_ignore
|
||||
):
|
||||
"""The alert message should contain a recognizable part of the normalized error."""
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
# Use a distinctive error phrase that should survive normalization
|
||||
log_entry = self._make_log_entry(
|
||||
"ConnectionError: database connection refused", service="karaoke-backend"
|
||||
)
|
||||
monitor.logging_client.list_entries.return_value = iter([log_entry])
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(
|
||||
pattern_id="def456", is_new=True
|
||||
)
|
||||
monitor.firestore_adapter.get_pattern.return_value = {"first_seen": "2026-01-01T00:00:00Z"}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
assert monitor.discord.send_message.call_count >= 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
# The normalized message should contain at least "connection" or "refused"
|
||||
assert "connection" in sent_content.lower() or "refused" in sent_content.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: analyze_patterns input format (P0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAnalyzePatternsInputFormat:
|
||||
"""When LLM is enabled, analyze_patterns is called with correctly shaped dicts."""
|
||||
|
||||
def _make_log_entry(self, message, service):
|
||||
entry = MagicMock()
|
||||
entry.payload = message
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.type = "cloud_run_revision"
|
||||
entry.resource.labels = {"service_name": service}
|
||||
entry.labels = {}
|
||||
return entry
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.analyze_patterns")
|
||||
@patch("backend.services.error_monitor.monitor.find_duplicate_patterns")
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=True)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_analyze_patterns_receives_correct_keys(
|
||||
self, mock_digest, mock_llm, mock_ignore, mock_find_dups, mock_analyze
|
||||
):
|
||||
"""analyze_patterns should receive a list of dicts with service, normalized_message, count."""
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
mock_analyze.return_value = MagicMock(incidents=[])
|
||||
mock_find_dups.return_value = []
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
|
||||
# Create 2 distinct log entries so LLM dedup + analysis paths are triggered
|
||||
entry1 = self._make_log_entry("RuntimeError: disk full", service="svc-one")
|
||||
entry2 = self._make_log_entry("ValueError: invalid input", service="svc-two")
|
||||
monitor.logging_client.list_entries.side_effect = [
|
||||
iter([entry1, entry2]),
|
||||
iter([]),
|
||||
iter([]),
|
||||
iter([]),
|
||||
]
|
||||
|
||||
call_count = [0]
|
||||
|
||||
def upsert_side_effect(data):
|
||||
call_count[0] += 1
|
||||
return UpsertResult(pattern_id=data.pattern_id, is_new=True)
|
||||
|
||||
monitor.firestore_adapter.upsert_pattern.side_effect = upsert_side_effect
|
||||
monitor.firestore_adapter.get_pattern.return_value = {"first_seen": "2026-01-01T00:00:00Z"}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
# analyze_patterns should have been called once both new patterns exist
|
||||
assert mock_analyze.call_count >= 1
|
||||
call_args = mock_analyze.call_args[0][0]
|
||||
assert isinstance(call_args, list)
|
||||
assert len(call_args) >= 1
|
||||
|
||||
for item in call_args:
|
||||
assert "service" in item, f"Missing 'service' key in {item}"
|
||||
assert "normalized_message" in item, f"Missing 'normalized_message' key in {item}"
|
||||
assert "count" in item, f"Missing 'count' key in {item}"
|
||||
assert isinstance(item["service"], str)
|
||||
assert isinstance(item["normalized_message"], str)
|
||||
assert isinstance(item["count"], int)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Spike detection flow (P1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSpikeDetectionFlow:
|
||||
"""_run_monitor_cycle should send a spike alert when counts exceed the threshold."""
|
||||
|
||||
def _make_log_entry(self, message="RuntimeError: timeout", service="karaoke-backend"):
|
||||
entry = MagicMock()
|
||||
entry.payload = message
|
||||
entry.resource = MagicMock()
|
||||
entry.resource.type = "cloud_run_revision"
|
||||
entry.resource.labels = {"service_name": service}
|
||||
entry.labels = {}
|
||||
return entry
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.should_ignore", return_value=None)
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False)
|
||||
def test_spike_alert_sent_when_count_exceeds_threshold(
|
||||
self, mock_digest, mock_llm, mock_ignore
|
||||
):
|
||||
"""A spike alert (⚠️) is sent when current count is >5x rolling average and >= min."""
|
||||
from backend.services.error_monitor.firestore_adapter import UpsertResult
|
||||
|
||||
monitor = _make_monitor_with_mocks()
|
||||
|
||||
# Send 10 identical entries to make count = 10 (> SPIKE_MIN_COUNT=5)
|
||||
entries = [self._make_log_entry() for _ in range(10)]
|
||||
monitor.logging_client.list_entries.return_value = iter(entries)
|
||||
|
||||
# Upsert returns is_new=False (existing pattern) so spike check runs
|
||||
monitor.firestore_adapter.upsert_pattern.return_value = UpsertResult(
|
||||
pattern_id="spike-pattern-1", is_new=False
|
||||
)
|
||||
|
||||
# get_pattern returns a pattern with rolling_counts that average to 1
|
||||
# avg=1.0, 5x=5.0, current_count=10 → spike detected
|
||||
monitor.firestore_adapter.get_pattern.return_value = {
|
||||
"pattern_id": "spike-pattern-1",
|
||||
"rolling_counts": [{"count": 1}, {"count": 1}, {"count": 1}, {"count": 1}],
|
||||
}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
# At least one Discord message should have been sent
|
||||
assert monitor.discord.send_message.call_count >= 1
|
||||
|
||||
# At least one of the messages should be a spike alert (⚠️)
|
||||
all_calls = monitor.discord.send_message.call_args_list
|
||||
spike_messages = [c for c in all_calls if "⚠️" in c[0][0]]
|
||||
assert len(spike_messages) >= 1, (
|
||||
"Expected at least one spike alert (⚠️) but none found. "
|
||||
f"Messages sent: {[c[0][0][:80] for c in all_calls]}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Auto-resolve alert (P1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoResolveAlert:
|
||||
"""_check_auto_resolve should resolve eligible patterns and send a ✅ alert."""
|
||||
|
||||
@pytest.fixture
|
||||
def monitor(self):
|
||||
ErrorMonitor = _import_error_monitor_class()
|
||||
m = ErrorMonitor.__new__(ErrorMonitor)
|
||||
m.logging_client = MagicMock()
|
||||
m.firestore_adapter = MagicMock()
|
||||
m.discord = MagicMock()
|
||||
m.discord.messages_sent = 0
|
||||
m.logger = MagicMock()
|
||||
return m
|
||||
|
||||
def test_auto_resolve_calls_resolve_and_sends_alert(self, monitor):
|
||||
"""When a pattern qualifies for auto-resolve, the pattern is resolved and ✅ is sent."""
|
||||
pattern = {
|
||||
"pattern_id": "resolve-me-123",
|
||||
"service": "karaoke-backend",
|
||||
"normalized_message": "Connection refused",
|
||||
"status": "new",
|
||||
}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern]
|
||||
monitor.firestore_adapter.check_auto_resolve.return_value = 25.5 # hours silent
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._check_auto_resolve()
|
||||
|
||||
# auto_resolve_pattern must have been called with the correct pattern_id
|
||||
monitor.firestore_adapter.auto_resolve_pattern.assert_called_once_with(
|
||||
"resolve-me-123", 25.5
|
||||
)
|
||||
|
||||
# A Discord message must have been sent
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert "✅" in sent_content
|
||||
|
||||
def test_auto_resolve_no_alert_when_no_patterns_qualify(self, monitor):
|
||||
"""When no patterns qualify for auto-resolve, no Discord alert is sent."""
|
||||
pattern = {
|
||||
"pattern_id": "still-active",
|
||||
"service": "svc-x",
|
||||
"normalized_message": "Disk full",
|
||||
"status": "acknowledged",
|
||||
}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern]
|
||||
# check_auto_resolve returns None → pattern does not qualify yet
|
||||
monitor.firestore_adapter.check_auto_resolve.return_value = None
|
||||
|
||||
monitor._check_auto_resolve()
|
||||
|
||||
monitor.firestore_adapter.auto_resolve_pattern.assert_not_called()
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
|
||||
def test_auto_resolve_via_monitor_cycle(self, monitor):
|
||||
"""_run_monitor_cycle with no errors should still trigger the auto-resolve check."""
|
||||
with patch(
|
||||
"backend.services.error_monitor.monitor.get_llm_enabled", return_value=False
|
||||
), patch("backend.services.error_monitor.monitor.get_digest_mode", return_value=False):
|
||||
monitor.logging_client.list_entries.return_value = iter([])
|
||||
|
||||
pattern = {
|
||||
"pattern_id": "cycle-resolve",
|
||||
"service": "bg-worker",
|
||||
"normalized_message": "queue timeout",
|
||||
"status": "new",
|
||||
}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern]
|
||||
monitor.firestore_adapter.check_auto_resolve.return_value = 12.0
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
monitor.firestore_adapter.auto_resolve_pattern.assert_called_once()
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert "✅" in sent_content
|
||||
@@ -0,0 +1,415 @@
|
||||
"""Tests for error_monitor.normalizer module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestUUIDReplacement:
|
||||
"""UUID patterns should be replaced with <ID>."""
|
||||
|
||||
def test_uuid_v4_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Job failed for job_id=550e8400-e29b-41d4-a716-446655440000"
|
||||
result = normalize_message(msg)
|
||||
assert "550e8400-e29b-41d4-a716-446655440000" not in result
|
||||
assert "<ID>" in result
|
||||
|
||||
def test_uuid_uppercase_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Error in request 550E8400-E29B-41D4-A716-446655440000"
|
||||
result = normalize_message(msg)
|
||||
assert "550E8400-E29B-41D4-A716-446655440000" not in result
|
||||
assert "<ID>" in result
|
||||
|
||||
def test_multiple_uuids_all_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Copy from 550e8400-e29b-41d4-a716-446655440000 to 660f9511-f30c-52e5-b827-557766551111"
|
||||
result = normalize_message(msg)
|
||||
assert "550e8400" not in result
|
||||
assert "660f9511" not in result
|
||||
|
||||
|
||||
class TestISOTimestampReplacement:
|
||||
"""ISO 8601 timestamps should be replaced with <TS>."""
|
||||
|
||||
def test_iso_datetime_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Error occurred at 2024-01-15T10:30:45Z"
|
||||
result = normalize_message(msg)
|
||||
assert "2024-01-15T10:30:45Z" not in result
|
||||
assert "<TS>" in result
|
||||
|
||||
def test_iso_datetime_with_ms_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Timeout at 2024-03-22T14:55:01.123Z"
|
||||
result = normalize_message(msg)
|
||||
assert "2024-03-22T14:55:01.123Z" not in result
|
||||
assert "<TS>" in result
|
||||
|
||||
def test_iso_datetime_with_offset_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Created at 2024-06-01T08:00:00+05:30"
|
||||
result = normalize_message(msg)
|
||||
assert "2024-06-01T08:00:00+05:30" not in result
|
||||
assert "<TS>" in result
|
||||
|
||||
|
||||
class TestEpochTimestampReplacement:
|
||||
"""Epoch timestamps (10+ digit floats) should be replaced with <EPOCH>."""
|
||||
|
||||
def test_epoch_float_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Received at t=1711234567.891234"
|
||||
result = normalize_message(msg)
|
||||
assert "1711234567.891234" not in result
|
||||
assert "<EPOCH>" in result
|
||||
|
||||
def test_epoch_integer_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "timestamp=1711234567890"
|
||||
result = normalize_message(msg)
|
||||
assert "1711234567890" not in result
|
||||
assert "<EPOCH>" in result
|
||||
|
||||
def test_small_number_not_epoch(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
# 4-digit number should become <NUM>, not <EPOCH>
|
||||
msg = "Status code 404 returned"
|
||||
result = normalize_message(msg)
|
||||
assert "<EPOCH>" not in result
|
||||
|
||||
|
||||
class TestEmailReplacement:
|
||||
"""Email addresses should be replaced with <EMAIL>."""
|
||||
|
||||
def test_simple_email_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "User user@example.com not found"
|
||||
result = normalize_message(msg)
|
||||
assert "user@example.com" not in result
|
||||
assert "<EMAIL>" in result
|
||||
|
||||
def test_email_with_subdomain_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Sent to andrew@api.nomadkaraoke.com"
|
||||
result = normalize_message(msg)
|
||||
assert "andrew@api.nomadkaraoke.com" not in result
|
||||
assert "<EMAIL>" in result
|
||||
|
||||
def test_email_with_plus_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Contact for test+filter@gmail.com"
|
||||
result = normalize_message(msg)
|
||||
assert "test+filter@gmail.com" not in result
|
||||
assert "<EMAIL>" in result
|
||||
|
||||
|
||||
class TestIPAddressReplacement:
|
||||
"""IP addresses should be replaced with <IP>."""
|
||||
|
||||
def test_ipv4_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Connection refused from 192.168.1.100"
|
||||
result = normalize_message(msg)
|
||||
assert "192.168.1.100" not in result
|
||||
assert "<IP>" in result
|
||||
|
||||
def test_ipv4_with_port_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Failed to connect to 10.0.0.1:8080"
|
||||
result = normalize_message(msg)
|
||||
assert "10.0.0.1" not in result
|
||||
assert "<IP>" in result
|
||||
|
||||
def test_multiple_ips_all_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Route from 10.0.0.1 to 172.16.0.254 failed"
|
||||
result = normalize_message(msg)
|
||||
assert "10.0.0.1" not in result
|
||||
assert "172.16.0.254" not in result
|
||||
|
||||
|
||||
class TestHexIDReplacement:
|
||||
"""Hex/alphanumeric IDs (8+ chars with both digits and letters) should be replaced with <ID>."""
|
||||
|
||||
def test_hex_id_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Resource abc12345def not found"
|
||||
result = normalize_message(msg)
|
||||
assert "abc12345def" not in result
|
||||
assert "<ID>" in result
|
||||
|
||||
def test_pure_alpha_word_not_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
# Pure alphabetic strings should not be replaced
|
||||
msg = "Authentication failed completely"
|
||||
result = normalize_message(msg)
|
||||
assert result == "Authentication failed completely"
|
||||
|
||||
def test_short_id_not_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
# IDs shorter than 8 chars should not be replaced
|
||||
msg = "Error in abc123 handler"
|
||||
result = normalize_message(msg)
|
||||
# Should not replace short mixed alphanum under 8 chars as <ID>
|
||||
assert "abc123" in result
|
||||
|
||||
|
||||
class TestLargeNumberReplacement:
|
||||
"""Numbers with 4+ digits should be replaced with <NUM>."""
|
||||
|
||||
def test_four_digit_number_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "HTTP error code 5000 encountered"
|
||||
result = normalize_message(msg)
|
||||
assert "5000" not in result
|
||||
assert "<NUM>" in result
|
||||
|
||||
def test_large_number_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Allocated 12345 bytes"
|
||||
result = normalize_message(msg)
|
||||
assert "12345" not in result
|
||||
assert "<NUM>" in result
|
||||
|
||||
def test_small_number_preserved(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
# 3-digit numbers like HTTP status codes should be preserved
|
||||
msg = "HTTP 404 Not Found"
|
||||
result = normalize_message(msg)
|
||||
assert "404" in result
|
||||
|
||||
def test_two_digit_number_preserved(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Retry attempt 42"
|
||||
result = normalize_message(msg)
|
||||
assert "42" in result
|
||||
|
||||
|
||||
class TestJobIDReplacement:
|
||||
"""Job IDs in paths should be replaced."""
|
||||
|
||||
def test_job_id_in_path_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Failed processing /jobs/abc123def456 in queue"
|
||||
result = normalize_message(msg)
|
||||
assert "abc123def456" not in result
|
||||
assert "/jobs/<ID>" in result
|
||||
|
||||
def test_job_id_alphanumeric_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Job at /jobs/XyZ9a1b2C3d4 timed out"
|
||||
result = normalize_message(msg)
|
||||
assert "XyZ9a1b2C3d4" not in result
|
||||
assert "/jobs/<ID>" in result
|
||||
|
||||
|
||||
class TestGCSPathReplacement:
|
||||
"""GCS paths should be normalized."""
|
||||
|
||||
def test_gcs_path_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Could not read gs://my-bucket/path/to/file.mp4"
|
||||
result = normalize_message(msg)
|
||||
assert "gs://my-bucket/path/to/file.mp4" not in result
|
||||
assert "gs://<BUCKET>/<PATH>" in result
|
||||
|
||||
def test_gcs_path_without_object_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Bucket access denied: gs://nomadkaraoke-prod-videos/"
|
||||
result = normalize_message(msg)
|
||||
assert "gs://nomadkaraoke-prod-videos/" not in result
|
||||
assert "gs://<BUCKET>" in result
|
||||
|
||||
def test_gcs_path_with_nested_path_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "File gs://nomad-audio/jobs/abc123/output/stem.wav not found"
|
||||
result = normalize_message(msg)
|
||||
assert "gs://nomad-audio/jobs/abc123/output/stem.wav" not in result
|
||||
assert "gs://<BUCKET>/<PATH>" in result
|
||||
|
||||
|
||||
class TestFirebaseUIDReplacement:
|
||||
"""Firebase UIDs (20-28 mixed-case alphanumeric) should be replaced with <UID>."""
|
||||
|
||||
def test_firebase_uid_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
# Firebase UIDs are 28 chars: letters and digits, mixed case
|
||||
msg = "User uid=ABC123defGHI456jklMNO7 not authorized"
|
||||
result = normalize_message(msg)
|
||||
assert "ABC123defGHI456jklMNO7" not in result
|
||||
assert "<UID>" in result
|
||||
|
||||
def test_firebase_uid_28_chars_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Auth failed for 8JkLmNoPqRsT1uVwXyZ23456AB"
|
||||
result = normalize_message(msg)
|
||||
assert "8JkLmNoPqRsT1uVwXyZ23456AB" not in result
|
||||
assert "<UID>" in result
|
||||
|
||||
|
||||
class TestFirestoreDocPathReplacement:
|
||||
"""Firestore document paths should be replaced with <DOC_PATH>."""
|
||||
|
||||
def test_firestore_doc_path_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Document jobs/abc123def/state_data/current does not exist"
|
||||
result = normalize_message(msg)
|
||||
assert "jobs/abc123def/state_data/current" not in result
|
||||
assert "<DOC_PATH>" in result
|
||||
|
||||
def test_firestore_users_path_replaced(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Permission denied for users/uid12345/profile/settings"
|
||||
result = normalize_message(msg)
|
||||
assert "users/uid12345/profile/settings" not in result
|
||||
assert "<DOC_PATH>" in result
|
||||
|
||||
|
||||
class TestURLNormalization:
|
||||
"""URLs should be stripped to domain only."""
|
||||
|
||||
def test_url_stripped_to_domain(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Failed to fetch https://api.example.com/v1/users/12345/data"
|
||||
result = normalize_message(msg)
|
||||
assert "/v1/users/12345/data" not in result
|
||||
assert "api.example.com" in result
|
||||
|
||||
def test_http_url_stripped(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Redirect from http://old.example.com/path?q=abc123"
|
||||
result = normalize_message(msg)
|
||||
assert "/path?q=abc123" not in result
|
||||
assert "old.example.com" in result
|
||||
|
||||
def test_url_with_port_domain_preserved(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Connection to https://internal.service.com:8080/api/endpoint failed"
|
||||
result = normalize_message(msg)
|
||||
assert "internal.service.com" in result
|
||||
|
||||
|
||||
class TestTruncation:
|
||||
"""Long messages should be truncated to MAX_NORMALIZED_MESSAGE_LENGTH (200 chars)."""
|
||||
|
||||
def test_long_message_truncated(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH
|
||||
|
||||
long_msg = "Error: " + "x" * 300
|
||||
result = normalize_message(long_msg)
|
||||
assert len(result) <= MAX_NORMALIZED_MESSAGE_LENGTH
|
||||
|
||||
def test_short_message_not_truncated(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Short error message"
|
||||
result = normalize_message(msg)
|
||||
assert result == msg
|
||||
|
||||
def test_exactly_200_chars_not_truncated(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "A" * 200
|
||||
result = normalize_message(msg)
|
||||
assert len(result) == 200
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
"""Edge cases for normalize_message."""
|
||||
|
||||
def test_empty_message_returns_empty_string(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
result = normalize_message("")
|
||||
assert result == ""
|
||||
|
||||
def test_no_replacements_returns_original(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Connection refused by server"
|
||||
result = normalize_message(msg)
|
||||
assert result == msg
|
||||
|
||||
def test_deterministic_same_input_same_output(self):
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
msg = "Job failed for 550e8400-e29b-41d4-a716-446655440000 at 2024-01-15T10:30:45Z"
|
||||
result1 = normalize_message(msg)
|
||||
result2 = normalize_message(msg)
|
||||
assert result1 == result2
|
||||
|
||||
|
||||
class TestComputePatternHash:
|
||||
"""Tests for the compute_pattern_hash function."""
|
||||
|
||||
def test_returns_64_char_hex(self):
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash
|
||||
|
||||
result = compute_pattern_hash("karaoke-backend", "Connection refused by server")
|
||||
assert isinstance(result, str)
|
||||
assert len(result) == 64
|
||||
# SHA-256 hex is lowercase hex
|
||||
assert all(c in "0123456789abcdef" for c in result)
|
||||
|
||||
def test_same_inputs_same_hash(self):
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash
|
||||
|
||||
h1 = compute_pattern_hash("karaoke-backend", "Connection refused by server")
|
||||
h2 = compute_pattern_hash("karaoke-backend", "Connection refused by server")
|
||||
assert h1 == h2
|
||||
|
||||
def test_different_services_different_hash(self):
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash
|
||||
|
||||
h1 = compute_pattern_hash("karaoke-backend", "Connection refused by server")
|
||||
h2 = compute_pattern_hash("audio-separator", "Connection refused by server")
|
||||
assert h1 != h2
|
||||
|
||||
def test_different_messages_different_hash(self):
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash
|
||||
|
||||
h1 = compute_pattern_hash("karaoke-backend", "Connection refused by server")
|
||||
h2 = compute_pattern_hash("karaoke-backend", "Timeout waiting for response")
|
||||
assert h1 != h2
|
||||
|
||||
def test_empty_service_and_message(self):
|
||||
from backend.services.error_monitor.normalizer import compute_pattern_hash
|
||||
|
||||
result = compute_pattern_hash("", "")
|
||||
assert isinstance(result, str)
|
||||
assert len(result) == 64
|
||||
Reference in new issue
Block a user