mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
107 lines
3.5 KiB
Python
107 lines
3.5 KiB
Python
"""Known infrastructure noise patterns for the production error monitoring service.
|
|
|
|
Filters out transient, expected log messages so the error monitor is not flooded
|
|
with Cloud Run cold-start probes, deploy events, spot VM preemptions, and similar
|
|
operational noise.
|
|
|
|
Usage:
|
|
from backend.services.error_monitor.known_issues import should_ignore
|
|
|
|
reason = should_ignore(service, message)
|
|
if reason:
|
|
# skip this log entry — it's known infrastructure noise
|
|
...
|
|
"""
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass
|
|
class IgnoreReason:
|
|
"""Describes why a log entry should be suppressed.
|
|
|
|
Attributes:
|
|
pattern_name: Machine-readable name of the matching ignore pattern.
|
|
reason: Human-readable explanation of why this is infrastructure noise.
|
|
"""
|
|
|
|
pattern_name: str
|
|
reason: str
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Compiled ignore patterns (applied in order)
|
|
# All patterns use re.IGNORECASE for case-insensitive matching.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_IGNORE_PATTERNS: list[tuple[str, re.Pattern, str]] = [
|
|
(
|
|
"startup_probe",
|
|
re.compile(r"startup probe failed", re.IGNORECASE),
|
|
"Cloud Run cold start — startup probe failure is transient and expected",
|
|
),
|
|
(
|
|
"ready_condition",
|
|
re.compile(r"ready condition status changed", re.IGNORECASE),
|
|
"Cloud Run deploy event — readiness condition change is expected on deploy",
|
|
),
|
|
(
|
|
"spot_preemption",
|
|
re.compile(r"instance was preempted", re.IGNORECASE),
|
|
"Spot/preemptible VM preemption — expected behaviour for spot instances",
|
|
),
|
|
(
|
|
"idle_shutdown",
|
|
re.compile(r"stopping idle encoding worker", re.IGNORECASE),
|
|
"Encoding worker idle shutdown — expected behaviour when no jobs are queued",
|
|
),
|
|
(
|
|
"health_check_404",
|
|
re.compile(
|
|
r"(?:GET|HEAD)\b[^\n]*?/health[^\n]*?\b(?:404|not found)\b",
|
|
re.IGNORECASE,
|
|
),
|
|
"Load balancer health check returning 404 — transient during cold start",
|
|
),
|
|
(
|
|
"scheduler_retry",
|
|
re.compile(r"cloud scheduler.*retry", re.IGNORECASE),
|
|
"Cloud Scheduler transient retry — expected on intermittent job failures",
|
|
),
|
|
(
|
|
"runner_startup",
|
|
re.compile(r"github\.runner\..*(?:starting|stopping|idle)", re.IGNORECASE),
|
|
"GitHub Actions runner lifecycle event — expected during runner start/stop",
|
|
),
|
|
(
|
|
"container_shutdown",
|
|
re.compile(r"container called exit\(0\)", re.IGNORECASE),
|
|
"Container clean exit — exit code 0 indicates a graceful, expected shutdown",
|
|
),
|
|
]
|
|
|
|
|
|
def should_ignore(service: str, message: str) -> IgnoreReason | None:
|
|
"""Check whether a log entry matches a known infrastructure noise pattern.
|
|
|
|
The check is purely message-based; ``service`` is accepted for API symmetry
|
|
and future service-specific overrides but does not gate any pattern today.
|
|
|
|
Args:
|
|
service: The originating service name (e.g. ``"karaoke-backend"``).
|
|
message: The raw log message string to evaluate.
|
|
|
|
Returns:
|
|
An :class:`IgnoreReason` if the message matches a known noise pattern,
|
|
or ``None`` if the message should be processed normally.
|
|
"""
|
|
if not message:
|
|
return None
|
|
|
|
for pattern_name, pattern, reason in _IGNORE_PATTERNS:
|
|
if pattern.search(message):
|
|
return IgnoreReason(pattern_name=pattern_name, reason=reason)
|
|
|
|
return None
|