mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat: production error monitor — log collection, LLM analysis, Discord alerts (#713)
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b847dba411
commit
9a5c9d497a
28 files changed
+11056
-2
No files matched your search
Executable
+193
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Query error patterns from Firestore.
|
||||
|
||||
Usage:
|
||||
python scripts/query-error-patterns.py # All active
|
||||
python scripts/query-error-patterns.py --status new # Only new
|
||||
python scripts/query-error-patterns.py --service karaoke-backend
|
||||
python scripts/query-error-patterns.py --hours 24 # Last 24h
|
||||
python scripts/query-error-patterns.py --json # JSON output
|
||||
|
||||
Requires:
|
||||
- google-cloud-firestore
|
||||
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
from google.cloud import firestore
|
||||
from google.cloud.firestore_v1 import FieldFilter
|
||||
|
||||
|
||||
ACTIVE_STATUSES = ["new", "acknowledged", "known"]
|
||||
|
||||
|
||||
def query_error_patterns(
|
||||
status: Optional[str] = None,
|
||||
service: Optional[str] = None,
|
||||
hours: Optional[int] = None,
|
||||
output_json: bool = False,
|
||||
project: str = "nomadkaraoke",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Query error patterns from Firestore.
|
||||
|
||||
Args:
|
||||
status: Filter by status (new/acknowledged/known/fixed). Default: active (new/acknowledged/known).
|
||||
service: Filter by service name.
|
||||
hours: Only return patterns last seen within this many hours.
|
||||
output_json: Print JSON output instead of a table.
|
||||
project: GCP project ID.
|
||||
|
||||
Returns:
|
||||
List of pattern dicts.
|
||||
"""
|
||||
db = firestore.Client(project=project)
|
||||
query = db.collection("error_patterns")
|
||||
|
||||
# Status filter
|
||||
if status:
|
||||
query = query.where(filter=FieldFilter("status", "==", status))
|
||||
else:
|
||||
query = query.where(filter=FieldFilter("status", "in", ACTIVE_STATUSES))
|
||||
|
||||
# Service filter
|
||||
if service:
|
||||
query = query.where(filter=FieldFilter("service", "==", service))
|
||||
|
||||
# Fetch all matching docs (streaming)
|
||||
patterns = []
|
||||
for doc in query.stream():
|
||||
data = doc.to_dict()
|
||||
data["_id"] = doc.id
|
||||
patterns.append(data)
|
||||
|
||||
# Hours filter (applied in Python — Firestore requires composite index for multi-field filters)
|
||||
if hours is not None:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
filtered = []
|
||||
for p in patterns:
|
||||
last_seen = p.get("last_seen")
|
||||
if last_seen is not None:
|
||||
# Firestore Timestamps have a .timestamp() method
|
||||
ts = last_seen.timestamp() if hasattr(last_seen, "timestamp") else float(last_seen)
|
||||
if ts >= cutoff.timestamp():
|
||||
filtered.append(p)
|
||||
patterns = filtered
|
||||
|
||||
# Sort by last_seen descending (most recent first)
|
||||
def sort_key(p):
|
||||
ls = p.get("last_seen")
|
||||
if ls is None:
|
||||
return 0.0
|
||||
return ls.timestamp() if hasattr(ls, "timestamp") else float(ls)
|
||||
|
||||
patterns.sort(key=sort_key, reverse=True)
|
||||
|
||||
if output_json:
|
||||
print(json.dumps(patterns, indent=2, default=str))
|
||||
else:
|
||||
_print_table(patterns)
|
||||
|
||||
return patterns
|
||||
|
||||
|
||||
def _fmt_last_seen(value) -> str:
|
||||
"""Return first 19 chars of ISO timestamp, or empty string."""
|
||||
if value is None:
|
||||
return ""
|
||||
if hasattr(value, "isoformat"):
|
||||
return value.isoformat()[:19]
|
||||
return str(value)[:19]
|
||||
|
||||
|
||||
def _print_table(patterns: list[dict[str, Any]]) -> None:
|
||||
"""Print patterns as a human-readable table."""
|
||||
if not patterns:
|
||||
print("No error patterns found.")
|
||||
return
|
||||
|
||||
# Column widths
|
||||
col_status = 14
|
||||
col_service = 24
|
||||
col_count = 6
|
||||
col_last_seen = 19
|
||||
col_message = 50
|
||||
|
||||
header = (
|
||||
f"{'Status':<{col_status}} "
|
||||
f"{'Service':<{col_service}} "
|
||||
f"{'Count':>{col_count}} "
|
||||
f"{'Last Seen':<{col_last_seen}} "
|
||||
f"{'Message':<{col_message}}"
|
||||
)
|
||||
separator = "-" * len(header)
|
||||
|
||||
print(separator)
|
||||
print(header)
|
||||
print(separator)
|
||||
|
||||
for p in patterns:
|
||||
status = str(p.get("status", ""))[:col_status]
|
||||
service = str(p.get("service", ""))[:col_service]
|
||||
count = p.get("total_count", p.get("occurrence_count", p.get("count", 0)))
|
||||
last_seen = _fmt_last_seen(p.get("last_seen"))
|
||||
message = str(p.get("normalized_message", p.get("error_message", p.get("message", ""))))[:col_message]
|
||||
|
||||
print(
|
||||
f"{status:<{col_status}} "
|
||||
f"{service:<{col_service}} "
|
||||
f"{count:>{col_count}} "
|
||||
f"{last_seen:<{col_last_seen}} "
|
||||
f"{message:<{col_message}}"
|
||||
)
|
||||
|
||||
print(separator)
|
||||
print(f"Total: {len(patterns)} pattern(s)")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Query error patterns from Firestore"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--status",
|
||||
help="Filter by status (new/acknowledged/known/fixed). Default: active patterns.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--service",
|
||||
help="Filter by service name (e.g. karaoke-backend).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hours",
|
||||
type=int,
|
||||
help="Only show patterns last seen within this many hours.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
dest="output_json",
|
||||
help="Output as JSON.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
default="nomadkaraoke",
|
||||
help="GCP project ID (default: nomadkaraoke).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
query_error_patterns(
|
||||
status=args.status,
|
||||
service=args.service,
|
||||
hours=args.hours,
|
||||
output_json=args.output_json,
|
||||
project=args.project,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user