mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
194 lines
5.5 KiB
Python
Executable File
194 lines
5.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Query error patterns from Firestore.
|
|
|
|
Usage:
|
|
python scripts/query-error-patterns.py # All active
|
|
python scripts/query-error-patterns.py --status new # Only new
|
|
python scripts/query-error-patterns.py --service karaoke-backend
|
|
python scripts/query-error-patterns.py --hours 24 # Last 24h
|
|
python scripts/query-error-patterns.py --json # JSON output
|
|
|
|
Requires:
|
|
- google-cloud-firestore
|
|
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, Optional
|
|
|
|
from google.cloud import firestore
|
|
from google.cloud.firestore_v1 import FieldFilter
|
|
|
|
|
|
ACTIVE_STATUSES = ["new", "acknowledged", "known"]
|
|
|
|
|
|
def query_error_patterns(
|
|
status: Optional[str] = None,
|
|
service: Optional[str] = None,
|
|
hours: Optional[int] = None,
|
|
output_json: bool = False,
|
|
project: str = "nomadkaraoke",
|
|
) -> list[dict[str, Any]]:
|
|
"""
|
|
Query error patterns from Firestore.
|
|
|
|
Args:
|
|
status: Filter by status (new/acknowledged/known/fixed). Default: active (new/acknowledged/known).
|
|
service: Filter by service name.
|
|
hours: Only return patterns last seen within this many hours.
|
|
output_json: Print JSON output instead of a table.
|
|
project: GCP project ID.
|
|
|
|
Returns:
|
|
List of pattern dicts.
|
|
"""
|
|
db = firestore.Client(project=project)
|
|
query = db.collection("error_patterns")
|
|
|
|
# Status filter
|
|
if status:
|
|
query = query.where(filter=FieldFilter("status", "==", status))
|
|
else:
|
|
query = query.where(filter=FieldFilter("status", "in", ACTIVE_STATUSES))
|
|
|
|
# Service filter
|
|
if service:
|
|
query = query.where(filter=FieldFilter("service", "==", service))
|
|
|
|
# Fetch all matching docs (streaming)
|
|
patterns = []
|
|
for doc in query.stream():
|
|
data = doc.to_dict()
|
|
data["_id"] = doc.id
|
|
patterns.append(data)
|
|
|
|
# Hours filter (applied in Python — Firestore requires composite index for multi-field filters)
|
|
if hours is not None:
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
filtered = []
|
|
for p in patterns:
|
|
last_seen = p.get("last_seen")
|
|
if last_seen is not None:
|
|
# Firestore Timestamps have a .timestamp() method
|
|
ts = last_seen.timestamp() if hasattr(last_seen, "timestamp") else float(last_seen)
|
|
if ts >= cutoff.timestamp():
|
|
filtered.append(p)
|
|
patterns = filtered
|
|
|
|
# Sort by last_seen descending (most recent first)
|
|
def sort_key(p):
|
|
ls = p.get("last_seen")
|
|
if ls is None:
|
|
return 0.0
|
|
return ls.timestamp() if hasattr(ls, "timestamp") else float(ls)
|
|
|
|
patterns.sort(key=sort_key, reverse=True)
|
|
|
|
if output_json:
|
|
print(json.dumps(patterns, indent=2, default=str))
|
|
else:
|
|
_print_table(patterns)
|
|
|
|
return patterns
|
|
|
|
|
|
def _fmt_last_seen(value) -> str:
|
|
"""Return first 19 chars of ISO timestamp, or empty string."""
|
|
if value is None:
|
|
return ""
|
|
if hasattr(value, "isoformat"):
|
|
return value.isoformat()[:19]
|
|
return str(value)[:19]
|
|
|
|
|
|
def _print_table(patterns: list[dict[str, Any]]) -> None:
|
|
"""Print patterns as a human-readable table."""
|
|
if not patterns:
|
|
print("No error patterns found.")
|
|
return
|
|
|
|
# Column widths
|
|
col_status = 14
|
|
col_service = 24
|
|
col_count = 6
|
|
col_last_seen = 19
|
|
col_message = 50
|
|
|
|
header = (
|
|
f"{'Status':<{col_status}} "
|
|
f"{'Service':<{col_service}} "
|
|
f"{'Count':>{col_count}} "
|
|
f"{'Last Seen':<{col_last_seen}} "
|
|
f"{'Message':<{col_message}}"
|
|
)
|
|
separator = "-" * len(header)
|
|
|
|
print(separator)
|
|
print(header)
|
|
print(separator)
|
|
|
|
for p in patterns:
|
|
status = str(p.get("status", ""))[:col_status]
|
|
service = str(p.get("service", ""))[:col_service]
|
|
count = p.get("total_count", p.get("occurrence_count", p.get("count", 0)))
|
|
last_seen = _fmt_last_seen(p.get("last_seen"))
|
|
message = str(p.get("normalized_message", p.get("error_message", p.get("message", ""))))[:col_message]
|
|
|
|
print(
|
|
f"{status:<{col_status}} "
|
|
f"{service:<{col_service}} "
|
|
f"{count:>{col_count}} "
|
|
f"{last_seen:<{col_last_seen}} "
|
|
f"{message:<{col_message}}"
|
|
)
|
|
|
|
print(separator)
|
|
print(f"Total: {len(patterns)} pattern(s)")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Query error patterns from Firestore"
|
|
)
|
|
parser.add_argument(
|
|
"--status",
|
|
help="Filter by status (new/acknowledged/known/fixed). Default: active patterns.",
|
|
)
|
|
parser.add_argument(
|
|
"--service",
|
|
help="Filter by service name (e.g. karaoke-backend).",
|
|
)
|
|
parser.add_argument(
|
|
"--hours",
|
|
type=int,
|
|
help="Only show patterns last seen within this many hours.",
|
|
)
|
|
parser.add_argument(
|
|
"--json",
|
|
action="store_true",
|
|
dest="output_json",
|
|
help="Output as JSON.",
|
|
)
|
|
parser.add_argument(
|
|
"--project",
|
|
default="nomadkaraoke",
|
|
help="GCP project ID (default: nomadkaraoke).",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
query_error_patterns(
|
|
status=args.status,
|
|
service=args.service,
|
|
hours=args.hours,
|
|
output_json=args.output_json,
|
|
project=args.project,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|