mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat: production error monitor — log collection, LLM analysis, Discord alerts (#713)
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b847dba411
commit
9a5c9d497a
28 files changed
+11056
-2
No files matched your search
Executable
+159
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Resolve an error pattern by marking it as fixed.
|
||||
|
||||
Usage:
|
||||
python scripts/resolve-error-pattern.py <pattern_prefix> --pr 42 --note "Fixed timeout"
|
||||
python scripts/resolve-error-pattern.py <pattern_prefix> --note "Added retry logic"
|
||||
|
||||
Requires:
|
||||
- google-cloud-firestore
|
||||
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from google.cloud import firestore
|
||||
|
||||
|
||||
def resolve_error_pattern(
|
||||
pattern_prefix: str,
|
||||
note: str,
|
||||
pr: Optional[int] = None,
|
||||
repo: str = "nomadkaraoke/karaoke-gen",
|
||||
project: str = "nomadkaraoke",
|
||||
) -> bool:
|
||||
"""
|
||||
Find an error pattern by ID prefix and mark it as fixed.
|
||||
|
||||
Args:
|
||||
pattern_prefix: Prefix of the pattern document ID.
|
||||
note: Resolution note explaining how it was fixed.
|
||||
pr: Optional PR number that fixed the issue.
|
||||
repo: GitHub repo slug used to build the PR URL.
|
||||
project: GCP project ID.
|
||||
|
||||
Returns:
|
||||
True if the pattern was resolved, False otherwise.
|
||||
"""
|
||||
try:
|
||||
db = firestore.Client(project=project)
|
||||
collection = db.collection("error_patterns")
|
||||
except Exception as exc:
|
||||
print(f"Error: Could not connect to Firestore (project={project}): {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
# Stream all docs and find those whose ID starts with the prefix
|
||||
matches = []
|
||||
try:
|
||||
for doc in collection.stream():
|
||||
if doc.id.startswith(pattern_prefix):
|
||||
matches.append(doc)
|
||||
except Exception as exc:
|
||||
print(f"Error: Could not query error_patterns: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if len(matches) == 0:
|
||||
print(f"Error: No pattern found with ID prefix '{pattern_prefix}'", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if len(matches) > 1:
|
||||
print(
|
||||
f"Ambiguous prefix '{pattern_prefix}' — matches {len(matches)} pattern(s). "
|
||||
"Please provide a longer prefix.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("\nFirst 5 matches:", file=sys.stderr)
|
||||
for doc in matches[:5]:
|
||||
data = doc.to_dict()
|
||||
service = data.get("service", "")
|
||||
message = str(data.get("error_message", data.get("message", "")))[:60]
|
||||
print(f" {doc.id} [{service}] {message}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
doc = matches[0]
|
||||
data = doc.to_dict()
|
||||
resolved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
fixed_by: dict = {
|
||||
"resolved_at": resolved_at,
|
||||
"note": note,
|
||||
}
|
||||
|
||||
if pr is not None:
|
||||
pr_url = f"https://github.com/{repo}/pull/{pr}"
|
||||
fixed_by["pr_url"] = pr_url
|
||||
fixed_by["pr_display"] = f"#{pr}"
|
||||
|
||||
update_payload = {
|
||||
"status": "fixed",
|
||||
"fixed_by": fixed_by,
|
||||
}
|
||||
|
||||
try:
|
||||
doc.reference.update(update_payload)
|
||||
except Exception as exc:
|
||||
print(f"Error: Could not update pattern {doc.id}: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
# Confirmation output
|
||||
service = data.get("service", "")
|
||||
message = str(data.get("error_message", data.get("message", "")))[:80]
|
||||
print(f"Resolved pattern: {doc.id}")
|
||||
print(f" Service: {service}")
|
||||
print(f" Message: {message}")
|
||||
print(f" Status: fixed")
|
||||
print(f" Note: {note}")
|
||||
if pr is not None:
|
||||
print(f" PR: {fixed_by['pr_url']}")
|
||||
print(f" Resolved: {resolved_at}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Resolve an error pattern by marking it as fixed"
|
||||
)
|
||||
parser.add_argument(
|
||||
"pattern_prefix",
|
||||
help="Prefix of the error pattern document ID.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--note",
|
||||
required=True,
|
||||
help="Resolution note (e.g. 'Fixed timeout in encoding worker').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pr",
|
||||
type=int,
|
||||
help="PR number that fixed the issue (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo",
|
||||
default="nomadkaraoke/karaoke-gen",
|
||||
help="GitHub repo slug for PR URL (default: nomadkaraoke/karaoke-gen).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
default="nomadkaraoke",
|
||||
help="GCP project ID (default: nomadkaraoke).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
success = resolve_error_pattern(
|
||||
pattern_prefix=args.pattern_prefix,
|
||||
note=args.note,
|
||||
pr=args.pr,
|
||||
repo=args.repo,
|
||||
project=args.project,
|
||||
)
|
||||
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user