mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
160 lines
4.6 KiB
Python
Executable File
160 lines
4.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Resolve an error pattern by marking it as fixed.
|
|
|
|
Usage:
|
|
python scripts/resolve-error-pattern.py <pattern_prefix> --pr 42 --note "Fixed timeout"
|
|
python scripts/resolve-error-pattern.py <pattern_prefix> --note "Added retry logic"
|
|
|
|
Requires:
|
|
- google-cloud-firestore
|
|
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
|
"""
|
|
import argparse
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
from google.cloud import firestore
|
|
|
|
|
|
def resolve_error_pattern(
|
|
pattern_prefix: str,
|
|
note: str,
|
|
pr: Optional[int] = None,
|
|
repo: str = "nomadkaraoke/karaoke-gen",
|
|
project: str = "nomadkaraoke",
|
|
) -> bool:
|
|
"""
|
|
Find an error pattern by ID prefix and mark it as fixed.
|
|
|
|
Args:
|
|
pattern_prefix: Prefix of the pattern document ID.
|
|
note: Resolution note explaining how it was fixed.
|
|
pr: Optional PR number that fixed the issue.
|
|
repo: GitHub repo slug used to build the PR URL.
|
|
project: GCP project ID.
|
|
|
|
Returns:
|
|
True if the pattern was resolved, False otherwise.
|
|
"""
|
|
try:
|
|
db = firestore.Client(project=project)
|
|
collection = db.collection("error_patterns")
|
|
except Exception as exc:
|
|
print(f"Error: Could not connect to Firestore (project={project}): {exc}", file=sys.stderr)
|
|
return False
|
|
|
|
# Stream all docs and find those whose ID starts with the prefix
|
|
matches = []
|
|
try:
|
|
for doc in collection.stream():
|
|
if doc.id.startswith(pattern_prefix):
|
|
matches.append(doc)
|
|
except Exception as exc:
|
|
print(f"Error: Could not query error_patterns: {exc}", file=sys.stderr)
|
|
return False
|
|
|
|
if len(matches) == 0:
|
|
print(f"Error: No pattern found with ID prefix '{pattern_prefix}'", file=sys.stderr)
|
|
return False
|
|
|
|
if len(matches) > 1:
|
|
print(
|
|
f"Ambiguous prefix '{pattern_prefix}' — matches {len(matches)} pattern(s). "
|
|
"Please provide a longer prefix.",
|
|
file=sys.stderr,
|
|
)
|
|
print("\nFirst 5 matches:", file=sys.stderr)
|
|
for doc in matches[:5]:
|
|
data = doc.to_dict()
|
|
service = data.get("service", "")
|
|
message = str(data.get("error_message", data.get("message", "")))[:60]
|
|
print(f" {doc.id} [{service}] {message}", file=sys.stderr)
|
|
return False
|
|
|
|
doc = matches[0]
|
|
data = doc.to_dict()
|
|
resolved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
fixed_by: dict = {
|
|
"resolved_at": resolved_at,
|
|
"note": note,
|
|
}
|
|
|
|
if pr is not None:
|
|
pr_url = f"https://github.com/{repo}/pull/{pr}"
|
|
fixed_by["pr_url"] = pr_url
|
|
fixed_by["pr_display"] = f"#{pr}"
|
|
|
|
update_payload = {
|
|
"status": "fixed",
|
|
"fixed_by": fixed_by,
|
|
}
|
|
|
|
try:
|
|
doc.reference.update(update_payload)
|
|
except Exception as exc:
|
|
print(f"Error: Could not update pattern {doc.id}: {exc}", file=sys.stderr)
|
|
return False
|
|
|
|
# Confirmation output
|
|
service = data.get("service", "")
|
|
message = str(data.get("error_message", data.get("message", "")))[:80]
|
|
print(f"Resolved pattern: {doc.id}")
|
|
print(f" Service: {service}")
|
|
print(f" Message: {message}")
|
|
print(f" Status: fixed")
|
|
print(f" Note: {note}")
|
|
if pr is not None:
|
|
print(f" PR: {fixed_by['pr_url']}")
|
|
print(f" Resolved: {resolved_at}")
|
|
|
|
return True
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Resolve an error pattern by marking it as fixed"
|
|
)
|
|
parser.add_argument(
|
|
"pattern_prefix",
|
|
help="Prefix of the error pattern document ID.",
|
|
)
|
|
parser.add_argument(
|
|
"--note",
|
|
required=True,
|
|
help="Resolution note (e.g. 'Fixed timeout in encoding worker').",
|
|
)
|
|
parser.add_argument(
|
|
"--pr",
|
|
type=int,
|
|
help="PR number that fixed the issue (optional).",
|
|
)
|
|
parser.add_argument(
|
|
"--repo",
|
|
default="nomadkaraoke/karaoke-gen",
|
|
help="GitHub repo slug for PR URL (default: nomadkaraoke/karaoke-gen).",
|
|
)
|
|
parser.add_argument(
|
|
"--project",
|
|
default="nomadkaraoke",
|
|
help="GCP project ID (default: nomadkaraoke).",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
success = resolve_error_pattern(
|
|
pattern_prefix=args.pattern_prefix,
|
|
note=args.note,
|
|
pr=args.pr,
|
|
repo=args.repo,
|
|
project=args.project,
|
|
)
|
|
|
|
if not success:
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|