mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
148 lines
4.6 KiB
Python
148 lines
4.6 KiB
Python
"""
|
|
Centralized configuration for Pulumi infrastructure.
|
|
|
|
This module provides shared constants and configuration used across all infrastructure modules.
|
|
"""
|
|
|
|
import pulumi
|
|
import pulumi_gcp as gcp
|
|
|
|
# GCP project info - lazily loaded to work with Pulumi runtime
|
|
_project = None
|
|
|
|
|
|
def _get_project():
|
|
"""Get GCP project info (lazy-loaded for Pulumi runtime compatibility)."""
|
|
global _project
|
|
if _project is None:
|
|
_project = gcp.organizations.get_project()
|
|
return _project
|
|
|
|
|
|
# Project ID and number are accessed as properties
|
|
# These will resolve correctly during Pulumi execution
|
|
PROJECT_ID = "nomadkaraoke" # Hardcoded for now - matches existing behavior
|
|
PROJECT_NUMBER = None # Set lazily when needed
|
|
|
|
|
|
def get_project_number() -> str:
|
|
"""Get the GCP project number."""
|
|
return _get_project().number
|
|
|
|
# Region configuration
|
|
REGION = "us-central1"
|
|
ZONE = f"{REGION}-a"
|
|
|
|
# Encoding worker zone - uses us-central1-c due to c4d-highcpu-32 availability
|
|
# (us-central1-a and us-central1-b often lack capacity for high-end C4D instances)
|
|
ENCODING_WORKER_ZONE = f"{REGION}-c"
|
|
|
|
# Pulumi config accessor
|
|
config = pulumi.Config()
|
|
|
|
|
|
class MachineTypes:
|
|
"""Machine type configurations for GCE instances."""
|
|
|
|
GITHUB_RUNNER = "e2-standard-4" # 4 vCPU, 16GB RAM
|
|
GITHUB_BUILD_RUNNER = "e2-standard-8" # 8 vCPU, 32GB RAM - dedicated Docker build runner
|
|
GITHUB_GPU_RUNNER = "n1-standard-4" # 4 vCPU, 15GB RAM - GPU runners need N1 series
|
|
ENCODING_WORKER = "c4d-highcpu-32" # 32 vCPU, AMD EPYC 9B45 Turin - 4.92x faster than c4-standard-8
|
|
FLACFETCH = "e2-small" # 0.5 vCPU, 2GB RAM
|
|
|
|
|
|
class DiskSizes:
|
|
"""Disk size configurations in GB."""
|
|
|
|
GITHUB_RUNNER = 200 # Large for Docker builds/caches
|
|
ENCODING_WORKER = 100 # For temp video files during encoding
|
|
FLACFETCH = 30 # For torrent storage
|
|
|
|
|
|
class QueueConfigs:
|
|
"""Cloud Tasks queue rate limiting configurations."""
|
|
|
|
class Audio:
|
|
MAX_DISPATCHES_PER_SECOND = 10
|
|
MAX_CONCURRENT_DISPATCHES = 50
|
|
MAX_RETRY_DURATION = "1800s" # 30 min
|
|
|
|
class Lyrics:
|
|
MAX_DISPATCHES_PER_SECOND = 10
|
|
MAX_CONCURRENT_DISPATCHES = 50
|
|
MAX_RETRY_DURATION = "1200s" # 20 min
|
|
|
|
class Screens:
|
|
MAX_DISPATCHES_PER_SECOND = 50
|
|
MAX_CONCURRENT_DISPATCHES = 100
|
|
MAX_RETRY_DURATION = "300s" # 5 min
|
|
|
|
class Render:
|
|
MAX_DISPATCHES_PER_SECOND = 5
|
|
MAX_CONCURRENT_DISPATCHES = 20
|
|
MAX_RETRY_DURATION = "3600s" # 60 min
|
|
|
|
class Video:
|
|
MAX_DISPATCHES_PER_SECOND = 3
|
|
MAX_CONCURRENT_DISPATCHES = 10
|
|
MAX_RETRY_DURATION = "7200s" # 2 hours
|
|
|
|
|
|
# Number of self-hosted GitHub Action runners
|
|
# Reduced from 20 to 3 - runners auto-scale with start/stop management
|
|
NUM_GITHUB_RUNNERS = 3
|
|
NUM_GPU_RUNNERS = 3
|
|
|
|
# Runner labels
|
|
GENERAL_RUNNER_LABELS = "self-hosted,linux,x64,gcp,large-disk"
|
|
BUILD_RUNNER_LABELS = "self-hosted,linux,x64,gcp,large-disk,docker-build"
|
|
GPU_RUNNER_LABELS = "self-hosted,linux,x64,gcp,gpu"
|
|
|
|
# Google Drive folder ID for validator (public share folder)
|
|
GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
|
|
|
|
|
|
class RunnerManagerConfig:
|
|
"""Configuration for GitHub runner auto-start/stop management."""
|
|
# Auto-scales runners: starts on CI demand, stops after idle timeout
|
|
|
|
FUNCTION_NAME = "github-runner-manager"
|
|
FUNCTION_MEMORY = "512M" # Increased from 256M due to memory usage
|
|
FUNCTION_TIMEOUT = 300 # 5 minutes
|
|
IDLE_TIMEOUT_HOURS = 0.25 # 15 minutes — solo dev, runners stop at next scheduler check
|
|
IDLE_CHECK_SCHEDULE = "*/15 * * * *" # Every 15 minutes
|
|
|
|
|
|
class SecretNames:
|
|
"""Secret Manager secret names."""
|
|
|
|
GITHUB_RUNNER_PAT = "github-runner-pat"
|
|
GITHUB_WEBHOOK_SECRET = "github-webhook-secret"
|
|
|
|
# GitHub repository for runner registration
|
|
GITHUB_REPO_OWNER = "nomadkaraoke"
|
|
GITHUB_REPO_NAME = "karaoke-gen"
|
|
|
|
|
|
class EncodingWorkerConfig:
|
|
"""Configuration for blue-green encoding worker VMs."""
|
|
VM_NAMES = ["encoding-worker-a", "encoding-worker-b"]
|
|
IP_NAMES = ["encoding-worker-ip-a", "encoding-worker-ip-b"]
|
|
IDLE_CHECK_SCHEDULE = "*/5 * * * *" # Every 5 minutes
|
|
IDLE_TIMEOUT_MINUTES = 15
|
|
FUNCTION_NAME = "encoding-worker-idle-shutdown"
|
|
FUNCTION_MEMORY = "256M"
|
|
FUNCTION_TIMEOUT = 120 # 2 minutes
|
|
|
|
|
|
class ErrorMonitorConfig:
|
|
"""Configuration for the error monitor Cloud Run Job."""
|
|
JOB_NAME = "nomad-error-monitor"
|
|
MEMORY = "512Mi"
|
|
CPU = "1"
|
|
TIMEOUT = "300s"
|
|
MAX_RETRIES = 0
|
|
MONITOR_SCHEDULE = "*/15 * * * *" # Every 15 minutes UTC
|
|
DIGEST_SCHEDULE = "0 8 * * *" # 08:00 UTC daily
|
|
SCHEDULER_SA_NAME = "error-monitor-scheduler"
|