mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Why The Phase 3d voice pool runs for ~1 week unattended. Failures must reach Discord #alerts (actionable-signal rule), and the error monitor only watches Cloud Run jobs listed in `MONITORED_CLOUD_RUN_JOBS`. ## Changes Added to `MONITORED_CLOUD_RUN_JOBS` (`backend/services/error_monitor/config.py`): - `voice-pool-select`, `voice-pool-fetch`, `voice-pool-analyse` (karaoke-decide, us-central1) - `vocal-range-gen` (karaoke-decide, us-central1; logs one ERROR from task 0 only when every attempted job fails) - `voice-pool-separate-job` (gen GPU job, us-east4) The monitor's log query filters on `resource.type="cloud_run_job"` + `resource.labels.job_name` only, with no region/location filter, so us-east4 is covered the same way as `audio-separation-job`. No docs list the monitored jobs, so none needed updating. ## Testing - Updated the pinned count/contents test (`tests/unit/services/error_monitor/test_config.py`) - Added a test that the built log filter includes the new jobs and has no location scope - error-monitor tests: 414 passed - Local CodeRabbit review: no findings - Version 0.283.1 -> 0.283.2 @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
222 lines
8.6 KiB
Python
222 lines
8.6 KiB
Python
"""Configuration for the production error monitoring service.
|
||
|
||
All constants, service lists, and environment-driven configuration
|
||
functions used across the error monitor modules.
|
||
"""
|
||
|
||
import os
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# GCP project settings
|
||
# ---------------------------------------------------------------------------
|
||
|
||
GCP_PROJECT: str = os.environ.get("GCP_PROJECT", "nomadkaraoke")
|
||
GCP_REGION: str = os.environ.get("GCP_REGION", "us-central1")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Log querying settings
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: How far back to look for errors on each run (minutes). Must equal the
|
||
#: scheduler interval (ErrorMonitorConfig.MONITOR_SCHEDULE, hourly since
|
||
#: 2026-09-26) so consecutive windows tile without gaps or double counting.
|
||
LOOKBACK_MINUTES: int = 60
|
||
|
||
#: Window (minutes) assumed for ``rolling_counts`` entries written before
|
||
#: entries recorded their own ``window_minutes`` (i.e. the 15-min era).
|
||
LEGACY_ROLLING_WINDOW_MINUTES: int = 15
|
||
|
||
#: Maximum number of log entries to fetch per resource type per run (scaled
|
||
#: 4x with the 15 -> 60 min window to keep the same per-minute capacity).
|
||
MAX_LOG_ENTRIES: int = 2000
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Monitored services
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MONITORED_CLOUD_RUN_SERVICES: list[str] = [
|
||
"karaoke-backend",
|
||
"karaoke-decide",
|
||
# karaoke-decide search service (offline song index + search API)
|
||
"karaoke-decide-search",
|
||
"audio-separator",
|
||
# Gen2 Cloud Functions appear as Cloud Run services in logs
|
||
"gdrive-validator",
|
||
"github-runner-manager",
|
||
"backup-to-aws",
|
||
"divebar-mirror",
|
||
"kn-data-sync",
|
||
"divebar-lookup",
|
||
"encoding-worker-idle-shutdown",
|
||
]
|
||
|
||
MONITORED_CLOUD_RUN_JOBS: list[str] = [
|
||
"video-encoding-job",
|
||
"lyrics-transcription-job",
|
||
"audio-separation-job",
|
||
"audio-download-job",
|
||
# karaoke-decide weekly MusicBrainz dump -> BigQuery refresh
|
||
"mb-refresh",
|
||
# karaoke-decide ListenBrainz statistics dump -> BigQuery refresh
|
||
"lb-refresh",
|
||
# karaoke-decide song search index build (eval-gated)
|
||
"song-index-build",
|
||
# karaoke-decide Phase 3d voice pool (us-central1): select -> fetch -> analyse
|
||
"voice-pool-select",
|
||
"voice-pool-fetch",
|
||
"voice-pool-analyse",
|
||
# karaoke-decide vocal-range generation (us-central1); logs one ERROR from task 0
|
||
# only when every attempted job fails
|
||
"vocal-range-gen",
|
||
# gen GPU separation job for the voice pool (us-east4; no region filter in the log query)
|
||
"voice-pool-separate-job",
|
||
]
|
||
|
||
# Gen2 Cloud Functions log as cloud_run_revision, not cloud_function.
|
||
# Their names are in MONITORED_CLOUD_RUN_SERVICES above.
|
||
MONITORED_CLOUD_FUNCTIONS: list[str] = []
|
||
|
||
MONITORED_GCE_INSTANCES: list[str] = [
|
||
"encoding-worker-a",
|
||
"encoding-worker-b",
|
||
"flacfetch-vm",
|
||
"divebar-sync-vm",
|
||
]
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Service dependency map
|
||
# Human-readable description of which services depend on which workers.
|
||
# Used as LLM context when generating root-cause analysis.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
SERVICE_DEPENDENCY_MAP: str = """\
|
||
karaoke-backend (main API)
|
||
→ video-encoding-job (Cloud Run Job, triggered per order)
|
||
→ lyrics-transcription-job (Cloud Run Job, triggered after audio-sep)
|
||
→ audio-separation-job (Cloud Run Job, runs on Cloud Run GPU)
|
||
→ audio-download-job (Cloud Run Job, uses flacfetch-vm)
|
||
→ gdrive-validator (Cloud Function, validates output files)
|
||
→ encoding-worker-a/b (GCE VMs, long-running encoding workers)
|
||
|
||
karaoke-decide (catalog / discovery API)
|
||
→ kn_data_sync (Cloud Function, syncs KaraokeNerds catalog)
|
||
→ divebar_mirror (Cloud Function, mirrors Divebar catalog)
|
||
→ divebar_lookup (Cloud Function, single-song lookup)
|
||
|
||
audio-separator (Cloud Run service, GPU stem separation)
|
||
→ audio-separation-job (Cloud Run Job, launched by audio-separator)
|
||
|
||
flacfetch-vm (GCE VM)
|
||
→ audio-download-job (initiated by karaoke-backend)
|
||
|
||
divebar-sync-vm (GCE VM)
|
||
→ divebar_mirror (Cloud Function, reads from this VM)
|
||
|
||
Shared infrastructure:
|
||
→ runner_manager (Cloud Function, manages encoding worker lifecycle)
|
||
→ encoding_worker_idle (Cloud Function, shuts down idle encoding workers)
|
||
→ backup_to_aws (Cloud Function, daily GCS → S3 backup)
|
||
"""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# LLM analysis settings
|
||
# ---------------------------------------------------------------------------
|
||
|
||
LLM_ANALYSIS_MODEL: str = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-3.8-flash")
|
||
|
||
#: Minimum number of active error patterns required before triggering LLM analysis.
|
||
MIN_PATTERNS_FOR_ANALYSIS: int = 3
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Spike detection
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: An error count is a spike if it exceeds SPIKE_MULTIPLIER × baseline average.
|
||
SPIKE_MULTIPLIER: float = 5.0
|
||
|
||
#: Minimum absolute error count before spike detection fires (avoids noise).
|
||
SPIKE_MIN_COUNT: int = 5
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Auto-resolution settings
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: Multiply LOOKBACK_MINUTES by this to get auto-resolve window in minutes.
|
||
AUTO_RESOLVE_MULTIPLIER: int = 8
|
||
|
||
#: Minimum hours before a pattern can be auto-resolved.
|
||
AUTO_RESOLVE_MIN_HOURS: int = 6
|
||
|
||
#: Maximum hours before a pattern is force-resolved even if still active.
|
||
AUTO_RESOLVE_MAX_HOURS: int = 168 # 1 week
|
||
|
||
#: Fallback resolve window (hours) if baseline data is unavailable.
|
||
AUTO_RESOLVE_FALLBACK_HOURS: int = 48
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Discord settings
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: Maximum Discord messages to send per monitor run (rate-limit guard).
|
||
MAX_DISCORD_MESSAGES_PER_RUN: int = 10
|
||
|
||
#: Discord API hard limit on message length.
|
||
DISCORD_MAX_MESSAGE_LENGTH: int = 2000
|
||
|
||
#: Discord message flag SUPPRESS_EMBEDS (1 << 2). Set on every error-monitor
|
||
#: post so quoted URLs (GCP docs links) don't unfurl into large preview cards.
|
||
DISCORD_SUPPRESS_EMBEDS_FLAG: int = 4
|
||
|
||
#: When one run has MORE than this many new patterns for one service, send a
|
||
#: single burst summary instead of one message each (2026-10-07/08: one bad
|
||
#: mb-refresh run produced 73 "New Error Pattern" messages, 10 an hour).
|
||
BURST_COLLAPSE_MIN_PATTERNS: int = 3
|
||
|
||
#: Number of patterns listed in a burst summary.
|
||
BURST_TOP_PATTERNS: int = 5
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Pattern tracking settings
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: Maximum number of active error patterns to track in Firestore.
|
||
MAX_ACTIVE_PATTERNS: int = 500
|
||
|
||
#: Rolling window used when computing error baselines.
|
||
ROLLING_WINDOW_DAYS: int = 7
|
||
|
||
#: Maximum length of a normalized error message stored in Firestore.
|
||
MAX_NORMALIZED_MESSAGE_LENGTH: int = 200
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helper functions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def get_llm_enabled() -> bool:
|
||
"""Return True if LLM analysis is enabled via the LLM_ANALYSIS_ENABLED env var."""
|
||
return os.environ.get("LLM_ANALYSIS_ENABLED", "false").strip().lower() == "true"
|
||
|
||
|
||
def get_discord_webhook_secret_name() -> str:
|
||
"""Return the Secret Manager secret name for the Discord webhook URL."""
|
||
return os.environ.get("DISCORD_WEBHOOK_SECRET_NAME", "error-monitor-discord-webhook")
|
||
|
||
|
||
def get_discord_noise_webhook_secret_name() -> str:
|
||
"""Return the Secret Manager secret name for the #noise Discord webhook.
|
||
|
||
#alerts is reserved for real problems (new patterns, spikes, incidents);
|
||
informational posts (auto-resolved summaries, the daily digest) go to #noise.
|
||
"""
|
||
return os.environ.get("DISCORD_NOISE_WEBHOOK_SECRET_NAME", "discord-noise-webhook")
|
||
|
||
|
||
def get_digest_mode() -> bool:
|
||
"""Return True if the monitor should run in digest mode (batch alerts instead of immediate).
|
||
|
||
In digest mode all alerts are bundled into a single Discord message sent at
|
||
the end of the run, rather than sending one message per error pattern.
|
||
"""
|
||
return os.environ.get("DIGEST_MODE", "false").strip().lower() == "true"
|