Files
karaoke-gen/backend/services/error_monitor/config.py
T
Andrew Beveridge ☄️andClaude Opus 5.5 9014754bda feat(error-monitor): watch voice pool jobs (v0.283.2) (#1219)
## Why
The Phase 3d voice pool runs for ~1 week unattended. Failures must reach
Discord #alerts (actionable-signal rule), and the error monitor only
watches Cloud Run jobs listed in `MONITORED_CLOUD_RUN_JOBS`.

## Changes
Added to `MONITORED_CLOUD_RUN_JOBS`
(`backend/services/error_monitor/config.py`):
- `voice-pool-select`, `voice-pool-fetch`, `voice-pool-analyse`
(karaoke-decide, us-central1)
- `vocal-range-gen` (karaoke-decide, us-central1; logs one ERROR from
task 0 only when every attempted job fails)
- `voice-pool-separate-job` (gen GPU job, us-east4)

The monitor's log query filters on `resource.type="cloud_run_job"` +
`resource.labels.job_name` only, with no region/location filter, so
us-east4 is covered the same way as `audio-separation-job`. No docs list
the monitored jobs, so none needed updating.

## Testing
- Updated the pinned count/contents test
(`tests/unit/services/error_monitor/test_config.py`)
- Added a test that the built log filter includes the new jobs and has
no location scope
- error-monitor tests: 414 passed
- Local CodeRabbit review: no findings
- Version 0.283.1 -> 0.283.2

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 16:37:49 -04:00

222 lines
8.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Configuration for the production error monitoring service.
All constants, service lists, and environment-driven configuration
functions used across the error monitor modules.
"""
import os
# ---------------------------------------------------------------------------
# GCP project settings
# ---------------------------------------------------------------------------
GCP_PROJECT: str = os.environ.get("GCP_PROJECT", "nomadkaraoke")
GCP_REGION: str = os.environ.get("GCP_REGION", "us-central1")
# ---------------------------------------------------------------------------
# Log querying settings
# ---------------------------------------------------------------------------
#: How far back to look for errors on each run (minutes). Must equal the
#: scheduler interval (ErrorMonitorConfig.MONITOR_SCHEDULE, hourly since
#: 2026-09-26) so consecutive windows tile without gaps or double counting.
LOOKBACK_MINUTES: int = 60
#: Window (minutes) assumed for ``rolling_counts`` entries written before
#: entries recorded their own ``window_minutes`` (i.e. the 15-min era).
LEGACY_ROLLING_WINDOW_MINUTES: int = 15
#: Maximum number of log entries to fetch per resource type per run (scaled
#: 4x with the 15 -> 60 min window to keep the same per-minute capacity).
MAX_LOG_ENTRIES: int = 2000
# ---------------------------------------------------------------------------
# Monitored services
# ---------------------------------------------------------------------------
MONITORED_CLOUD_RUN_SERVICES: list[str] = [
"karaoke-backend",
"karaoke-decide",
# karaoke-decide search service (offline song index + search API)
"karaoke-decide-search",
"audio-separator",
# Gen2 Cloud Functions appear as Cloud Run services in logs
"gdrive-validator",
"github-runner-manager",
"backup-to-aws",
"divebar-mirror",
"kn-data-sync",
"divebar-lookup",
"encoding-worker-idle-shutdown",
]
MONITORED_CLOUD_RUN_JOBS: list[str] = [
"video-encoding-job",
"lyrics-transcription-job",
"audio-separation-job",
"audio-download-job",
# karaoke-decide weekly MusicBrainz dump -> BigQuery refresh
"mb-refresh",
# karaoke-decide ListenBrainz statistics dump -> BigQuery refresh
"lb-refresh",
# karaoke-decide song search index build (eval-gated)
"song-index-build",
# karaoke-decide Phase 3d voice pool (us-central1): select -> fetch -> analyse
"voice-pool-select",
"voice-pool-fetch",
"voice-pool-analyse",
# karaoke-decide vocal-range generation (us-central1); logs one ERROR from task 0
# only when every attempted job fails
"vocal-range-gen",
# gen GPU separation job for the voice pool (us-east4; no region filter in the log query)
"voice-pool-separate-job",
]
# Gen2 Cloud Functions log as cloud_run_revision, not cloud_function.
# Their names are in MONITORED_CLOUD_RUN_SERVICES above.
MONITORED_CLOUD_FUNCTIONS: list[str] = []
MONITORED_GCE_INSTANCES: list[str] = [
"encoding-worker-a",
"encoding-worker-b",
"flacfetch-vm",
"divebar-sync-vm",
]
# ---------------------------------------------------------------------------
# Service dependency map
# Human-readable description of which services depend on which workers.
# Used as LLM context when generating root-cause analysis.
# ---------------------------------------------------------------------------
SERVICE_DEPENDENCY_MAP: str = """\
karaoke-backend (main API)
→ video-encoding-job (Cloud Run Job, triggered per order)
→ lyrics-transcription-job (Cloud Run Job, triggered after audio-sep)
→ audio-separation-job (Cloud Run Job, runs on Cloud Run GPU)
→ audio-download-job (Cloud Run Job, uses flacfetch-vm)
→ gdrive-validator (Cloud Function, validates output files)
→ encoding-worker-a/b (GCE VMs, long-running encoding workers)
karaoke-decide (catalog / discovery API)
→ kn_data_sync (Cloud Function, syncs KaraokeNerds catalog)
→ divebar_mirror (Cloud Function, mirrors Divebar catalog)
→ divebar_lookup (Cloud Function, single-song lookup)
audio-separator (Cloud Run service, GPU stem separation)
→ audio-separation-job (Cloud Run Job, launched by audio-separator)
flacfetch-vm (GCE VM)
→ audio-download-job (initiated by karaoke-backend)
divebar-sync-vm (GCE VM)
→ divebar_mirror (Cloud Function, reads from this VM)
Shared infrastructure:
→ runner_manager (Cloud Function, manages encoding worker lifecycle)
→ encoding_worker_idle (Cloud Function, shuts down idle encoding workers)
→ backup_to_aws (Cloud Function, daily GCS → S3 backup)
"""
# ---------------------------------------------------------------------------
# LLM analysis settings
# ---------------------------------------------------------------------------
LLM_ANALYSIS_MODEL: str = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-3.8-flash")
#: Minimum number of active error patterns required before triggering LLM analysis.
MIN_PATTERNS_FOR_ANALYSIS: int = 3
# ---------------------------------------------------------------------------
# Spike detection
# ---------------------------------------------------------------------------
#: An error count is a spike if it exceeds SPIKE_MULTIPLIER × baseline average.
SPIKE_MULTIPLIER: float = 5.0
#: Minimum absolute error count before spike detection fires (avoids noise).
SPIKE_MIN_COUNT: int = 5
# ---------------------------------------------------------------------------
# Auto-resolution settings
# ---------------------------------------------------------------------------
#: Multiply LOOKBACK_MINUTES by this to get auto-resolve window in minutes.
AUTO_RESOLVE_MULTIPLIER: int = 8
#: Minimum hours before a pattern can be auto-resolved.
AUTO_RESOLVE_MIN_HOURS: int = 6
#: Maximum hours before a pattern is force-resolved even if still active.
AUTO_RESOLVE_MAX_HOURS: int = 168 # 1 week
#: Fallback resolve window (hours) if baseline data is unavailable.
AUTO_RESOLVE_FALLBACK_HOURS: int = 48
# ---------------------------------------------------------------------------
# Discord settings
# ---------------------------------------------------------------------------
#: Maximum Discord messages to send per monitor run (rate-limit guard).
MAX_DISCORD_MESSAGES_PER_RUN: int = 10
#: Discord API hard limit on message length.
DISCORD_MAX_MESSAGE_LENGTH: int = 2000
#: Discord message flag SUPPRESS_EMBEDS (1 << 2). Set on every error-monitor
#: post so quoted URLs (GCP docs links) don't unfurl into large preview cards.
DISCORD_SUPPRESS_EMBEDS_FLAG: int = 4
#: When one run has MORE than this many new patterns for one service, send a
#: single burst summary instead of one message each (2026-10-07/08: one bad
#: mb-refresh run produced 73 "New Error Pattern" messages, 10 an hour).
BURST_COLLAPSE_MIN_PATTERNS: int = 3
#: Number of patterns listed in a burst summary.
BURST_TOP_PATTERNS: int = 5
# ---------------------------------------------------------------------------
# Pattern tracking settings
# ---------------------------------------------------------------------------
#: Maximum number of active error patterns to track in Firestore.
MAX_ACTIVE_PATTERNS: int = 500
#: Rolling window used when computing error baselines.
ROLLING_WINDOW_DAYS: int = 7
#: Maximum length of a normalized error message stored in Firestore.
MAX_NORMALIZED_MESSAGE_LENGTH: int = 200
# ---------------------------------------------------------------------------
# Helper functions
# ---------------------------------------------------------------------------
def get_llm_enabled() -> bool:
"""Return True if LLM analysis is enabled via the LLM_ANALYSIS_ENABLED env var."""
return os.environ.get("LLM_ANALYSIS_ENABLED", "false").strip().lower() == "true"
def get_discord_webhook_secret_name() -> str:
"""Return the Secret Manager secret name for the Discord webhook URL."""
return os.environ.get("DISCORD_WEBHOOK_SECRET_NAME", "error-monitor-discord-webhook")
def get_discord_noise_webhook_secret_name() -> str:
"""Return the Secret Manager secret name for the #noise Discord webhook.
#alerts is reserved for real problems (new patterns, spikes, incidents);
informational posts (auto-resolved summaries, the daily digest) go to #noise.
"""
return os.environ.get("DISCORD_NOISE_WEBHOOK_SECRET_NAME", "discord-noise-webhook")
def get_digest_mode() -> bool:
"""Return True if the monitor should run in digest mode (batch alerts instead of immediate).
In digest mode all alerts are bundled into a single Discord message sent at
the end of the run, rather than sending one message per error pattern.
"""
return os.environ.get("DIGEST_MODE", "false").strip().lower() == "true"