mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Add a production error monitoring service that runs as a Cloud Run Job every 15 minutes - Queries Cloud Logging across all NK services (Cloud Run, Jobs, Functions, GCE VMs), normalizes/deduplicates errors, uses Gemini Flash for incident grouping, sends Discord alerts, and auto-resolves silent patterns - Updates all `/prod-*` slash commands to use pre-computed Firestore patterns as primary data source ## Architecture Based on the proven Aquarius error bot system, adapted for Nomad Karaoke's multi-service architecture (20+ resources across 4 resource types). **New backend modules** (`backend/services/error_monitor/`): - `config.py` — constants, monitored services, env var overrides - `normalizer.py` — regex normalization + SHA-256 pattern hashing - `known_issues.py` — 8 ignore patterns for infrastructure noise - `firestore_adapter.py` — CRUD for `error_patterns`, `error_incidents`, `discord_alerts` - `discord.py` — webhook client with 5 alert formatters (new pattern, incident, spike, auto-resolved, daily digest) - `llm_analysis.py` — Gemini Flash incident grouping + near-duplicate detection - `monitor.py` — orchestrator entry point (`python -m backend.services.error_monitor.monitor`) **New infrastructure** (`infrastructure/modules/error_monitor.py`): - Cloud Run Job (`nomad-error-monitor`) sharing karaoke-backend image - Cloud Scheduler: every 15 min monitor + 08:00 UTC daily digest - Scheduler service account with `roles/run.invoker` - Backend SA gets `roles/logging.viewer` **New CLI scripts:** - `scripts/query-error-patterns.py` — query patterns from Firestore - `scripts/resolve-error-pattern.py` — mark patterns as fixed with PR reference ## Testing - 310 unit tests across 7 test modules (all passing) - Covers: config, normalizer, known issues, Firestore adapter, Discord formatting, LLM analysis, monitor orchestrator - P0/P1 caller-callee contract tests for monitor integration points - All tests use mocked external dependencies (no GCP calls) ## Review - [x] CodeRabbit CLI review completed locally (1 cycle, 3 issues fixed) - [x] Test review completed (P0/P1 gaps addressed) - [x] Documentation updated (README, ARCHITECTURE) - [x] Version bumped (0.168.2 → 0.169.0) ## Design Spec `docs/archive/2026-04-12-error-monitor-design.md` @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
149 lines
5.1 KiB
Python
149 lines
5.1 KiB
Python
"""
|
|
Backend service account and IAM bindings.
|
|
|
|
Manages the karaoke-backend service account used by Cloud Run and its associated
|
|
IAM roles for accessing Firestore, GCS, Secret Manager, and other services.
|
|
"""
|
|
|
|
import pulumi
|
|
import pulumi_gcp as gcp
|
|
from pulumi_gcp import serviceaccount
|
|
|
|
from config import PROJECT_ID, get_project_number
|
|
|
|
|
|
def create_backend_service_account() -> serviceaccount.Account:
|
|
"""
|
|
Create the main backend service account.
|
|
|
|
Returns:
|
|
serviceaccount.Account: The service account resource.
|
|
"""
|
|
return serviceaccount.Account(
|
|
"karaoke-backend-sa",
|
|
account_id="karaoke-backend",
|
|
display_name="Karaoke Backend Service Account",
|
|
)
|
|
|
|
|
|
def grant_backend_permissions(
|
|
service_account: serviceaccount.Account,
|
|
) -> dict:
|
|
"""
|
|
Grant all necessary permissions to the backend service account.
|
|
|
|
Args:
|
|
service_account: The backend service account.
|
|
|
|
Returns:
|
|
dict: Dictionary of IAM binding resources.
|
|
"""
|
|
bindings = {}
|
|
|
|
# Grant Firestore permissions
|
|
bindings["firestore_iam"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-firestore-access",
|
|
project=PROJECT_ID,
|
|
role="roles/datastore.user",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Grant Storage permissions
|
|
bindings["storage_iam"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-storage-access",
|
|
project=PROJECT_ID,
|
|
role="roles/storage.objectAdmin",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Grant Secret Manager read permissions
|
|
bindings["secrets_read_iam"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-secrets-access",
|
|
project=PROJECT_ID,
|
|
role="roles/secretmanager.secretAccessor",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Grant Secret Manager write permissions (for OAuth token refresh/device auth)
|
|
bindings["secrets_write_iam"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-secrets-write",
|
|
project=PROJECT_ID,
|
|
role="roles/secretmanager.secretVersionManager",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Grant Cloud Trace write permissions (for OpenTelemetry tracing)
|
|
bindings["cloud_trace_iam"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-cloudtrace-agent",
|
|
project=PROJECT_ID,
|
|
role="roles/cloudtrace.agent",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Grant Vertex AI User permissions (for Gemini agentic AI correction)
|
|
bindings["vertex_ai_iam"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-vertexai-user",
|
|
project=PROJECT_ID,
|
|
role="roles/aiplatform.user",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Grant backend service account permission to enqueue Cloud Tasks
|
|
bindings["cloud_tasks_enqueuer"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-cloudtasks-enqueuer",
|
|
project=PROJECT_ID,
|
|
role="roles/cloudtasks.enqueuer",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Service Account User - to act as itself for Cloud Tasks OIDC
|
|
bindings["act_as_self"] = serviceaccount.IAMBinding(
|
|
"karaoke-backend-act-as-self",
|
|
service_account_id=service_account.name,
|
|
role="roles/iam.serviceAccountUser",
|
|
members=[service_account.email.apply(lambda email: f"serviceAccount:{email}")],
|
|
)
|
|
|
|
# Cloud Run Jobs Developer - run encoding jobs
|
|
bindings["run_jobs_developer"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-run-jobs-developer",
|
|
project=PROJECT_ID,
|
|
role="roles/run.developer",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Cloud Monitoring Viewer - read quota metrics for YouTube API quota tracking
|
|
bindings["monitoring_viewer"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-monitoring-viewer",
|
|
project=PROJECT_ID,
|
|
role="roles/monitoring.viewer",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
# Cloud Logging Viewer - error monitor reads Cloud Logging for error detection
|
|
bindings["logging_viewer"] = gcp.projects.IAMMember(
|
|
"karaoke-backend-logging-viewer",
|
|
project=PROJECT_ID,
|
|
role="roles/logging.viewer",
|
|
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
|
)
|
|
|
|
return bindings
|
|
|
|
|
|
def grant_cloud_tasks_invoker_permission() -> gcp.projects.IAMMember:
|
|
"""
|
|
Grant Cloud Tasks service agent permission to invoke Cloud Run.
|
|
|
|
This allows Cloud Tasks to authenticate when calling Cloud Run endpoints.
|
|
|
|
Returns:
|
|
gcp.projects.IAMMember: The IAM binding resource.
|
|
"""
|
|
return gcp.projects.IAMMember(
|
|
"cloud-tasks-invoker",
|
|
project=PROJECT_ID,
|
|
role="roles/run.invoker",
|
|
member=f"serviceAccount:service-{get_project_number()}@gcp-sa-cloudtasks.iam.gserviceaccount.com",
|
|
)
|