mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(alerts): route non-signal Discord posts to #noise (v0.278.8) (#1186)
## Summary Andrew asked for #alerts to carry only real problems. Routine and informational posts now go to the new **#noise** channel. The GitHub org PR webhook was already moved to #noise separately. | Message | Before | After | |---|---|---| | Error monitor: New Error Pattern / Spike / Incident | #alerts | #alerts | | Error monitor: ✅ Patterns Auto-Resolved | #alerts | **#noise** | | Error monitor: 📊 Daily Error Digest | #alerts | **#noise** | | Nightly Backup Report (clean run) | #alerts | **#noise** | | Nightly backup failed/partial (title is now "🚨 Nightly Backup FAILED") | #alerts | #alerts | | ✅ E2E Daily Test PASSED | #alerts | **#noise** | | 🩹 E2E self-healed Stripe (auto-merge PR opened) | #alerts | **#noise** | | 🩹 self-heal PR needs CI approval click / could not open | #alerts | #alerts | | E2E / Tenant E2E failures, DR freshness stale, Job failed, Publish completeness SHADOW, Gemini unavailable, OAuth credential alerts, quarterly DR drill reminder | #alerts | #alerts (unchanged) | ## How the noise webhook is resolved - **Backend / error monitor and the backup function** read it at runtime: env `DISCORD_NOISE_WEBHOOK_URL` first, then the Secret Manager secret `discord-noise-webhook` (the name can be overridden with `DISCORD_NOISE_WEBHOOK_SECRET_NAME`). No Pulumi change is needed, because both service accounts already have project-wide `secretAccessor`. - **GitHub Actions** uses `secrets.DISCORD_NOISE_WEBHOOK_URL || secrets.DISCORD_ALERT_WEBHOOK_URL`. - **Nothing is silently dropped.** If the noise webhook can't be resolved, or a post to it fails, the message goes to #alerts instead. In the workflow this uses `curl -fsS` with a retry on the alert webhook. - The error monitor now rate-limits each channel separately (10 per run each), so noise can't use up the alert budget during an incident. ## Deploy notes - **Backup Cloud Function:** CI doesn't deploy its source. Run `infrastructure/functions/backup_to_aws/deploy.sh` after merge. - **Error monitor job:** runs `karaoke-backend:latest`, so it picks up the change once CI pushes the image. - **No `pulumi up` needed.** ## Testing - `tests/unit/services/error_monitor`: 376 passed. New tests cover noise vs alert routing, fallback when the noise webhook is missing or fails, separate rate limits, and secret resolution. - `backend/tests` (excluding emulator): 6418 passed, 36 skipped. - `infrastructure/functions/backup_to_aws`: 60 passed, including the new `test_discord_alert.py` and routing tests in `test_main.py`. These were run in a scratch venv because `functions_framework` isn't in the conda env, which was already the case before this change. - `actionlint` on `e2e-daily.yml`: no new findings. The only findings are info-level shellcheck notes that were already there. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_019pmG2XAgtUguEJWLFpkKHN Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
290f927175
commit
a6d4163d2e
14 files changed
+554
-57
No files matched your search
@@ -411,22 +411,31 @@ jobs:
|
||||
if: always() && steps.pr.outputs.unchanged != 'true'
|
||||
run: |
|
||||
PR="${{ steps.pr.outputs.pr_url }}"
|
||||
# #alerts = needs a human; #noise = informational (auto-merge handles it).
|
||||
# Noise falls back to the alert webhook if its secret is unset.
|
||||
ALERT_WEBHOOK="${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}"
|
||||
NOISE_WEBHOOK="${{ secrets.DISCORD_NOISE_WEBHOOK_URL || secrets.DISCORD_ALERT_WEBHOOK_URL }}"
|
||||
if [ -n "$PR" ] && [ "${{ steps.pr.outputs.ci_needs_approval }}" = "true" ]; then
|
||||
TITLE="🩹 E2E self-healed Stripe — PR needs a CI approval click"
|
||||
DESC="LLM fallback healed a Stripe Checkout change. The auto-merge PR's CI run awaits approval (open the PR → Checks → Approve and run): $PR"
|
||||
COLOR=16776960
|
||||
WEBHOOK="$ALERT_WEBHOOK"
|
||||
elif [ -n "$PR" ]; then
|
||||
TITLE="🩹 E2E self-healed a Stripe Checkout change"
|
||||
DESC="Stripe changed its checkout page; the LLM fallback got through and the purchase was verified. Auto-merge PR: $PR"
|
||||
COLOR=16776960
|
||||
WEBHOOK="$NOISE_WEBHOOK"
|
||||
else
|
||||
TITLE="⚠️ E2E self-heal PR could not be opened"
|
||||
DESC="The LLM fallback healed a Stripe step but opening the PR failed — see the run."
|
||||
COLOR=15105570
|
||||
WEBHOOK="$ALERT_WEBHOOK"
|
||||
fi
|
||||
curl -s -H "Content-Type: application/json" \
|
||||
-d "{\"embeds\": [{\"title\": \"$TITLE\", \"description\": \"$DESC\n\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"color\": $COLOR}]}" \
|
||||
"${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}"
|
||||
PAYLOAD="{\"embeds\": [{\"title\": \"$TITLE\", \"description\": \"$DESC\n\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"color\": $COLOR}]}"
|
||||
# A failed #noise post retries on #alerts so it is never silently lost.
|
||||
curl -fsS -H "Content-Type: application/json" -d "$PAYLOAD" "$WEBHOOK" \
|
||||
|| { [ "$WEBHOOK" != "$ALERT_WEBHOOK" ] && curl -s -H "Content-Type: application/json" -d "$PAYLOAD" "$ALERT_WEBHOOK"; } \
|
||||
|| true
|
||||
|
||||
# =========================================================================
|
||||
# Stage 2: Happy Path (uses credit from Stage 1)
|
||||
@@ -600,12 +609,17 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Discord notification (success)
|
||||
# Passes are not signal: they go to #noise (fallback: #alerts if the
|
||||
# noise secret is unset, so nothing is ever silently dropped).
|
||||
if: needs.e2e-happy-path.result == 'success'
|
||||
run: |
|
||||
STAGE1="${{ needs.e2e-credit-purchase.result || 'skipped' }}"
|
||||
curl -s -H "Content-Type: application/json" \
|
||||
-d "{\"embeds\": [{\"title\": \"✅ E2E Daily Test PASSED\", \"description\": \"Payment: ${STAGE1} | Happy Path: success\n\n**Run:** #${{ github.run_number }}\", \"color\": 3066993}]}" \
|
||||
"${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}"
|
||||
ALERT_WEBHOOK="${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}"
|
||||
NOISE_WEBHOOK="${{ secrets.DISCORD_NOISE_WEBHOOK_URL || secrets.DISCORD_ALERT_WEBHOOK_URL }}"
|
||||
PAYLOAD="{\"embeds\": [{\"title\": \"✅ E2E Daily Test PASSED\", \"description\": \"Payment: ${STAGE1} | Happy Path: success\n\n**Run:** #${{ github.run_number }}\", \"color\": 3066993}]}"
|
||||
curl -fsS -H "Content-Type: application/json" -d "$PAYLOAD" "$NOISE_WEBHOOK" \
|
||||
|| { [ "$NOISE_WEBHOOK" != "$ALERT_WEBHOOK" ] && curl -s -H "Content-Type: application/json" -d "$PAYLOAD" "$ALERT_WEBHOOK"; } \
|
||||
|| true
|
||||
|
||||
- name: Send email notification
|
||||
if: github.event_name == 'schedule'
|
||||
|
||||
@@ -178,6 +178,15 @@ def get_discord_webhook_secret_name() -> str:
|
||||
return os.environ.get("DISCORD_WEBHOOK_SECRET_NAME", "error-monitor-discord-webhook")
|
||||
|
||||
|
||||
def get_discord_noise_webhook_secret_name() -> str:
|
||||
"""Return the Secret Manager secret name for the #noise Discord webhook.
|
||||
|
||||
#alerts is reserved for real problems (new patterns, spikes, incidents);
|
||||
informational posts (auto-resolved summaries, the daily digest) go to #noise.
|
||||
"""
|
||||
return os.environ.get("DISCORD_NOISE_WEBHOOK_SECRET_NAME", "discord-noise-webhook")
|
||||
|
||||
|
||||
def get_digest_mode() -> bool:
|
||||
"""Return True if the monitor should run in digest mode (batch alerts instead of immediate).
|
||||
|
||||
|
||||
@@ -14,9 +14,12 @@ Usage:
|
||||
format_daily_digest,
|
||||
)
|
||||
|
||||
discord = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/...")
|
||||
msg = format_new_pattern_alert(...)
|
||||
discord.send_message(msg)
|
||||
discord = ErrorMonitorDiscord(
|
||||
webhook_url="https://discord.com/api/webhooks/...", # #alerts
|
||||
noise_webhook_url="https://discord.com/api/webhooks/...", # #noise
|
||||
)
|
||||
discord.send_message(format_new_pattern_alert(...)) # a real problem
|
||||
discord.send_noise_message(format_auto_resolved_alert(...)) # informational
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -272,22 +275,46 @@ def format_daily_digest(
|
||||
|
||||
|
||||
class ErrorMonitorDiscord:
|
||||
"""Webhook client for sending error monitor alerts to Discord.
|
||||
"""Webhook client for sending error monitor messages to Discord.
|
||||
|
||||
Enforces a per-run rate limit of MAX_DISCORD_MESSAGES_PER_RUN to avoid
|
||||
flooding the channel during high-volume incident windows.
|
||||
Two channels:
|
||||
|
||||
* **#alerts** (``webhook_url``, :meth:`send_message`) — signal only: real
|
||||
problems someone should look at (new patterns, spikes, incidents).
|
||||
* **#noise** (``noise_webhook_url``, :meth:`send_noise_message`) —
|
||||
informational posts (auto-resolved summaries, daily digest). If no noise
|
||||
webhook is configured, or a noise post fails, the message falls back to
|
||||
the alert webhook so it is never silently dropped.
|
||||
|
||||
Each channel has its own per-run rate limit of MAX_DISCORD_MESSAGES_PER_RUN,
|
||||
so noise can never eat the alert budget during an incident window.
|
||||
|
||||
Attributes:
|
||||
webhook_url: The Discord incoming webhook URL.
|
||||
messages_sent: Counter of messages attempted this run (success or failure).
|
||||
webhook_url: The #alerts Discord incoming webhook URL.
|
||||
noise_webhook_url: The #noise webhook URL (None → use webhook_url).
|
||||
messages_sent: #alerts messages attempted this run (success or failure).
|
||||
noise_messages_sent: #noise messages attempted this run.
|
||||
"""
|
||||
|
||||
def __init__(self, webhook_url: str) -> None:
|
||||
def __init__(self, webhook_url: str, noise_webhook_url: str | None = None) -> None:
|
||||
self.webhook_url = webhook_url
|
||||
self.noise_webhook_url = noise_webhook_url
|
||||
self.messages_sent: int = 0
|
||||
self.noise_messages_sent: int = 0
|
||||
|
||||
@staticmethod
|
||||
def _post(url: str, content: str) -> bool:
|
||||
try:
|
||||
response = requests.post(url, json={"content": content}, timeout=30)
|
||||
response.raise_for_status()
|
||||
logger.debug("Discord message sent successfully.")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("Failed to send Discord message: %s", exc)
|
||||
return False
|
||||
|
||||
def send_message(self, content: str) -> bool:
|
||||
"""Send a message to the configured Discord webhook.
|
||||
"""Send a message to the #alerts webhook (real problems only).
|
||||
|
||||
Rate-limits: if messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN, the
|
||||
message is silently skipped and False is returned without incrementing
|
||||
@@ -308,16 +335,33 @@ class ErrorMonitorDiscord:
|
||||
return False
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
self.webhook_url,
|
||||
json={"content": content},
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.debug("Discord message sent successfully.")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("Failed to send Discord message: %s", exc)
|
||||
return False
|
||||
return self._post(self.webhook_url, content)
|
||||
finally:
|
||||
self.messages_sent += 1
|
||||
|
||||
def send_noise_message(self, content: str) -> bool:
|
||||
"""Send an informational message to the #noise webhook.
|
||||
|
||||
Falls back to the #alerts webhook if no noise webhook is configured or
|
||||
the noise post fails. Rate-limited separately from :meth:`send_message`.
|
||||
|
||||
Returns:
|
||||
True if the message was delivered to either channel.
|
||||
"""
|
||||
if self.noise_messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN:
|
||||
logger.warning(
|
||||
"Discord noise rate limit reached (%d messages sent this run); skipping message.",
|
||||
self.noise_messages_sent,
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
noise_url = self.noise_webhook_url or self.webhook_url
|
||||
if self._post(noise_url, content):
|
||||
return True
|
||||
if noise_url != self.webhook_url:
|
||||
logger.warning("Noise webhook post failed; falling back to the alert webhook.")
|
||||
return self._post(self.webhook_url, content)
|
||||
return False
|
||||
finally:
|
||||
self.noise_messages_sent += 1
|
||||
@@ -33,6 +33,7 @@ from backend.services.error_monitor.config import (
|
||||
SPIKE_MIN_COUNT,
|
||||
SPIKE_MULTIPLIER,
|
||||
get_digest_mode,
|
||||
get_discord_noise_webhook_secret_name,
|
||||
get_discord_webhook_secret_name,
|
||||
get_llm_enabled,
|
||||
)
|
||||
@@ -279,7 +280,11 @@ class ErrorMonitor:
|
||||
def __init__(self) -> None:
|
||||
self.logging_client = cloud_logging.Client(project=GCP_PROJECT)
|
||||
self.firestore_adapter = ErrorPatternsAdapter()
|
||||
self.discord = ErrorMonitorDiscord(webhook_url=self._get_discord_webhook())
|
||||
alert_webhook = self._get_discord_webhook()
|
||||
self.discord = ErrorMonitorDiscord(
|
||||
webhook_url=alert_webhook,
|
||||
noise_webhook_url=self._get_discord_noise_webhook(alert_webhook),
|
||||
)
|
||||
|
||||
def _get_discord_webhook(self) -> str:
|
||||
"""Retrieve the Discord webhook URL.
|
||||
@@ -300,6 +305,33 @@ class ErrorMonitor:
|
||||
response = client.access_secret_version(name=secret_path)
|
||||
return response.payload.data.decode("utf-8").strip()
|
||||
|
||||
def _get_discord_noise_webhook(self, alert_webhook: str) -> str:
|
||||
"""Retrieve the #noise Discord webhook URL (informational posts).
|
||||
|
||||
``DISCORD_NOISE_WEBHOOK_URL`` env var first, then the Secret Manager
|
||||
secret ``discord-noise-webhook`` (override the name with
|
||||
``DISCORD_NOISE_WEBHOOK_SECRET_NAME``). If neither yields a URL, falls
|
||||
back to ``alert_webhook`` so informational posts are never dropped.
|
||||
"""
|
||||
url = (os.environ.get("DISCORD_NOISE_WEBHOOK_URL") or "").strip()
|
||||
if url:
|
||||
return url
|
||||
|
||||
secret_name = get_discord_noise_webhook_secret_name()
|
||||
try:
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
secret_path = f"projects/{GCP_PROJECT}/secrets/{secret_name}/versions/latest"
|
||||
response = client.access_secret_version(name=secret_path)
|
||||
url = response.payload.data.decode("utf-8").strip()
|
||||
except Exception as exc: # noqa: BLE001 - noise channel is optional
|
||||
logger.warning(
|
||||
"Could not read %s (%s); informational posts will go to the alert webhook",
|
||||
secret_name,
|
||||
exc,
|
||||
)
|
||||
return alert_webhook
|
||||
return url or alert_webhook
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ------------------------------------------------------------------
|
||||
@@ -601,8 +633,9 @@ class ErrorMonitor:
|
||||
)
|
||||
|
||||
if resolved:
|
||||
# Good news, not a problem → #noise (keeps #alerts pure signal).
|
||||
msg = format_auto_resolved_alert(resolved)
|
||||
self.discord.send_message(msg)
|
||||
self.discord.send_noise_message(msg)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Daily digest
|
||||
@@ -659,7 +692,8 @@ class ErrorMonitor:
|
||||
active_patterns=len(active_patterns),
|
||||
per_service_counts=per_service_counts,
|
||||
)
|
||||
self.discord.send_message(msg)
|
||||
# Informational summary → #noise.
|
||||
self.discord.send_noise_message(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+36
-1
@@ -56,7 +56,8 @@ Frontend clients ───────────────→ POST /api/clie
|
||||
│
|
||||
├─ Normalize & deduplicate
|
||||
├─ LLM incident grouping (Gemini Flash)
|
||||
├─ Discord alerts (new patterns, spikes, daily digest)
|
||||
├─ Discord #alerts (new patterns, spikes, incidents)
|
||||
├─ Discord #noise (auto-resolved summary, daily digest)
|
||||
├─ Firestore: error_patterns, error_incidents
|
||||
└─ Auto-resolve silent patterns
|
||||
|
||||
@@ -65,6 +66,40 @@ Design spec: docs/archive/2026-04-12-error-monitor-design.md
|
||||
Frontend crash reporting: docs/archive/2026-04-19-frontend-crash-reporting-plan.md
|
||||
```
|
||||
|
||||
#### Discord channels: #alerts = signal, #noise = everything else
|
||||
|
||||
`#alerts` is reserved for **actual problems someone should look at**. Anything
|
||||
routine/informational goes to `#noise`. When adding a new Discord post, pick the
|
||||
channel by asking "does this need a human?" — if not, it's noise.
|
||||
|
||||
| Message | Channel | Source |
|
||||
|---------|---------|--------|
|
||||
| 🔴 New Error Pattern / ⚠️ Spike / incident | #alerts | error monitor `send_message` |
|
||||
| ✅ Patterns Auto-Resolved, 📊 Daily Error Digest | #noise | error monitor `send_noise_message` |
|
||||
| 🔴 Job failed, 🟠 Audio download stalled, 🟠 Publish completeness SHADOW, 🟠 Gemini unavailable | #alerts | `backend/services/ops_alerts.py` |
|
||||
| OAuth credential invalid/expired (startup check, `/test-alert`) | #alerts | `backend/main.py`, `auth.py` |
|
||||
| Nightly Backup Report (clean run) | #noise | `infrastructure/functions/backup_to_aws` |
|
||||
| 🚨 Nightly Backup FAILED (any step errored) | #alerts | same |
|
||||
| 📋 Quarterly DR restore drill due | #alerts | same (`?mode=drill_reminder`, actionable) |
|
||||
| 🚨 DR backup is stale / monitor failed | `DR_MONITOR_DISCORD_WEBHOOK` | `.github/workflows/dr-backup-freshness.yml` (failures only) |
|
||||
| ✅ E2E Daily Test PASSED, 🩹 E2E self-healed (auto-merge PR opened) | #noise | `.github/workflows/e2e-daily.yml` |
|
||||
| 🚨 E2E Daily FAILED, 🩹 self-heal PR needs a CI approval click / could not open, ❌ Tenant E2E FAILED | #alerts | `e2e-daily.yml`, `e2e-tenant-daily.yml` |
|
||||
| GitHub PR opened/closed (org webhook) | #noise | GitHub org settings |
|
||||
| New YouTube video | #releases | `DEFAULT_DISCORD_WEBHOOK_URL` ← `discord-releases-webhook` |
|
||||
|
||||
How the webhooks resolve:
|
||||
|
||||
- **#alerts**: Secret Manager `discord-alert-webhook` (error monitor gets it as
|
||||
`DISCORD_WEBHOOK_URL` from Pulumi; the backup function likewise; GitHub Actions
|
||||
secret `DISCORD_ALERT_WEBHOOK_URL`).
|
||||
- **#noise**: Secret Manager `discord-noise-webhook`, read **at runtime** (env
|
||||
`DISCORD_NOISE_WEBHOOK_URL` overrides; secret name overridable with
|
||||
`DISCORD_NOISE_WEBHOOK_SECRET_NAME`) — no Pulumi env wiring needed. GitHub
|
||||
Actions secret `DISCORD_NOISE_WEBHOOK_URL`.
|
||||
- **Never silently dropped**: if the noise webhook can't be resolved, or a post
|
||||
to it fails, the message falls back to #alerts. The error monitor rate-limits
|
||||
each channel separately (10/run each), so noise can't consume the alert budget.
|
||||
|
||||
Frontend crashes are captured via `window.onerror`, `unhandledrejection`, Next.js
|
||||
`error.tsx`, and a `CrashReportBoundary` around the review UIs. They POST to
|
||||
`/api/client-errors`, which normalises and upserts into the same `error_patterns`
|
||||
|
||||
@@ -482,7 +482,7 @@ This monitor runs in GitHub, not GCP, so it survives a complete GCP project loss
|
||||
**Required GitHub secrets** (set via `gh secret set`):
|
||||
- `AWS_BACKUP_READONLY_ACCESS_KEY_ID` — IAM user with `s3:ListBucket` + `s3:GetObject` only on `nomadkaraoke-backup`
|
||||
- `AWS_BACKUP_READONLY_SECRET_ACCESS_KEY`
|
||||
- `DR_MONITOR_DISCORD_WEBHOOK` — Discord webhook URL (separate channel from the nightly success/failure alert is fine)
|
||||
- `DR_MONITOR_DISCORD_WEBHOOK` — Discord webhook URL (this monitor only posts on stale/failed, so #alerts is right). The nightly function itself posts a clean run's "Nightly Backup Report" to #noise (secret `discord-noise-webhook`, read at runtime) and a failed/partial run as "🚨 Nightly Backup FAILED" to #alerts — see ARCHITECTURE.md § Discord channels
|
||||
|
||||
**Manual setup (one-time, in AWS Console):**
|
||||
1. Create IAM user `nomadkaraoke-backup-monitor` (no console access, programmatic only)
|
||||
|
||||
@@ -1,18 +1,68 @@
|
||||
"""Discord webhook notifications for backup status."""
|
||||
"""Discord webhook notifications for backup status.
|
||||
|
||||
Two channels:
|
||||
|
||||
* **#alerts** (``DISCORD_WEBHOOK_URL`` env, from secret ``discord-alert-webhook``)
|
||||
— real problems only: a failed or partial backup run.
|
||||
* **#noise** (secret ``discord-noise-webhook``, read at runtime) — the routine
|
||||
"all good" nightly report. If the noise webhook can't be resolved, or a noise
|
||||
post fails, the message falls back to #alerts so nothing is silently dropped.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import requests
|
||||
from google.cloud import secretmanager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NOISE_WEBHOOK_SECRET_DEFAULT = "discord-noise-webhook"
|
||||
|
||||
def send_alert(webhook_url: str, title: str, fields: list[dict], success: bool = True):
|
||||
"""Send a Discord embed notification."""
|
||||
|
||||
def send_alert(webhook_url: str, title: str, fields: list[dict], success: bool = True) -> bool:
|
||||
"""Send a Discord embed notification. Returns True if delivered."""
|
||||
color = 0x00FF00 if success else 0xFF0000
|
||||
embed = {"title": title, "color": color, "fields": fields}
|
||||
|
||||
try:
|
||||
resp = requests.post(webhook_url, json={"embeds": [embed]}, timeout=10)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Discord alert: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_noise_webhook_url(project: str, fallback_url: str) -> str:
|
||||
"""Resolve the #noise webhook URL, falling back to ``fallback_url``.
|
||||
|
||||
``DISCORD_NOISE_WEBHOOK_URL`` env first, else Secret Manager secret
|
||||
``discord-noise-webhook`` (name overridable via
|
||||
``DISCORD_NOISE_WEBHOOK_SECRET_NAME``). Read at runtime rather than as a
|
||||
deploy-time secret env var, so no Pulumi change is needed; the function's
|
||||
SA holds project-wide ``secretmanager.secretAccessor``. Any error → fallback.
|
||||
"""
|
||||
url = os.environ.get("DISCORD_NOISE_WEBHOOK_URL", "").strip()
|
||||
if url:
|
||||
return url
|
||||
secret_id = os.environ.get("DISCORD_NOISE_WEBHOOK_SECRET_NAME", NOISE_WEBHOOK_SECRET_DEFAULT)
|
||||
try:
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
name = f"projects/{project}/secrets/{secret_id}/versions/latest"
|
||||
resp = client.access_secret_version(request={"name": name})
|
||||
url = resp.payload.data.decode("utf-8").strip()
|
||||
except Exception as e: # noqa: BLE001 — noise channel is optional
|
||||
logger.warning(f"Could not read {secret_id} ({e}); routine reports go to the alert webhook")
|
||||
return fallback_url
|
||||
return url or fallback_url
|
||||
|
||||
|
||||
def send_noise(noise_url: str, alert_url: str, title: str, fields: list[dict]) -> bool:
|
||||
"""Post an informational (success) embed to #noise; on failure retry on #alerts."""
|
||||
if noise_url and send_alert(webhook_url=noise_url, title=title, fields=fields, success=True):
|
||||
return True
|
||||
if alert_url and alert_url != noise_url:
|
||||
logger.warning("Noise webhook post failed; falling back to the alert webhook")
|
||||
return send_alert(webhook_url=alert_url, title=title, fields=fields, success=True)
|
||||
return False
|
||||
@@ -14,7 +14,7 @@ Nightly backup pipeline:
|
||||
bundle last reached S3; full re-bundle on the first Sunday of each month;
|
||||
survives loss of GitHub access, e.g. an account ban)
|
||||
5. Upload staging files to S3 (Firestore export held back except Sundays)
|
||||
6. Discord alert
|
||||
6. Discord report (success → #noise, failure → #alerts)
|
||||
|
||||
Triggered by Cloud Scheduler at 1:00 AM ET daily.
|
||||
|
||||
@@ -30,7 +30,7 @@ import os
|
||||
|
||||
import functions_framework
|
||||
|
||||
from discord_alert import send_alert
|
||||
from discord_alert import get_noise_webhook_url, send_alert, send_noise
|
||||
from firestore_export import export_firestore
|
||||
from bigquery_export import export_bigquery_tables
|
||||
from gcs_sync import sync_gcs_to_staging
|
||||
@@ -239,12 +239,21 @@ def backup_to_aws(request):
|
||||
if isinstance(value, str):
|
||||
fields.append({"name": key, "value": value, "inline": True})
|
||||
|
||||
if DISCORD_WEBHOOK_URL:
|
||||
send_alert(
|
||||
webhook_url=DISCORD_WEBHOOK_URL,
|
||||
# #alerts is signal only: a clean run is routine → #noise; any failure or
|
||||
# partial run stays in #alerts.
|
||||
if success:
|
||||
send_noise(
|
||||
noise_url=get_noise_webhook_url(GCP_PROJECT, DISCORD_WEBHOOK_URL),
|
||||
alert_url=DISCORD_WEBHOOK_URL,
|
||||
title="Nightly Backup Report",
|
||||
fields=fields,
|
||||
success=success,
|
||||
)
|
||||
elif DISCORD_WEBHOOK_URL:
|
||||
send_alert(
|
||||
webhook_url=DISCORD_WEBHOOK_URL,
|
||||
title="🚨 Nightly Backup FAILED",
|
||||
fields=fields,
|
||||
success=False,
|
||||
)
|
||||
|
||||
status_code = 200 if success else 500
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Tests for discord_alert: #alerts vs #noise webhook resolution + delivery."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import requests
|
||||
|
||||
import discord_alert
|
||||
|
||||
ALERT = "https://discord.com/api/webhooks/1/alert"
|
||||
NOISE = "https://discord.com/api/webhooks/2/noise"
|
||||
|
||||
|
||||
def _ok():
|
||||
resp = MagicMock()
|
||||
resp.raise_for_status.return_value = None
|
||||
return resp
|
||||
|
||||
|
||||
def test_send_alert_returns_true_on_success():
|
||||
with patch("discord_alert.requests.post", return_value=_ok()) as post:
|
||||
assert discord_alert.send_alert(ALERT, "t", [], success=True) is True
|
||||
assert post.call_args.args[0] == ALERT
|
||||
|
||||
|
||||
def test_send_alert_returns_false_on_failure():
|
||||
bad = MagicMock()
|
||||
bad.raise_for_status.side_effect = requests.HTTPError("404")
|
||||
with patch("discord_alert.requests.post", return_value=bad):
|
||||
assert discord_alert.send_alert(ALERT, "t", [], success=False) is False
|
||||
|
||||
|
||||
def test_noise_url_env_var_wins(monkeypatch):
|
||||
monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_URL", NOISE)
|
||||
with patch("discord_alert.secretmanager") as sm:
|
||||
assert discord_alert.get_noise_webhook_url("proj", ALERT) == NOISE
|
||||
sm.SecretManagerServiceClient.assert_not_called()
|
||||
|
||||
|
||||
def test_noise_url_read_from_secret_manager(monkeypatch):
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False)
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", raising=False)
|
||||
with patch("discord_alert.secretmanager") as sm:
|
||||
client = sm.SecretManagerServiceClient.return_value
|
||||
client.access_secret_version.return_value.payload.data = (NOISE + "\n").encode()
|
||||
assert discord_alert.get_noise_webhook_url("proj", ALERT) == NOISE
|
||||
name = client.access_secret_version.call_args.kwargs["request"]["name"]
|
||||
assert name == "projects/proj/secrets/discord-noise-webhook/versions/latest"
|
||||
|
||||
|
||||
def test_noise_url_falls_back_to_alert_on_secret_error(monkeypatch):
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False)
|
||||
with patch("discord_alert.secretmanager") as sm:
|
||||
sm.SecretManagerServiceClient.return_value.access_secret_version.side_effect = RuntimeError("403")
|
||||
assert discord_alert.get_noise_webhook_url("proj", ALERT) == ALERT
|
||||
|
||||
|
||||
def test_noise_url_falls_back_to_alert_when_secret_empty(monkeypatch):
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False)
|
||||
with patch("discord_alert.secretmanager") as sm:
|
||||
sm.SecretManagerServiceClient.return_value.access_secret_version.return_value.payload.data = b" "
|
||||
assert discord_alert.get_noise_webhook_url("proj", ALERT) == ALERT
|
||||
|
||||
|
||||
def test_send_noise_posts_to_noise_webhook():
|
||||
with patch("discord_alert.send_alert", return_value=True) as send:
|
||||
assert discord_alert.send_noise(NOISE, ALERT, "t", []) is True
|
||||
send.assert_called_once_with(webhook_url=NOISE, title="t", fields=[], success=True)
|
||||
|
||||
|
||||
def test_send_noise_falls_back_to_alert_webhook_on_failure():
|
||||
with patch("discord_alert.send_alert", side_effect=[False, True]) as send:
|
||||
assert discord_alert.send_noise(NOISE, ALERT, "t", []) is True
|
||||
assert [c.kwargs["webhook_url"] for c in send.call_args_list] == [NOISE, ALERT]
|
||||
|
||||
|
||||
def test_send_noise_no_retry_when_noise_is_alert():
|
||||
with patch("discord_alert.send_alert", return_value=False) as send:
|
||||
assert discord_alert.send_noise(ALERT, ALERT, "t", []) is False
|
||||
send.assert_called_once()
|
||||
@@ -25,7 +25,8 @@ def _run(today):
|
||||
patch("main.export_secrets", return_value="sec"), \
|
||||
patch("main._run_git_repos_backup", return_value="git") as git, \
|
||||
patch("main.upload_staging_to_s3", return_value="s3") as upload, \
|
||||
patch("main.send_alert"):
|
||||
patch("main.send_alert"), \
|
||||
patch("main.send_noise"):
|
||||
main.backup_to_aws(request)
|
||||
return git, upload
|
||||
|
||||
@@ -73,9 +74,55 @@ def test_job_files_bucket_is_not_synced_off_site():
|
||||
patch("main.export_secrets", return_value="sec") as secrets, \
|
||||
patch("main._run_git_repos_backup", return_value="git"), \
|
||||
patch("main.upload_staging_to_s3", return_value="s3"), \
|
||||
patch("main.send_alert"):
|
||||
patch("main.send_alert"), \
|
||||
patch("main.send_noise"):
|
||||
main.backup_to_aws(request)
|
||||
sources = [c.kwargs["source_bucket"] for c in sync.call_args_list]
|
||||
assert sources == ["nomadkaraoke-kn-data"]
|
||||
assert all(c.kwargs["staging_prefix"] != "gcs/job-files/" for c in sync.call_args_list)
|
||||
secrets.assert_called_once()
|
||||
|
||||
|
||||
# --- #alerts vs #noise routing of the nightly report --------------------------
|
||||
|
||||
ALERT = "https://discord.com/api/webhooks/1/alert"
|
||||
NOISE = "https://discord.com/api/webhooks/2/noise"
|
||||
|
||||
|
||||
def _run_report(fail_s3=False):
|
||||
_FakeDate.today_value = datetime.date(2026, 9, 29)
|
||||
request = MagicMock()
|
||||
request.args = {}
|
||||
upload_kwargs = {"side_effect": RuntimeError("boom")} if fail_s3 else {"return_value": "s3"}
|
||||
with patch("main.datetime.date", _FakeDate), \
|
||||
patch("main.DISCORD_WEBHOOK_URL", ALERT), \
|
||||
patch("main.get_noise_webhook_url", return_value=NOISE), \
|
||||
patch("main.export_firestore", return_value="fs"), \
|
||||
patch("main.export_bigquery_tables", return_value={}), \
|
||||
patch("main.sync_gcs_to_staging", return_value="gcs"), \
|
||||
patch("main.export_secrets", return_value="sec"), \
|
||||
patch("main.upload_staging_to_s3", **upload_kwargs), \
|
||||
patch("main.send_alert") as alert, \
|
||||
patch("main.send_noise") as noise:
|
||||
_, status = main.backup_to_aws(request)
|
||||
return status, alert, noise
|
||||
|
||||
|
||||
def test_successful_report_goes_to_noise():
|
||||
status, alert, noise = _run_report()
|
||||
assert status == 200
|
||||
alert.assert_not_called()
|
||||
noise.assert_called_once()
|
||||
assert noise.call_args.kwargs["noise_url"] == NOISE
|
||||
assert noise.call_args.kwargs["alert_url"] == ALERT
|
||||
assert noise.call_args.kwargs["title"] == "Nightly Backup Report"
|
||||
|
||||
|
||||
def test_failed_report_goes_to_alerts():
|
||||
status, alert, noise = _run_report(fail_s3=True)
|
||||
assert status == 500
|
||||
noise.assert_not_called()
|
||||
alert.assert_called_once()
|
||||
assert alert.call_args.kwargs["webhook_url"] == ALERT
|
||||
assert alert.call_args.kwargs["success"] is False
|
||||
assert "FAILED" in alert.call_args.kwargs["title"]
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.278.7"
|
||||
version = "0.278.8"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -274,3 +274,17 @@ class TestHelperFunctions:
|
||||
from backend.services.error_monitor import config as cfg
|
||||
importlib.reload(cfg)
|
||||
assert cfg.get_digest_mode() is True
|
||||
|
||||
|
||||
class TestNoiseWebhookSecretName:
|
||||
def test_default_noise_secret_name(self, monkeypatch):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", raising=False)
|
||||
assert config.get_discord_noise_webhook_secret_name() == "discord-noise-webhook"
|
||||
|
||||
def test_noise_secret_name_env_override(self, monkeypatch):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", "other-noise")
|
||||
assert config.get_discord_noise_webhook_secret_name() == "other-noise"
|
||||
@@ -617,3 +617,71 @@ class TestErrorMonitorDiscordSendMessage:
|
||||
client.send_message("Skipped")
|
||||
# Counter should not go beyond limit when rate-limited
|
||||
assert client.messages_sent == MAX_DISCORD_MESSAGES_PER_RUN
|
||||
|
||||
|
||||
class TestErrorMonitorDiscordNoiseChannel:
|
||||
"""#alerts = signal only; informational posts go to the #noise webhook."""
|
||||
|
||||
ALERT = "https://discord.com/api/webhooks/1/alert"
|
||||
NOISE = "https://discord.com/api/webhooks/2/noise"
|
||||
|
||||
def _ok(self):
|
||||
resp = mock.MagicMock()
|
||||
resp.raise_for_status.return_value = None
|
||||
return resp
|
||||
|
||||
def test_send_noise_message_posts_to_noise_webhook(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE)
|
||||
with mock.patch("requests.post", return_value=self._ok()) as mock_post:
|
||||
assert client.send_noise_message("fyi") is True
|
||||
mock_post.assert_called_once_with(self.NOISE, json={"content": "fyi"}, timeout=30)
|
||||
|
||||
def test_send_message_still_posts_to_alert_webhook(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE)
|
||||
with mock.patch("requests.post", return_value=self._ok()) as mock_post:
|
||||
assert client.send_message("problem") is True
|
||||
mock_post.assert_called_once_with(self.ALERT, json={"content": "problem"}, timeout=30)
|
||||
|
||||
def test_noise_without_noise_webhook_falls_back_to_alert_webhook(self):
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url=self.ALERT)
|
||||
with mock.patch("requests.post", return_value=self._ok()) as mock_post:
|
||||
assert client.send_noise_message("fyi") is True
|
||||
mock_post.assert_called_once_with(self.ALERT, json={"content": "fyi"}, timeout=30)
|
||||
|
||||
def test_noise_delivery_failure_retries_on_alert_webhook(self):
|
||||
"""A broken noise webhook must never silently drop the message."""
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
bad = mock.MagicMock()
|
||||
bad.raise_for_status.side_effect = requests.HTTPError("404 Unknown Webhook")
|
||||
client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE)
|
||||
with mock.patch("requests.post", side_effect=[bad, self._ok()]) as mock_post:
|
||||
assert client.send_noise_message("fyi") is True
|
||||
assert [c.args[0] for c in mock_post.call_args_list] == [self.NOISE, self.ALERT]
|
||||
|
||||
def test_noise_messages_do_not_consume_alert_budget(self):
|
||||
from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE)
|
||||
with mock.patch("requests.post", return_value=self._ok()):
|
||||
for _ in range(MAX_DISCORD_MESSAGES_PER_RUN):
|
||||
client.send_noise_message("fyi")
|
||||
assert client.messages_sent == 0
|
||||
assert client.send_message("problem") is True
|
||||
|
||||
def test_noise_messages_are_rate_limited_separately(self):
|
||||
from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN
|
||||
from backend.services.error_monitor.discord import ErrorMonitorDiscord
|
||||
|
||||
client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE)
|
||||
client.noise_messages_sent = MAX_DISCORD_MESSAGES_PER_RUN
|
||||
with mock.patch("requests.post") as mock_post:
|
||||
assert client.send_noise_message("fyi") is False
|
||||
mock_post.assert_not_called()
|
||||
@@ -558,6 +558,7 @@ class TestRunDailyDigest:
|
||||
m.firestore_adapter = MagicMock()
|
||||
m.discord = MagicMock()
|
||||
m.discord.messages_sent = 0
|
||||
m.discord.noise_messages_sent = 0
|
||||
m.logger = MagicMock()
|
||||
return m
|
||||
|
||||
@@ -569,12 +570,12 @@ class TestRunDailyDigest:
|
||||
{"service": "video-worker", "total_count": 5, "status": "acknowledged"},
|
||||
]
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
monitor.discord.send_noise_message.return_value = True
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert monitor.discord.send_noise_message.call_count == 1
|
||||
sent_content = monitor.discord.send_noise_message.call_args[0][0]
|
||||
assert "📊" in sent_content
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
@@ -586,11 +587,11 @@ class TestRunDailyDigest:
|
||||
{"service": "svc-beta", "total_count": 2, "status": "new"},
|
||||
]
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
monitor.discord.send_noise_message.return_value = True
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
sent_content = monitor.discord.send_noise_message.call_args[0][0]
|
||||
# Both services should be mentioned in the digest
|
||||
assert "svc-alpha" in sent_content
|
||||
assert "svc-beta" in sent_content
|
||||
@@ -600,15 +601,29 @@ class TestRunDailyDigest:
|
||||
"""_run_daily_digest should still send a digest even when there are no patterns."""
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
monitor.discord.send_noise_message.return_value = True
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert monitor.discord.send_noise_message.call_count == 1
|
||||
sent_content = monitor.discord.send_noise_message.call_args[0][0]
|
||||
assert "📊" in sent_content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False)
|
||||
def test_digest_goes_to_noise_not_alerts(self, mock_llm, monitor):
|
||||
"""The digest is informational: it must never land in #alerts."""
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
|
||||
monitor._run_daily_digest()
|
||||
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
monitor.discord.send_noise_message.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: PatternData construction (P0)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -911,7 +926,7 @@ class TestAutoResolveAlert:
|
||||
}
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern]
|
||||
monitor.firestore_adapter.check_auto_resolve.return_value = 25.5 # hours silent
|
||||
monitor.discord.send_message.return_value = True
|
||||
monitor.discord.send_noise_message.return_value = True
|
||||
|
||||
monitor._check_auto_resolve()
|
||||
|
||||
@@ -921,8 +936,8 @@ class TestAutoResolveAlert:
|
||||
)
|
||||
|
||||
# A Discord message must have been sent
|
||||
assert monitor.discord.send_message.call_count == 1
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
assert monitor.discord.send_noise_message.call_count == 1
|
||||
sent_content = monitor.discord.send_noise_message.call_args[0][0]
|
||||
assert "✅" in sent_content
|
||||
|
||||
def test_auto_resolve_no_alert_when_no_patterns_qualify(self, monitor):
|
||||
@@ -941,6 +956,7 @@ class TestAutoResolveAlert:
|
||||
|
||||
monitor.firestore_adapter.auto_resolve_pattern.assert_not_called()
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
monitor.discord.send_noise_message.assert_not_called()
|
||||
|
||||
def test_auto_resolve_via_monitor_cycle(self, monitor):
|
||||
"""_run_monitor_cycle with no errors should still trigger the auto-resolve check."""
|
||||
@@ -958,10 +974,88 @@ class TestAutoResolveAlert:
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern]
|
||||
monitor.firestore_adapter.check_auto_resolve.return_value = 12.0
|
||||
monitor.firestore_adapter.get_active_patterns.return_value = []
|
||||
monitor.discord.send_message.return_value = True
|
||||
monitor.discord.send_noise_message.return_value = True
|
||||
|
||||
monitor._run_monitor_cycle()
|
||||
|
||||
monitor.firestore_adapter.auto_resolve_pattern.assert_called_once()
|
||||
sent_content = monitor.discord.send_message.call_args[0][0]
|
||||
sent_content = monitor.discord.send_noise_message.call_args[0][0]
|
||||
assert "✅" in sent_content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: #alerts vs #noise routing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoResolveRoutesToNoise:
|
||||
def test_auto_resolved_summary_goes_to_noise(self):
|
||||
monitor = _make_monitor_with_mocks()
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [
|
||||
{"pattern_id": "p1", "service": "karaoke-backend", "normalized_message": "boom"}
|
||||
]
|
||||
monitor.firestore_adapter.check_auto_resolve.return_value = 48.0
|
||||
|
||||
monitor._check_auto_resolve()
|
||||
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
monitor.discord.send_noise_message.assert_called_once()
|
||||
assert "Auto-Resolved" in monitor.discord.send_noise_message.call_args[0][0]
|
||||
|
||||
def test_nothing_resolved_sends_nothing(self):
|
||||
monitor = _make_monitor_with_mocks()
|
||||
monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = []
|
||||
|
||||
monitor._check_auto_resolve()
|
||||
|
||||
monitor.discord.send_message.assert_not_called()
|
||||
monitor.discord.send_noise_message.assert_not_called()
|
||||
|
||||
|
||||
class TestNoiseWebhookResolution:
|
||||
ALERT = "https://discord.com/api/webhooks/1/alert"
|
||||
NOISE = "https://discord.com/api/webhooks/2/noise"
|
||||
|
||||
def _monitor(self):
|
||||
ErrorMonitor = _import_error_monitor_class()
|
||||
return ErrorMonitor.__new__(ErrorMonitor)
|
||||
|
||||
def test_env_var_wins(self, monkeypatch):
|
||||
monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_URL", self.NOISE)
|
||||
with patch("backend.services.error_monitor.monitor.secretmanager") as sm:
|
||||
assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.NOISE
|
||||
sm.SecretManagerServiceClient.assert_not_called()
|
||||
|
||||
def test_reads_noise_secret_from_secret_manager(self, monkeypatch):
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False)
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", raising=False)
|
||||
with patch("backend.services.error_monitor.monitor.secretmanager") as sm:
|
||||
client = sm.SecretManagerServiceClient.return_value
|
||||
client.access_secret_version.return_value.payload.data = (self.NOISE + "\n").encode()
|
||||
assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.NOISE
|
||||
name = client.access_secret_version.call_args.kwargs["name"]
|
||||
assert name.endswith("/secrets/discord-noise-webhook/versions/latest")
|
||||
|
||||
def test_falls_back_to_alert_webhook_when_secret_unavailable(self, monkeypatch):
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False)
|
||||
with patch("backend.services.error_monitor.monitor.secretmanager") as sm:
|
||||
sm.SecretManagerServiceClient.return_value.access_secret_version.side_effect = (
|
||||
RuntimeError("403 permission denied")
|
||||
)
|
||||
assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.ALERT
|
||||
|
||||
def test_falls_back_to_alert_webhook_when_secret_empty(self, monkeypatch):
|
||||
monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False)
|
||||
with patch("backend.services.error_monitor.monitor.secretmanager") as sm:
|
||||
client = sm.SecretManagerServiceClient.return_value
|
||||
client.access_secret_version.return_value.payload.data = b" "
|
||||
assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.ALERT
|
||||
|
||||
def test_init_wires_both_webhooks(self, monkeypatch):
|
||||
monkeypatch.setenv("DISCORD_WEBHOOK_URL", self.ALERT)
|
||||
monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_URL", self.NOISE)
|
||||
m = _import_monitor_module()
|
||||
with patch.object(m.cloud_logging, "Client"), patch.object(m, "ErrorPatternsAdapter"):
|
||||
monitor = m.ErrorMonitor()
|
||||
assert monitor.discord.webhook_url == self.ALERT
|
||||
assert monitor.discord.noise_webhook_url == self.NOISE
|
||||
Reference in new issue
Block a user