diff --git a/.github/workflows/e2e-daily.yml b/.github/workflows/e2e-daily.yml index 0ac2e595..0d8f9e57 100644 --- a/.github/workflows/e2e-daily.yml +++ b/.github/workflows/e2e-daily.yml @@ -411,22 +411,31 @@ jobs: if: always() && steps.pr.outputs.unchanged != 'true' run: | PR="${{ steps.pr.outputs.pr_url }}" + # #alerts = needs a human; #noise = informational (auto-merge handles it). + # Noise falls back to the alert webhook if its secret is unset. + ALERT_WEBHOOK="${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}" + NOISE_WEBHOOK="${{ secrets.DISCORD_NOISE_WEBHOOK_URL || secrets.DISCORD_ALERT_WEBHOOK_URL }}" if [ -n "$PR" ] && [ "${{ steps.pr.outputs.ci_needs_approval }}" = "true" ]; then TITLE="๐Ÿฉน E2E self-healed Stripe โ€” PR needs a CI approval click" DESC="LLM fallback healed a Stripe Checkout change. The auto-merge PR's CI run awaits approval (open the PR โ†’ Checks โ†’ Approve and run): $PR" COLOR=16776960 + WEBHOOK="$ALERT_WEBHOOK" elif [ -n "$PR" ]; then TITLE="๐Ÿฉน E2E self-healed a Stripe Checkout change" DESC="Stripe changed its checkout page; the LLM fallback got through and the purchase was verified. Auto-merge PR: $PR" COLOR=16776960 + WEBHOOK="$NOISE_WEBHOOK" else TITLE="โš ๏ธ E2E self-heal PR could not be opened" DESC="The LLM fallback healed a Stripe step but opening the PR failed โ€” see the run." COLOR=15105570 + WEBHOOK="$ALERT_WEBHOOK" fi - curl -s -H "Content-Type: application/json" \ - -d "{\"embeds\": [{\"title\": \"$TITLE\", \"description\": \"$DESC\n\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"color\": $COLOR}]}" \ - "${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}" + PAYLOAD="{\"embeds\": [{\"title\": \"$TITLE\", \"description\": \"$DESC\n\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"color\": $COLOR}]}" + # A failed #noise post retries on #alerts so it is never silently lost. + curl -fsS -H "Content-Type: application/json" -d "$PAYLOAD" "$WEBHOOK" \ + || { [ "$WEBHOOK" != "$ALERT_WEBHOOK" ] && curl -s -H "Content-Type: application/json" -d "$PAYLOAD" "$ALERT_WEBHOOK"; } \ + || true # ========================================================================= # Stage 2: Happy Path (uses credit from Stage 1) @@ -600,12 +609,17 @@ jobs: steps: - name: Discord notification (success) + # Passes are not signal: they go to #noise (fallback: #alerts if the + # noise secret is unset, so nothing is ever silently dropped). if: needs.e2e-happy-path.result == 'success' run: | STAGE1="${{ needs.e2e-credit-purchase.result || 'skipped' }}" - curl -s -H "Content-Type: application/json" \ - -d "{\"embeds\": [{\"title\": \"โœ… E2E Daily Test PASSED\", \"description\": \"Payment: ${STAGE1} | Happy Path: success\n\n**Run:** #${{ github.run_number }}\", \"color\": 3066993}]}" \ - "${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}" + ALERT_WEBHOOK="${{ secrets.DISCORD_ALERT_WEBHOOK_URL }}" + NOISE_WEBHOOK="${{ secrets.DISCORD_NOISE_WEBHOOK_URL || secrets.DISCORD_ALERT_WEBHOOK_URL }}" + PAYLOAD="{\"embeds\": [{\"title\": \"โœ… E2E Daily Test PASSED\", \"description\": \"Payment: ${STAGE1} | Happy Path: success\n\n**Run:** #${{ github.run_number }}\", \"color\": 3066993}]}" + curl -fsS -H "Content-Type: application/json" -d "$PAYLOAD" "$NOISE_WEBHOOK" \ + || { [ "$NOISE_WEBHOOK" != "$ALERT_WEBHOOK" ] && curl -s -H "Content-Type: application/json" -d "$PAYLOAD" "$ALERT_WEBHOOK"; } \ + || true - name: Send email notification if: github.event_name == 'schedule' diff --git a/backend/services/error_monitor/config.py b/backend/services/error_monitor/config.py index 108d562e..16cbdc97 100644 --- a/backend/services/error_monitor/config.py +++ b/backend/services/error_monitor/config.py @@ -178,6 +178,15 @@ def get_discord_webhook_secret_name() -> str: return os.environ.get("DISCORD_WEBHOOK_SECRET_NAME", "error-monitor-discord-webhook") +def get_discord_noise_webhook_secret_name() -> str: + """Return the Secret Manager secret name for the #noise Discord webhook. + + #alerts is reserved for real problems (new patterns, spikes, incidents); + informational posts (auto-resolved summaries, the daily digest) go to #noise. + """ + return os.environ.get("DISCORD_NOISE_WEBHOOK_SECRET_NAME", "discord-noise-webhook") + + def get_digest_mode() -> bool: """Return True if the monitor should run in digest mode (batch alerts instead of immediate). diff --git a/backend/services/error_monitor/discord.py b/backend/services/error_monitor/discord.py index 6f5c52f1..547deaa5 100644 --- a/backend/services/error_monitor/discord.py +++ b/backend/services/error_monitor/discord.py @@ -14,9 +14,12 @@ Usage: format_daily_digest, ) - discord = ErrorMonitorDiscord(webhook_url="https://discord.com/api/webhooks/...") - msg = format_new_pattern_alert(...) - discord.send_message(msg) + discord = ErrorMonitorDiscord( + webhook_url="https://discord.com/api/webhooks/...", # #alerts + noise_webhook_url="https://discord.com/api/webhooks/...", # #noise + ) + discord.send_message(format_new_pattern_alert(...)) # a real problem + discord.send_noise_message(format_auto_resolved_alert(...)) # informational """ import logging @@ -272,22 +275,46 @@ def format_daily_digest( class ErrorMonitorDiscord: - """Webhook client for sending error monitor alerts to Discord. + """Webhook client for sending error monitor messages to Discord. - Enforces a per-run rate limit of MAX_DISCORD_MESSAGES_PER_RUN to avoid - flooding the channel during high-volume incident windows. + Two channels: + + * **#alerts** (``webhook_url``, :meth:`send_message`) โ€” signal only: real + problems someone should look at (new patterns, spikes, incidents). + * **#noise** (``noise_webhook_url``, :meth:`send_noise_message`) โ€” + informational posts (auto-resolved summaries, daily digest). If no noise + webhook is configured, or a noise post fails, the message falls back to + the alert webhook so it is never silently dropped. + + Each channel has its own per-run rate limit of MAX_DISCORD_MESSAGES_PER_RUN, + so noise can never eat the alert budget during an incident window. Attributes: - webhook_url: The Discord incoming webhook URL. - messages_sent: Counter of messages attempted this run (success or failure). + webhook_url: The #alerts Discord incoming webhook URL. + noise_webhook_url: The #noise webhook URL (None โ†’ use webhook_url). + messages_sent: #alerts messages attempted this run (success or failure). + noise_messages_sent: #noise messages attempted this run. """ - def __init__(self, webhook_url: str) -> None: + def __init__(self, webhook_url: str, noise_webhook_url: str | None = None) -> None: self.webhook_url = webhook_url + self.noise_webhook_url = noise_webhook_url self.messages_sent: int = 0 + self.noise_messages_sent: int = 0 + + @staticmethod + def _post(url: str, content: str) -> bool: + try: + response = requests.post(url, json={"content": content}, timeout=30) + response.raise_for_status() + logger.debug("Discord message sent successfully.") + return True + except Exception as exc: + logger.error("Failed to send Discord message: %s", exc) + return False def send_message(self, content: str) -> bool: - """Send a message to the configured Discord webhook. + """Send a message to the #alerts webhook (real problems only). Rate-limits: if messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN, the message is silently skipped and False is returned without incrementing @@ -308,16 +335,33 @@ class ErrorMonitorDiscord: return False try: - response = requests.post( - self.webhook_url, - json={"content": content}, - timeout=30, - ) - response.raise_for_status() - logger.debug("Discord message sent successfully.") - return True - except Exception as exc: - logger.error("Failed to send Discord message: %s", exc) - return False + return self._post(self.webhook_url, content) finally: self.messages_sent += 1 + + def send_noise_message(self, content: str) -> bool: + """Send an informational message to the #noise webhook. + + Falls back to the #alerts webhook if no noise webhook is configured or + the noise post fails. Rate-limited separately from :meth:`send_message`. + + Returns: + True if the message was delivered to either channel. + """ + if self.noise_messages_sent >= MAX_DISCORD_MESSAGES_PER_RUN: + logger.warning( + "Discord noise rate limit reached (%d messages sent this run); skipping message.", + self.noise_messages_sent, + ) + return False + + try: + noise_url = self.noise_webhook_url or self.webhook_url + if self._post(noise_url, content): + return True + if noise_url != self.webhook_url: + logger.warning("Noise webhook post failed; falling back to the alert webhook.") + return self._post(self.webhook_url, content) + return False + finally: + self.noise_messages_sent += 1 diff --git a/backend/services/error_monitor/monitor.py b/backend/services/error_monitor/monitor.py index fe49e527..4edbc7b3 100644 --- a/backend/services/error_monitor/monitor.py +++ b/backend/services/error_monitor/monitor.py @@ -33,6 +33,7 @@ from backend.services.error_monitor.config import ( SPIKE_MIN_COUNT, SPIKE_MULTIPLIER, get_digest_mode, + get_discord_noise_webhook_secret_name, get_discord_webhook_secret_name, get_llm_enabled, ) @@ -279,7 +280,11 @@ class ErrorMonitor: def __init__(self) -> None: self.logging_client = cloud_logging.Client(project=GCP_PROJECT) self.firestore_adapter = ErrorPatternsAdapter() - self.discord = ErrorMonitorDiscord(webhook_url=self._get_discord_webhook()) + alert_webhook = self._get_discord_webhook() + self.discord = ErrorMonitorDiscord( + webhook_url=alert_webhook, + noise_webhook_url=self._get_discord_noise_webhook(alert_webhook), + ) def _get_discord_webhook(self) -> str: """Retrieve the Discord webhook URL. @@ -300,6 +305,33 @@ class ErrorMonitor: response = client.access_secret_version(name=secret_path) return response.payload.data.decode("utf-8").strip() + def _get_discord_noise_webhook(self, alert_webhook: str) -> str: + """Retrieve the #noise Discord webhook URL (informational posts). + + ``DISCORD_NOISE_WEBHOOK_URL`` env var first, then the Secret Manager + secret ``discord-noise-webhook`` (override the name with + ``DISCORD_NOISE_WEBHOOK_SECRET_NAME``). If neither yields a URL, falls + back to ``alert_webhook`` so informational posts are never dropped. + """ + url = (os.environ.get("DISCORD_NOISE_WEBHOOK_URL") or "").strip() + if url: + return url + + secret_name = get_discord_noise_webhook_secret_name() + try: + client = secretmanager.SecretManagerServiceClient() + secret_path = f"projects/{GCP_PROJECT}/secrets/{secret_name}/versions/latest" + response = client.access_secret_version(name=secret_path) + url = response.payload.data.decode("utf-8").strip() + except Exception as exc: # noqa: BLE001 - noise channel is optional + logger.warning( + "Could not read %s (%s); informational posts will go to the alert webhook", + secret_name, + exc, + ) + return alert_webhook + return url or alert_webhook + # ------------------------------------------------------------------ # Main entry point # ------------------------------------------------------------------ @@ -601,8 +633,9 @@ class ErrorMonitor: ) if resolved: + # Good news, not a problem โ†’ #noise (keeps #alerts pure signal). msg = format_auto_resolved_alert(resolved) - self.discord.send_message(msg) + self.discord.send_noise_message(msg) # ------------------------------------------------------------------ # Daily digest @@ -659,7 +692,8 @@ class ErrorMonitor: active_patterns=len(active_patterns), per_service_counts=per_service_counts, ) - self.discord.send_message(msg) + # Informational summary โ†’ #noise. + self.discord.send_noise_message(msg) # --------------------------------------------------------------------------- diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 96d0abb3..81d37000 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -56,7 +56,8 @@ Frontend clients โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ†’ POST /api/clie โ”‚ โ”œโ”€ Normalize & deduplicate โ”œโ”€ LLM incident grouping (Gemini Flash) - โ”œโ”€ Discord alerts (new patterns, spikes, daily digest) + โ”œโ”€ Discord #alerts (new patterns, spikes, incidents) + โ”œโ”€ Discord #noise (auto-resolved summary, daily digest) โ”œโ”€ Firestore: error_patterns, error_incidents โ””โ”€ Auto-resolve silent patterns @@ -65,6 +66,40 @@ Design spec: docs/archive/2026-04-12-error-monitor-design.md Frontend crash reporting: docs/archive/2026-04-19-frontend-crash-reporting-plan.md ``` +#### Discord channels: #alerts = signal, #noise = everything else + +`#alerts` is reserved for **actual problems someone should look at**. Anything +routine/informational goes to `#noise`. When adding a new Discord post, pick the +channel by asking "does this need a human?" โ€” if not, it's noise. + +| Message | Channel | Source | +|---------|---------|--------| +| ๐Ÿ”ด New Error Pattern / โš ๏ธ Spike / incident | #alerts | error monitor `send_message` | +| โœ… Patterns Auto-Resolved, ๐Ÿ“Š Daily Error Digest | #noise | error monitor `send_noise_message` | +| ๐Ÿ”ด Job failed, ๐ŸŸ  Audio download stalled, ๐ŸŸ  Publish completeness SHADOW, ๐ŸŸ  Gemini unavailable | #alerts | `backend/services/ops_alerts.py` | +| OAuth credential invalid/expired (startup check, `/test-alert`) | #alerts | `backend/main.py`, `auth.py` | +| Nightly Backup Report (clean run) | #noise | `infrastructure/functions/backup_to_aws` | +| ๐Ÿšจ Nightly Backup FAILED (any step errored) | #alerts | same | +| ๐Ÿ“‹ Quarterly DR restore drill due | #alerts | same (`?mode=drill_reminder`, actionable) | +| ๐Ÿšจ DR backup is stale / monitor failed | `DR_MONITOR_DISCORD_WEBHOOK` | `.github/workflows/dr-backup-freshness.yml` (failures only) | +| โœ… E2E Daily Test PASSED, ๐Ÿฉน E2E self-healed (auto-merge PR opened) | #noise | `.github/workflows/e2e-daily.yml` | +| ๐Ÿšจ E2E Daily FAILED, ๐Ÿฉน self-heal PR needs a CI approval click / could not open, โŒ Tenant E2E FAILED | #alerts | `e2e-daily.yml`, `e2e-tenant-daily.yml` | +| GitHub PR opened/closed (org webhook) | #noise | GitHub org settings | +| New YouTube video | #releases | `DEFAULT_DISCORD_WEBHOOK_URL` โ† `discord-releases-webhook` | + +How the webhooks resolve: + +- **#alerts**: Secret Manager `discord-alert-webhook` (error monitor gets it as + `DISCORD_WEBHOOK_URL` from Pulumi; the backup function likewise; GitHub Actions + secret `DISCORD_ALERT_WEBHOOK_URL`). +- **#noise**: Secret Manager `discord-noise-webhook`, read **at runtime** (env + `DISCORD_NOISE_WEBHOOK_URL` overrides; secret name overridable with + `DISCORD_NOISE_WEBHOOK_SECRET_NAME`) โ€” no Pulumi env wiring needed. GitHub + Actions secret `DISCORD_NOISE_WEBHOOK_URL`. +- **Never silently dropped**: if the noise webhook can't be resolved, or a post + to it fails, the message falls back to #alerts. The error monitor rate-limits + each channel separately (10/run each), so noise can't consume the alert budget. + Frontend crashes are captured via `window.onerror`, `unhandledrejection`, Next.js `error.tsx`, and a `CrashReportBoundary` around the review UIs. They POST to `/api/client-errors`, which normalises and upserts into the same `error_patterns` diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index 8cce1faf..751cd500 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -482,7 +482,7 @@ This monitor runs in GitHub, not GCP, so it survives a complete GCP project loss **Required GitHub secrets** (set via `gh secret set`): - `AWS_BACKUP_READONLY_ACCESS_KEY_ID` โ€” IAM user with `s3:ListBucket` + `s3:GetObject` only on `nomadkaraoke-backup` - `AWS_BACKUP_READONLY_SECRET_ACCESS_KEY` -- `DR_MONITOR_DISCORD_WEBHOOK` โ€” Discord webhook URL (separate channel from the nightly success/failure alert is fine) +- `DR_MONITOR_DISCORD_WEBHOOK` โ€” Discord webhook URL (this monitor only posts on stale/failed, so #alerts is right). The nightly function itself posts a clean run's "Nightly Backup Report" to #noise (secret `discord-noise-webhook`, read at runtime) and a failed/partial run as "๐Ÿšจ Nightly Backup FAILED" to #alerts โ€” see ARCHITECTURE.md ยง Discord channels **Manual setup (one-time, in AWS Console):** 1. Create IAM user `nomadkaraoke-backup-monitor` (no console access, programmatic only) diff --git a/infrastructure/functions/backup_to_aws/discord_alert.py b/infrastructure/functions/backup_to_aws/discord_alert.py index 96061b27..35d860bd 100644 --- a/infrastructure/functions/backup_to_aws/discord_alert.py +++ b/infrastructure/functions/backup_to_aws/discord_alert.py @@ -1,18 +1,68 @@ -"""Discord webhook notifications for backup status.""" +"""Discord webhook notifications for backup status. + +Two channels: + +* **#alerts** (``DISCORD_WEBHOOK_URL`` env, from secret ``discord-alert-webhook``) + โ€” real problems only: a failed or partial backup run. +* **#noise** (secret ``discord-noise-webhook``, read at runtime) โ€” the routine + "all good" nightly report. If the noise webhook can't be resolved, or a noise + post fails, the message falls back to #alerts so nothing is silently dropped. +""" import logging +import os + import requests +from google.cloud import secretmanager logger = logging.getLogger(__name__) +NOISE_WEBHOOK_SECRET_DEFAULT = "discord-noise-webhook" -def send_alert(webhook_url: str, title: str, fields: list[dict], success: bool = True): - """Send a Discord embed notification.""" + +def send_alert(webhook_url: str, title: str, fields: list[dict], success: bool = True) -> bool: + """Send a Discord embed notification. Returns True if delivered.""" color = 0x00FF00 if success else 0xFF0000 embed = {"title": title, "color": color, "fields": fields} try: resp = requests.post(webhook_url, json={"embeds": [embed]}, timeout=10) resp.raise_for_status() + return True except Exception as e: logger.error(f"Failed to send Discord alert: {e}") + return False + + +def get_noise_webhook_url(project: str, fallback_url: str) -> str: + """Resolve the #noise webhook URL, falling back to ``fallback_url``. + + ``DISCORD_NOISE_WEBHOOK_URL`` env first, else Secret Manager secret + ``discord-noise-webhook`` (name overridable via + ``DISCORD_NOISE_WEBHOOK_SECRET_NAME``). Read at runtime rather than as a + deploy-time secret env var, so no Pulumi change is needed; the function's + SA holds project-wide ``secretmanager.secretAccessor``. Any error โ†’ fallback. + """ + url = os.environ.get("DISCORD_NOISE_WEBHOOK_URL", "").strip() + if url: + return url + secret_id = os.environ.get("DISCORD_NOISE_WEBHOOK_SECRET_NAME", NOISE_WEBHOOK_SECRET_DEFAULT) + try: + client = secretmanager.SecretManagerServiceClient() + name = f"projects/{project}/secrets/{secret_id}/versions/latest" + resp = client.access_secret_version(request={"name": name}) + url = resp.payload.data.decode("utf-8").strip() + except Exception as e: # noqa: BLE001 โ€” noise channel is optional + logger.warning(f"Could not read {secret_id} ({e}); routine reports go to the alert webhook") + return fallback_url + return url or fallback_url + + +def send_noise(noise_url: str, alert_url: str, title: str, fields: list[dict]) -> bool: + """Post an informational (success) embed to #noise; on failure retry on #alerts.""" + if noise_url and send_alert(webhook_url=noise_url, title=title, fields=fields, success=True): + return True + if alert_url and alert_url != noise_url: + logger.warning("Noise webhook post failed; falling back to the alert webhook") + return send_alert(webhook_url=alert_url, title=title, fields=fields, success=True) + return False diff --git a/infrastructure/functions/backup_to_aws/main.py b/infrastructure/functions/backup_to_aws/main.py index e9a9ec8e..0b703731 100644 --- a/infrastructure/functions/backup_to_aws/main.py +++ b/infrastructure/functions/backup_to_aws/main.py @@ -14,7 +14,7 @@ Nightly backup pipeline: bundle last reached S3; full re-bundle on the first Sunday of each month; survives loss of GitHub access, e.g. an account ban) 5. Upload staging files to S3 (Firestore export held back except Sundays) -6. Discord alert +6. Discord report (success โ†’ #noise, failure โ†’ #alerts) Triggered by Cloud Scheduler at 1:00 AM ET daily. @@ -30,7 +30,7 @@ import os import functions_framework -from discord_alert import send_alert +from discord_alert import get_noise_webhook_url, send_alert, send_noise from firestore_export import export_firestore from bigquery_export import export_bigquery_tables from gcs_sync import sync_gcs_to_staging @@ -239,12 +239,21 @@ def backup_to_aws(request): if isinstance(value, str): fields.append({"name": key, "value": value, "inline": True}) - if DISCORD_WEBHOOK_URL: - send_alert( - webhook_url=DISCORD_WEBHOOK_URL, + # #alerts is signal only: a clean run is routine โ†’ #noise; any failure or + # partial run stays in #alerts. + if success: + send_noise( + noise_url=get_noise_webhook_url(GCP_PROJECT, DISCORD_WEBHOOK_URL), + alert_url=DISCORD_WEBHOOK_URL, title="Nightly Backup Report", fields=fields, - success=success, + ) + elif DISCORD_WEBHOOK_URL: + send_alert( + webhook_url=DISCORD_WEBHOOK_URL, + title="๐Ÿšจ Nightly Backup FAILED", + fields=fields, + success=False, ) status_code = 200 if success else 500 diff --git a/infrastructure/functions/backup_to_aws/test_discord_alert.py b/infrastructure/functions/backup_to_aws/test_discord_alert.py new file mode 100644 index 00000000..a2cbc561 --- /dev/null +++ b/infrastructure/functions/backup_to_aws/test_discord_alert.py @@ -0,0 +1,79 @@ +"""Tests for discord_alert: #alerts vs #noise webhook resolution + delivery.""" + +from unittest.mock import MagicMock, patch + +import requests + +import discord_alert + +ALERT = "https://discord.com/api/webhooks/1/alert" +NOISE = "https://discord.com/api/webhooks/2/noise" + + +def _ok(): + resp = MagicMock() + resp.raise_for_status.return_value = None + return resp + + +def test_send_alert_returns_true_on_success(): + with patch("discord_alert.requests.post", return_value=_ok()) as post: + assert discord_alert.send_alert(ALERT, "t", [], success=True) is True + assert post.call_args.args[0] == ALERT + + +def test_send_alert_returns_false_on_failure(): + bad = MagicMock() + bad.raise_for_status.side_effect = requests.HTTPError("404") + with patch("discord_alert.requests.post", return_value=bad): + assert discord_alert.send_alert(ALERT, "t", [], success=False) is False + + +def test_noise_url_env_var_wins(monkeypatch): + monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_URL", NOISE) + with patch("discord_alert.secretmanager") as sm: + assert discord_alert.get_noise_webhook_url("proj", ALERT) == NOISE + sm.SecretManagerServiceClient.assert_not_called() + + +def test_noise_url_read_from_secret_manager(monkeypatch): + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False) + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", raising=False) + with patch("discord_alert.secretmanager") as sm: + client = sm.SecretManagerServiceClient.return_value + client.access_secret_version.return_value.payload.data = (NOISE + "\n").encode() + assert discord_alert.get_noise_webhook_url("proj", ALERT) == NOISE + name = client.access_secret_version.call_args.kwargs["request"]["name"] + assert name == "projects/proj/secrets/discord-noise-webhook/versions/latest" + + +def test_noise_url_falls_back_to_alert_on_secret_error(monkeypatch): + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False) + with patch("discord_alert.secretmanager") as sm: + sm.SecretManagerServiceClient.return_value.access_secret_version.side_effect = RuntimeError("403") + assert discord_alert.get_noise_webhook_url("proj", ALERT) == ALERT + + +def test_noise_url_falls_back_to_alert_when_secret_empty(monkeypatch): + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False) + with patch("discord_alert.secretmanager") as sm: + sm.SecretManagerServiceClient.return_value.access_secret_version.return_value.payload.data = b" " + assert discord_alert.get_noise_webhook_url("proj", ALERT) == ALERT + + +def test_send_noise_posts_to_noise_webhook(): + with patch("discord_alert.send_alert", return_value=True) as send: + assert discord_alert.send_noise(NOISE, ALERT, "t", []) is True + send.assert_called_once_with(webhook_url=NOISE, title="t", fields=[], success=True) + + +def test_send_noise_falls_back_to_alert_webhook_on_failure(): + with patch("discord_alert.send_alert", side_effect=[False, True]) as send: + assert discord_alert.send_noise(NOISE, ALERT, "t", []) is True + assert [c.kwargs["webhook_url"] for c in send.call_args_list] == [NOISE, ALERT] + + +def test_send_noise_no_retry_when_noise_is_alert(): + with patch("discord_alert.send_alert", return_value=False) as send: + assert discord_alert.send_noise(ALERT, ALERT, "t", []) is False + send.assert_called_once() diff --git a/infrastructure/functions/backup_to_aws/test_main.py b/infrastructure/functions/backup_to_aws/test_main.py index 8394117b..58b954fd 100644 --- a/infrastructure/functions/backup_to_aws/test_main.py +++ b/infrastructure/functions/backup_to_aws/test_main.py @@ -25,7 +25,8 @@ def _run(today): patch("main.export_secrets", return_value="sec"), \ patch("main._run_git_repos_backup", return_value="git") as git, \ patch("main.upload_staging_to_s3", return_value="s3") as upload, \ - patch("main.send_alert"): + patch("main.send_alert"), \ + patch("main.send_noise"): main.backup_to_aws(request) return git, upload @@ -73,9 +74,55 @@ def test_job_files_bucket_is_not_synced_off_site(): patch("main.export_secrets", return_value="sec") as secrets, \ patch("main._run_git_repos_backup", return_value="git"), \ patch("main.upload_staging_to_s3", return_value="s3"), \ - patch("main.send_alert"): + patch("main.send_alert"), \ + patch("main.send_noise"): main.backup_to_aws(request) sources = [c.kwargs["source_bucket"] for c in sync.call_args_list] assert sources == ["nomadkaraoke-kn-data"] assert all(c.kwargs["staging_prefix"] != "gcs/job-files/" for c in sync.call_args_list) secrets.assert_called_once() + + +# --- #alerts vs #noise routing of the nightly report -------------------------- + +ALERT = "https://discord.com/api/webhooks/1/alert" +NOISE = "https://discord.com/api/webhooks/2/noise" + + +def _run_report(fail_s3=False): + _FakeDate.today_value = datetime.date(2026, 9, 29) + request = MagicMock() + request.args = {} + upload_kwargs = {"side_effect": RuntimeError("boom")} if fail_s3 else {"return_value": "s3"} + with patch("main.datetime.date", _FakeDate), \ + patch("main.DISCORD_WEBHOOK_URL", ALERT), \ + patch("main.get_noise_webhook_url", return_value=NOISE), \ + patch("main.export_firestore", return_value="fs"), \ + patch("main.export_bigquery_tables", return_value={}), \ + patch("main.sync_gcs_to_staging", return_value="gcs"), \ + patch("main.export_secrets", return_value="sec"), \ + patch("main.upload_staging_to_s3", **upload_kwargs), \ + patch("main.send_alert") as alert, \ + patch("main.send_noise") as noise: + _, status = main.backup_to_aws(request) + return status, alert, noise + + +def test_successful_report_goes_to_noise(): + status, alert, noise = _run_report() + assert status == 200 + alert.assert_not_called() + noise.assert_called_once() + assert noise.call_args.kwargs["noise_url"] == NOISE + assert noise.call_args.kwargs["alert_url"] == ALERT + assert noise.call_args.kwargs["title"] == "Nightly Backup Report" + + +def test_failed_report_goes_to_alerts(): + status, alert, noise = _run_report(fail_s3=True) + assert status == 500 + noise.assert_not_called() + alert.assert_called_once() + assert alert.call_args.kwargs["webhook_url"] == ALERT + assert alert.call_args.kwargs["success"] is False + assert "FAILED" in alert.call_args.kwargs["title"] diff --git a/pyproject.toml b/pyproject.toml index de55ce62..51f89e56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.278.7" +version = "0.278.8" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/services/error_monitor/test_config.py b/tests/unit/services/error_monitor/test_config.py index df1baf0c..cc4fabb2 100644 --- a/tests/unit/services/error_monitor/test_config.py +++ b/tests/unit/services/error_monitor/test_config.py @@ -274,3 +274,17 @@ class TestHelperFunctions: from backend.services.error_monitor import config as cfg importlib.reload(cfg) assert cfg.get_digest_mode() is True + + +class TestNoiseWebhookSecretName: + def test_default_noise_secret_name(self, monkeypatch): + from backend.services.error_monitor import config + + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", raising=False) + assert config.get_discord_noise_webhook_secret_name() == "discord-noise-webhook" + + def test_noise_secret_name_env_override(self, monkeypatch): + from backend.services.error_monitor import config + + monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", "other-noise") + assert config.get_discord_noise_webhook_secret_name() == "other-noise" diff --git a/tests/unit/services/error_monitor/test_discord.py b/tests/unit/services/error_monitor/test_discord.py index 9033eebc..69939bec 100644 --- a/tests/unit/services/error_monitor/test_discord.py +++ b/tests/unit/services/error_monitor/test_discord.py @@ -617,3 +617,71 @@ class TestErrorMonitorDiscordSendMessage: client.send_message("Skipped") # Counter should not go beyond limit when rate-limited assert client.messages_sent == MAX_DISCORD_MESSAGES_PER_RUN + + +class TestErrorMonitorDiscordNoiseChannel: + """#alerts = signal only; informational posts go to the #noise webhook.""" + + ALERT = "https://discord.com/api/webhooks/1/alert" + NOISE = "https://discord.com/api/webhooks/2/noise" + + def _ok(self): + resp = mock.MagicMock() + resp.raise_for_status.return_value = None + return resp + + def test_send_noise_message_posts_to_noise_webhook(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) + with mock.patch("requests.post", return_value=self._ok()) as mock_post: + assert client.send_noise_message("fyi") is True + mock_post.assert_called_once_with(self.NOISE, json={"content": "fyi"}, timeout=30) + + def test_send_message_still_posts_to_alert_webhook(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) + with mock.patch("requests.post", return_value=self._ok()) as mock_post: + assert client.send_message("problem") is True + mock_post.assert_called_once_with(self.ALERT, json={"content": "problem"}, timeout=30) + + def test_noise_without_noise_webhook_falls_back_to_alert_webhook(self): + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url=self.ALERT) + with mock.patch("requests.post", return_value=self._ok()) as mock_post: + assert client.send_noise_message("fyi") is True + mock_post.assert_called_once_with(self.ALERT, json={"content": "fyi"}, timeout=30) + + def test_noise_delivery_failure_retries_on_alert_webhook(self): + """A broken noise webhook must never silently drop the message.""" + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + bad = mock.MagicMock() + bad.raise_for_status.side_effect = requests.HTTPError("404 Unknown Webhook") + client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) + with mock.patch("requests.post", side_effect=[bad, self._ok()]) as mock_post: + assert client.send_noise_message("fyi") is True + assert [c.args[0] for c in mock_post.call_args_list] == [self.NOISE, self.ALERT] + + def test_noise_messages_do_not_consume_alert_budget(self): + from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) + with mock.patch("requests.post", return_value=self._ok()): + for _ in range(MAX_DISCORD_MESSAGES_PER_RUN): + client.send_noise_message("fyi") + assert client.messages_sent == 0 + assert client.send_message("problem") is True + + def test_noise_messages_are_rate_limited_separately(self): + from backend.services.error_monitor.config import MAX_DISCORD_MESSAGES_PER_RUN + from backend.services.error_monitor.discord import ErrorMonitorDiscord + + client = ErrorMonitorDiscord(webhook_url=self.ALERT, noise_webhook_url=self.NOISE) + client.noise_messages_sent = MAX_DISCORD_MESSAGES_PER_RUN + with mock.patch("requests.post") as mock_post: + assert client.send_noise_message("fyi") is False + mock_post.assert_not_called() diff --git a/tests/unit/services/error_monitor/test_monitor.py b/tests/unit/services/error_monitor/test_monitor.py index 520f69ad..07ddd797 100644 --- a/tests/unit/services/error_monitor/test_monitor.py +++ b/tests/unit/services/error_monitor/test_monitor.py @@ -558,6 +558,7 @@ class TestRunDailyDigest: m.firestore_adapter = MagicMock() m.discord = MagicMock() m.discord.messages_sent = 0 + m.discord.noise_messages_sent = 0 m.logger = MagicMock() return m @@ -569,12 +570,12 @@ class TestRunDailyDigest: {"service": "video-worker", "total_count": 5, "status": "acknowledged"}, ] monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] - monitor.discord.send_message.return_value = True + monitor.discord.send_noise_message.return_value = True monitor._run_daily_digest() - assert monitor.discord.send_message.call_count == 1 - sent_content = monitor.discord.send_message.call_args[0][0] + assert monitor.discord.send_noise_message.call_count == 1 + sent_content = monitor.discord.send_noise_message.call_args[0][0] assert "๐Ÿ“Š" in sent_content @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) @@ -586,11 +587,11 @@ class TestRunDailyDigest: {"service": "svc-beta", "total_count": 2, "status": "new"}, ] monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] - monitor.discord.send_message.return_value = True + monitor.discord.send_noise_message.return_value = True monitor._run_daily_digest() - sent_content = monitor.discord.send_message.call_args[0][0] + sent_content = monitor.discord.send_noise_message.call_args[0][0] # Both services should be mentioned in the digest assert "svc-alpha" in sent_content assert "svc-beta" in sent_content @@ -600,15 +601,29 @@ class TestRunDailyDigest: """_run_daily_digest should still send a digest even when there are no patterns.""" monitor.firestore_adapter.get_active_patterns.return_value = [] monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] - monitor.discord.send_message.return_value = True + monitor.discord.send_noise_message.return_value = True monitor._run_daily_digest() - assert monitor.discord.send_message.call_count == 1 - sent_content = monitor.discord.send_message.call_args[0][0] + assert monitor.discord.send_noise_message.call_count == 1 + sent_content = monitor.discord.send_noise_message.call_args[0][0] assert "๐Ÿ“Š" in sent_content +# --------------------------------------------------------------------------- + + @patch("backend.services.error_monitor.monitor.get_llm_enabled", return_value=False) + def test_digest_goes_to_noise_not_alerts(self, mock_llm, monitor): + """The digest is informational: it must never land in #alerts.""" + monitor.firestore_adapter.get_active_patterns.return_value = [] + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + + monitor._run_daily_digest() + + monitor.discord.send_message.assert_not_called() + monitor.discord.send_noise_message.assert_called_once() + + # --------------------------------------------------------------------------- # Tests: PatternData construction (P0) # --------------------------------------------------------------------------- @@ -911,7 +926,7 @@ class TestAutoResolveAlert: } monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern] monitor.firestore_adapter.check_auto_resolve.return_value = 25.5 # hours silent - monitor.discord.send_message.return_value = True + monitor.discord.send_noise_message.return_value = True monitor._check_auto_resolve() @@ -921,8 +936,8 @@ class TestAutoResolveAlert: ) # A Discord message must have been sent - assert monitor.discord.send_message.call_count == 1 - sent_content = monitor.discord.send_message.call_args[0][0] + assert monitor.discord.send_noise_message.call_count == 1 + sent_content = monitor.discord.send_noise_message.call_args[0][0] assert "โœ…" in sent_content def test_auto_resolve_no_alert_when_no_patterns_qualify(self, monitor): @@ -941,6 +956,7 @@ class TestAutoResolveAlert: monitor.firestore_adapter.auto_resolve_pattern.assert_not_called() monitor.discord.send_message.assert_not_called() + monitor.discord.send_noise_message.assert_not_called() def test_auto_resolve_via_monitor_cycle(self, monitor): """_run_monitor_cycle with no errors should still trigger the auto-resolve check.""" @@ -958,10 +974,88 @@ class TestAutoResolveAlert: monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [pattern] monitor.firestore_adapter.check_auto_resolve.return_value = 12.0 monitor.firestore_adapter.get_active_patterns.return_value = [] - monitor.discord.send_message.return_value = True + monitor.discord.send_noise_message.return_value = True monitor._run_monitor_cycle() monitor.firestore_adapter.auto_resolve_pattern.assert_called_once() - sent_content = monitor.discord.send_message.call_args[0][0] + sent_content = monitor.discord.send_noise_message.call_args[0][0] assert "โœ…" in sent_content + + +# --------------------------------------------------------------------------- +# Tests: #alerts vs #noise routing +# --------------------------------------------------------------------------- + + +class TestAutoResolveRoutesToNoise: + def test_auto_resolved_summary_goes_to_noise(self): + monitor = _make_monitor_with_mocks() + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [ + {"pattern_id": "p1", "service": "karaoke-backend", "normalized_message": "boom"} + ] + monitor.firestore_adapter.check_auto_resolve.return_value = 48.0 + + monitor._check_auto_resolve() + + monitor.discord.send_message.assert_not_called() + monitor.discord.send_noise_message.assert_called_once() + assert "Auto-Resolved" in monitor.discord.send_noise_message.call_args[0][0] + + def test_nothing_resolved_sends_nothing(self): + monitor = _make_monitor_with_mocks() + monitor.firestore_adapter.get_patterns_for_auto_resolve.return_value = [] + + monitor._check_auto_resolve() + + monitor.discord.send_message.assert_not_called() + monitor.discord.send_noise_message.assert_not_called() + + +class TestNoiseWebhookResolution: + ALERT = "https://discord.com/api/webhooks/1/alert" + NOISE = "https://discord.com/api/webhooks/2/noise" + + def _monitor(self): + ErrorMonitor = _import_error_monitor_class() + return ErrorMonitor.__new__(ErrorMonitor) + + def test_env_var_wins(self, monkeypatch): + monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_URL", self.NOISE) + with patch("backend.services.error_monitor.monitor.secretmanager") as sm: + assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.NOISE + sm.SecretManagerServiceClient.assert_not_called() + + def test_reads_noise_secret_from_secret_manager(self, monkeypatch): + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False) + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_SECRET_NAME", raising=False) + with patch("backend.services.error_monitor.monitor.secretmanager") as sm: + client = sm.SecretManagerServiceClient.return_value + client.access_secret_version.return_value.payload.data = (self.NOISE + "\n").encode() + assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.NOISE + name = client.access_secret_version.call_args.kwargs["name"] + assert name.endswith("/secrets/discord-noise-webhook/versions/latest") + + def test_falls_back_to_alert_webhook_when_secret_unavailable(self, monkeypatch): + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False) + with patch("backend.services.error_monitor.monitor.secretmanager") as sm: + sm.SecretManagerServiceClient.return_value.access_secret_version.side_effect = ( + RuntimeError("403 permission denied") + ) + assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.ALERT + + def test_falls_back_to_alert_webhook_when_secret_empty(self, monkeypatch): + monkeypatch.delenv("DISCORD_NOISE_WEBHOOK_URL", raising=False) + with patch("backend.services.error_monitor.monitor.secretmanager") as sm: + client = sm.SecretManagerServiceClient.return_value + client.access_secret_version.return_value.payload.data = b" " + assert self._monitor()._get_discord_noise_webhook(self.ALERT) == self.ALERT + + def test_init_wires_both_webhooks(self, monkeypatch): + monkeypatch.setenv("DISCORD_WEBHOOK_URL", self.ALERT) + monkeypatch.setenv("DISCORD_NOISE_WEBHOOK_URL", self.NOISE) + m = _import_monitor_module() + with patch.object(m.cloud_logging, "Client"), patch.object(m, "ErrorPatternsAdapter"): + monitor = m.ErrorMonitor() + assert monitor.discord.webhook_url == self.ALERT + assert monitor.discord.noise_webhook_url == self.NOISE