Files
karaoke-gen/backend/services/error_monitor/normalizer.py
T
Andrew BeveridgeandClaude Opus 5.5 f006246bd6 fix(alerts): start-review 409 on finished jobs, per-cause crash patterns, one alert per job failure (v0.290.1)
From the 2026-11-03 #alerts triage.

- start-review returns 409 for a job not in review (and when losing the
  auto-approver race) instead of a 500; correction-data's same race too.
- The error monitor builds unhandled-exception patterns from the exception's
  cause, so a second crash on another endpoint is no longer swallowed by an
  already-open "Exception in ASGI application" pattern.
- YouTube video ids are normalized out of signatures; one job failure is one
  alert (fail_job and the known download-failure lines log WARNING).
- Suppressions: Zalo's injected script, python -c experiments on job images,
  decide-search health-probe aborts. Demotions: Spotify 403 on /lyrics, Dropbox
  token probe (ERROR moves to the refresh failing).
- Cold start: import the anyio backend and warm the middleware stack off-loop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
2026-11-03 17:53:57 -05:00

164 lines
6.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Message normalizer for the production error monitoring service.
Replaces variable parts of error messages (IDs, timestamps, paths, etc.)
with stable placeholders so the same logical error always produces the same
pattern hash, regardless of which specific resource or timestamp was involved.
Normalizer rules are applied in order — more specific patterns first.
"""
import hashlib
import re
from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH
# ---------------------------------------------------------------------------
# Compiled regex patterns (applied in order — more specific first)
# ---------------------------------------------------------------------------
_NORMALIZERS: list[tuple[re.Pattern, str]] = [
# 1. GCS paths: gs://bucket/path/to/object → gs://<BUCKET>/<PATH>
# Must come before Firestore and URL normalization.
(
re.compile(r"gs://([^/\s]+)/([^\s]+)"),
r"gs://<BUCKET>/<PATH>",
),
# 1b. GCS bucket root (no object path): gs://bucket/ or gs://bucket
# Uses a lookahead so the slash/end-of-string is not consumed, and uses
# a character class that excludes '<' to avoid re-matching placeholders.
(
re.compile(r"gs://([^/<\s]+)(?=[/\s]|$)"),
r"gs://<BUCKET>",
),
# 2. Firestore document paths (collection/doc/…/doc — at least 2 segments)
# Negative lookbehind for '.' prevents matching inside domain names (URLs).
# Negative lookbehind for '/' prevents matching inside already-replaced paths.
(
re.compile(
r"(?<![./])\b([a-zA-Z_][a-zA-Z0-9_-]*/[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_-]+){1,})\b"
),
r"<DOC_PATH>",
),
# 3. UUIDs (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)
(
re.compile(
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
),
"<ID>",
),
# 4. ISO 8601 timestamps (2024-01-15T10:30:45Z / …+05:30 / ….123Z)
(
re.compile(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})"
),
"<TS>",
),
# 5. Epoch timestamps: 10+ digit number optionally followed by fractional seconds
(
re.compile(r"\b\d{10,}(?:\.\d+)?\b"),
"<EPOCH>",
),
# 6. Email addresses
(
re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"),
"<EMAIL>",
),
# 7. IP addresses (IPv4) — must come before large-number replacement
(
re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"),
"<IP>",
),
# 8. URLs — strip to domain only (https://host/path → https://host)
# Handles http and https; preserves the scheme+domain for context.
(
re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"),
r"https://\1",
),
# 8b. YouTube video IDs: yt-dlp's "[youtube] <id>: <error>" and bare watch links.
# Otherwise the same error on each video is a new pattern and a "first time"
# job-failure signature (ops_alerts._signature uses this normalizer).
(
re.compile(r"(\[youtube(?::[\w-]+)?\]) [A-Za-z0-9_-]{11}(?=:)"),
r"\1 <VIDEO_ID>",
),
(
re.compile(r"(youtube\.com/watch\?v=|youtu\.be/)[A-Za-z0-9_-]{11}(?![A-Za-z0-9_-])"),
r"\1<VIDEO_ID>",
),
# 9. Job IDs in paths (/jobs/<alphanumeric>)
(
re.compile(r"/jobs/[A-Za-z0-9]+"),
"/jobs/<ID>",
),
# 9b. Preview render job IDs: preview_<jobid>_<renderid> (underscore-joined
# hex runs). The generic hex rule (11) is \b-anchored and '_' is a word
# char, so it can't reach these — leaving every preview render of the
# same job as a distinct pattern (N alerts for one incident). Collapse
# the whole token so they map to a single pattern.
(
re.compile(r"\bpreview_[0-9a-fA-F]{6,}(?:_[0-9a-fA-F]{6,})+\b"),
"preview_<ID>",
),
# 10. Firebase UIDs (20–28 mixed-case alphanumeric characters)
# Firebase Auth UIDs are typically 28 chars; allow 20-28 for flexibility.
# Must contain both letters and digits (to avoid replacing plain words).
(
re.compile(r"\b(?=[A-Za-z0-9]{20,28}\b)(?=[A-Za-z0-9]*[A-Z])(?=[A-Za-z0-9]*[a-z])(?=[A-Za-z0-9]*\d)[A-Za-z0-9]{20,28}\b"),
"<UID>",
),
# 11. Hex/alphanumeric IDs: 8+ chars containing both letters and digits
# (e.g. git SHAs, Cloud Run revision IDs, job IDs)
# Uses a lookahead to require at least one digit AND one letter.
(
re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"),
"<ID>",
),
# 12. Request counts in loop-stall lines (EVENT_LOOP_STALL ... requests_affected=29):
# small numbers that would otherwise make every stall a "new" pattern.
(
re.compile(r"\brequests_affected=\d+"),
"requests_affected=<N>",
),
# 13. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
(
re.compile(r"\b\d{4,}\b"),
"<NUM>",
),
]
def normalize_message(message: str) -> str:
"""Normalize a raw log message by replacing variable parts with placeholders.
Applies each normalizer rule in order (more specific patterns first) and
truncates the result to MAX_NORMALIZED_MESSAGE_LENGTH characters.
Args:
message: Raw log message string.
Returns:
Normalized message string, at most MAX_NORMALIZED_MESSAGE_LENGTH chars.
"""
if not message:
return ""
result = message
for pattern, replacement in _NORMALIZERS:
result = pattern.sub(replacement, result)
return result[:MAX_NORMALIZED_MESSAGE_LENGTH]
def compute_pattern_hash(service: str, normalized_message: str) -> str:
"""Compute a stable SHA-256 fingerprint for an error pattern.
Args:
service: The originating service name (e.g. "karaoke-backend").
normalized_message: The output of normalize_message().
Returns:
64-character lowercase hex SHA-256 digest.
"""
payload = f"{service}:{normalized_message}"
return hashlib.sha256(payload.encode("utf-8")).hexdigest()