mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
From the 2026-11-03 #alerts triage. - start-review returns 409 for a job not in review (and when losing the auto-approver race) instead of a 500; correction-data's same race too. - The error monitor builds unhandled-exception patterns from the exception's cause, so a second crash on another endpoint is no longer swallowed by an already-open "Exception in ASGI application" pattern. - YouTube video ids are normalized out of signatures; one job failure is one alert (fail_job and the known download-failure lines log WARNING). - Suppressions: Zalo's injected script, python -c experiments on job images, decide-search health-probe aborts. Demotions: Spotify 403 on /lyrics, Dropbox token probe (ERROR moves to the refresh failing). - Cold start: import the anyio backend and warm the middleware stack off-loop. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
164 lines
6.1 KiB
Python
164 lines
6.1 KiB
Python
"""Message normalizer for the production error monitoring service.
|
||
|
||
Replaces variable parts of error messages (IDs, timestamps, paths, etc.)
|
||
with stable placeholders so the same logical error always produces the same
|
||
pattern hash, regardless of which specific resource or timestamp was involved.
|
||
|
||
Normalizer rules are applied in order — more specific patterns first.
|
||
"""
|
||
|
||
import hashlib
|
||
import re
|
||
|
||
from backend.services.error_monitor.config import MAX_NORMALIZED_MESSAGE_LENGTH
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Compiled regex patterns (applied in order — more specific first)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_NORMALIZERS: list[tuple[re.Pattern, str]] = [
|
||
# 1. GCS paths: gs://bucket/path/to/object → gs://<BUCKET>/<PATH>
|
||
# Must come before Firestore and URL normalization.
|
||
(
|
||
re.compile(r"gs://([^/\s]+)/([^\s]+)"),
|
||
r"gs://<BUCKET>/<PATH>",
|
||
),
|
||
# 1b. GCS bucket root (no object path): gs://bucket/ or gs://bucket
|
||
# Uses a lookahead so the slash/end-of-string is not consumed, and uses
|
||
# a character class that excludes '<' to avoid re-matching placeholders.
|
||
(
|
||
re.compile(r"gs://([^/<\s]+)(?=[/\s]|$)"),
|
||
r"gs://<BUCKET>",
|
||
),
|
||
# 2. Firestore document paths (collection/doc/…/doc — at least 2 segments)
|
||
# Negative lookbehind for '.' prevents matching inside domain names (URLs).
|
||
# Negative lookbehind for '/' prevents matching inside already-replaced paths.
|
||
(
|
||
re.compile(
|
||
r"(?<![./])\b([a-zA-Z_][a-zA-Z0-9_-]*/[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_-]+){1,})\b"
|
||
),
|
||
r"<DOC_PATH>",
|
||
),
|
||
# 3. UUIDs (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)
|
||
(
|
||
re.compile(
|
||
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
|
||
),
|
||
"<ID>",
|
||
),
|
||
# 4. ISO 8601 timestamps (2024-01-15T10:30:45Z / …+05:30 / ….123Z)
|
||
(
|
||
re.compile(
|
||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})"
|
||
),
|
||
"<TS>",
|
||
),
|
||
# 5. Epoch timestamps: 10+ digit number optionally followed by fractional seconds
|
||
(
|
||
re.compile(r"\b\d{10,}(?:\.\d+)?\b"),
|
||
"<EPOCH>",
|
||
),
|
||
# 6. Email addresses
|
||
(
|
||
re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"),
|
||
"<EMAIL>",
|
||
),
|
||
# 7. IP addresses (IPv4) — must come before large-number replacement
|
||
(
|
||
re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"),
|
||
"<IP>",
|
||
),
|
||
# 8. URLs — strip to domain only (https://host/path → https://host)
|
||
# Handles http and https; preserves the scheme+domain for context.
|
||
(
|
||
re.compile(r"https?://([A-Za-z0-9.\-]+(:\d+)?)/[^\s]*"),
|
||
r"https://\1",
|
||
),
|
||
# 8b. YouTube video IDs: yt-dlp's "[youtube] <id>: <error>" and bare watch links.
|
||
# Otherwise the same error on each video is a new pattern and a "first time"
|
||
# job-failure signature (ops_alerts._signature uses this normalizer).
|
||
(
|
||
re.compile(r"(\[youtube(?::[\w-]+)?\]) [A-Za-z0-9_-]{11}(?=:)"),
|
||
r"\1 <VIDEO_ID>",
|
||
),
|
||
(
|
||
re.compile(r"(youtube\.com/watch\?v=|youtu\.be/)[A-Za-z0-9_-]{11}(?![A-Za-z0-9_-])"),
|
||
r"\1<VIDEO_ID>",
|
||
),
|
||
# 9. Job IDs in paths (/jobs/<alphanumeric>)
|
||
(
|
||
re.compile(r"/jobs/[A-Za-z0-9]+"),
|
||
"/jobs/<ID>",
|
||
),
|
||
# 9b. Preview render job IDs: preview_<jobid>_<renderid> (underscore-joined
|
||
# hex runs). The generic hex rule (11) is \b-anchored and '_' is a word
|
||
# char, so it can't reach these — leaving every preview render of the
|
||
# same job as a distinct pattern (N alerts for one incident). Collapse
|
||
# the whole token so they map to a single pattern.
|
||
(
|
||
re.compile(r"\bpreview_[0-9a-fA-F]{6,}(?:_[0-9a-fA-F]{6,})+\b"),
|
||
"preview_<ID>",
|
||
),
|
||
# 10. Firebase UIDs (20–28 mixed-case alphanumeric characters)
|
||
# Firebase Auth UIDs are typically 28 chars; allow 20-28 for flexibility.
|
||
# Must contain both letters and digits (to avoid replacing plain words).
|
||
(
|
||
re.compile(r"\b(?=[A-Za-z0-9]{20,28}\b)(?=[A-Za-z0-9]*[A-Z])(?=[A-Za-z0-9]*[a-z])(?=[A-Za-z0-9]*\d)[A-Za-z0-9]{20,28}\b"),
|
||
"<UID>",
|
||
),
|
||
# 11. Hex/alphanumeric IDs: 8+ chars containing both letters and digits
|
||
# (e.g. git SHAs, Cloud Run revision IDs, job IDs)
|
||
# Uses a lookahead to require at least one digit AND one letter.
|
||
(
|
||
re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"),
|
||
"<ID>",
|
||
),
|
||
# 12. Request counts in loop-stall lines (EVENT_LOOP_STALL ... requests_affected=29):
|
||
# small numbers that would otherwise make every stall a "new" pattern.
|
||
(
|
||
re.compile(r"\brequests_affected=\d+"),
|
||
"requests_affected=<N>",
|
||
),
|
||
# 13. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
|
||
(
|
||
re.compile(r"\b\d{4,}\b"),
|
||
"<NUM>",
|
||
),
|
||
]
|
||
|
||
|
||
def normalize_message(message: str) -> str:
|
||
"""Normalize a raw log message by replacing variable parts with placeholders.
|
||
|
||
Applies each normalizer rule in order (more specific patterns first) and
|
||
truncates the result to MAX_NORMALIZED_MESSAGE_LENGTH characters.
|
||
|
||
Args:
|
||
message: Raw log message string.
|
||
|
||
Returns:
|
||
Normalized message string, at most MAX_NORMALIZED_MESSAGE_LENGTH chars.
|
||
"""
|
||
if not message:
|
||
return ""
|
||
|
||
result = message
|
||
for pattern, replacement in _NORMALIZERS:
|
||
result = pattern.sub(replacement, result)
|
||
|
||
return result[:MAX_NORMALIZED_MESSAGE_LENGTH]
|
||
|
||
|
||
def compute_pattern_hash(service: str, normalized_message: str) -> str:
|
||
"""Compute a stable SHA-256 fingerprint for an error pattern.
|
||
|
||
Args:
|
||
service: The originating service name (e.g. "karaoke-backend").
|
||
normalized_message: The output of normalize_message().
|
||
|
||
Returns:
|
||
64-character lowercase hex SHA-256 digest.
|
||
"""
|
||
payload = f"{service}:{normalized_message}"
|
||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|