mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(error-monitor): ignore runner-manager's other Cloud Run abort wording (#1252)
## Summary On 2026-11-04 at 10:00Z a PR opened on python-audio-separator queued a burst of `workflow_job` webhooks. 22 of 54 deliveries got a 500 from Cloud Run while `github-runner-manager` scaled up. The error monitor alerted because Cloud Run used a second wording, "aborted after waiting too long for an available instance", which the existing ignore pattern didn't match. There was no impact: all self-hosted jobs in the affected runs got runners within about 3.5 minutes and passed. ## Changes - **Error monitor:** match both Cloud Run abort wordings for `github-runner-manager`, but only on **webhook** requests (the bare function URL). Cloud Scheduler ticks (`?action=check_idle`) and entries with no URL still alert. `_SERVICE_URL_IGNORE_PATTERNS` now accepts a predicate on the split URL as well as an exact path. - **Runner manager (Cloud Function):** the ignore relies on the 5-minute stalled-job tick catching dropped webhooks, so that tick's own failures now log as structured `severity: ERROR` and page. That covers a redispatch crash, a repo skipped in the scan, a job given up at 60 min, and a failed re-dispatch. Before this, they were severity-less prints that nothing alerted on. Both changes came from CodeRabbit feedback. - Corrected the pattern's reason: GitHub did not redeliver these (`redelivery: false`). - Version 0.292.1; merged latest main. ## ⚠️ Deploy `infrastructure/functions/runner_manager/` only goes live with a local `pulumi up` (CI never runs Pulumi). Apply it **before** merging, so the safety net's alerting is live before the webhook-abort ignore ships with the backend deploy. ## Testing - `tests/unit/services/error_monitor/`: 452 passed (webhook, scheduler-tick, no-URL and other-service cases for both wordings). - `infrastructure/functions/runner_manager`: 71 passed. The 4 updated tests fail against main's `ephemeral.py`. - Full `make test` not run locally; relying on CI. Alert: https://discord.com/channels/1198691101943541770/1265517528608669707/1567479055887114363 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVVTKcWCcDArTQrjS82 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> <!-- coderabbit-ai-generated:v1 --> ## Summary by CodeRabbit * **Bug Fixes** * Runner-manager webhook errors are filtered more precisely. Recognized abort messages are ignored only for matching webhook requests; scheduler-tick errors, requests with other URLs, and requests without a URL continue to trigger alerts. * Runner-manager job-processing failures are recorded as structured error logs, including stalled-job scans and dispatch failures. Give-up logs now report the configured limit rather than the job’s current time in the queue. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a1cbe1c0d4
commit
5c41be1d3b
5 files changed
+106
-41
No files matched your search
@@ -14,8 +14,9 @@ Usage:
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.parse import SplitResult, urlsplit
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -119,17 +120,6 @@ _IGNORE_PATTERNS: list[tuple[str, re.Pattern, str]] = [
|
||||
# Patterns that are noise only for specific services: the same message from a
|
||||
# customer-facing service is a real problem. (pattern_name, services, regex, reason)
|
||||
_SERVICE_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str]] = [
|
||||
(
|
||||
"runner_manager_no_available_instance",
|
||||
frozenset({"github-runner-manager"}),
|
||||
re.compile(
|
||||
r"request was aborted because there was no available instance",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"github-runner-manager scale-up burst: Cloud Run aborts webhook requests "
|
||||
"while instances start, GitHub redelivers and the runners still dispatch "
|
||||
"(seen 2026-09-20, 09-23, 09-27, 10-08)",
|
||||
),
|
||||
(
|
||||
"audio_separation_experiment_traceback",
|
||||
frozenset({"audio-separation-job"}),
|
||||
@@ -151,9 +141,33 @@ _SERVICE_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str]] = [
|
||||
]
|
||||
|
||||
|
||||
# Noise only for one service AND request URL path (Cloud Run request logs carry the
|
||||
# URL outside the message). (pattern_name, services, message regex, exact URL path, reason)
|
||||
_SERVICE_URL_IGNORE_PATTERNS: list[tuple[str, frozenset[str], re.Pattern, str, str]] = [
|
||||
def _is_runner_manager_webhook(url: SplitResult) -> bool:
|
||||
"""GitHub webhooks hit the bare function URL; Cloud Scheduler ticks add ``?action=``."""
|
||||
return url.path == "/github-runner-manager" and not url.query
|
||||
|
||||
|
||||
# Noise only for one service AND request URL (Cloud Run request logs carry the URL
|
||||
# outside the message). The URL matcher is an exact path, or a predicate on the split
|
||||
# URL when the path alone can't tell requests apart.
|
||||
# (pattern_name, services, message regex, URL path or predicate, reason)
|
||||
_SERVICE_URL_IGNORE_PATTERNS: list[
|
||||
tuple[str, frozenset[str], re.Pattern, str | Callable[[SplitResult], bool], str]
|
||||
] = [
|
||||
(
|
||||
"runner_manager_no_available_instance",
|
||||
frozenset({"github-runner-manager"}),
|
||||
re.compile(
|
||||
r"request was aborted (because there was no available instance"
|
||||
r"|after waiting too long for an available instance)",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
_is_runner_manager_webhook,
|
||||
"github-runner-manager scale-up burst: Cloud Run aborts webhook requests "
|
||||
"while instances start. GitHub does not redeliver them; the 5-minute "
|
||||
"redispatch_stalled_jobs tick launches a runner for any job left queued "
|
||||
"(seen 2026-09-20, 09-23, 09-27, 10-08, 11-04). Scheduler ticks are not "
|
||||
"ignored: a failed tick means that safety net didn't run.",
|
||||
),
|
||||
(
|
||||
"decide_search_health_abort",
|
||||
frozenset({"karaoke-decide-search"}),
|
||||
@@ -196,9 +210,10 @@ def should_ignore(service: str, message: str, request_url: str | None = None) ->
|
||||
return IgnoreReason(pattern_name=pattern_name, reason=reason)
|
||||
|
||||
if request_url:
|
||||
url_path = urlsplit(request_url).path
|
||||
for pattern_name, services, pattern, path, reason in _SERVICE_URL_IGNORE_PATTERNS:
|
||||
if service in services and url_path == path and pattern.search(message):
|
||||
url = urlsplit(request_url)
|
||||
for pattern_name, services, pattern, url_match, reason in _SERVICE_URL_IGNORE_PATTERNS:
|
||||
url_ok = url_match(url) if callable(url_match) else url.path == url_match
|
||||
if service in services and url_ok and pattern.search(message):
|
||||
return IgnoreReason(pattern_name=pattern_name, reason=reason)
|
||||
|
||||
return None
|
||||
@@ -13,7 +13,6 @@ This module is invoked from main.py when RUNNER_MODE=ephemeral.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
@@ -174,6 +173,17 @@ try {
|
||||
_compute_client: compute_v1.InstancesClient | None = None
|
||||
|
||||
|
||||
def _log_error(message: str) -> None:
|
||||
"""Log at ERROR so the error monitor (severity>=ERROR) alerts on it.
|
||||
|
||||
Plain ``print`` lines have no severity in Cloud Logging. Use this for
|
||||
failures of the stalled-job safety net: the error monitor ignores
|
||||
webhook aborts on the assumption that this net catches dropped jobs,
|
||||
so the net failing must page.
|
||||
"""
|
||||
print(json.dumps({"severity": "ERROR", "message": message}), flush=True)
|
||||
|
||||
|
||||
def get_compute_client() -> compute_v1.InstancesClient:
|
||||
global _compute_client
|
||||
if _compute_client is None:
|
||||
@@ -716,7 +726,7 @@ def cleanup_orphans(pat: str) -> dict:
|
||||
try:
|
||||
result["redispatch"] = redispatch_stalled_jobs(pat, live_vms, runner_by_name)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"redispatch_stalled_jobs failed: {exc!r}")
|
||||
_log_error(f"redispatch_stalled_jobs failed: {exc!r}")
|
||||
result["redispatch"] = {"error": str(exc)}
|
||||
|
||||
return result
|
||||
@@ -781,7 +791,7 @@ def list_queued_self_hosted_jobs(pat: str) -> list[dict]:
|
||||
if job.get("status") == "queued" and "self-hosted" in labels and can_serve(labels):
|
||||
jobs.append({**job, "repo": repo})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"Skipping {repo} in stalled-job scan: {exc!r}")
|
||||
_log_error(f"Skipping {repo} in stalled-job scan: {exc!r}")
|
||||
return jobs
|
||||
|
||||
|
||||
@@ -849,7 +859,11 @@ def redispatch_stalled_jobs(
|
||||
if age >= REDISPATCH_GIVE_UP_MINUTES:
|
||||
# Excluded from demand too, so a job we've abandoned doesn't make
|
||||
# every later stalled job in its family look under-supplied.
|
||||
print(f"Not re-dispatching {job['repo']} job {job.get('id')}: queued {age:.0f}min (> give-up)")
|
||||
# Fixed text (no live age) so every tick groups into one error pattern.
|
||||
_log_error(
|
||||
f"Not re-dispatching {job['repo']} job {job.get('id')}: "
|
||||
f"queued longer than the {REDISPATCH_GIVE_UP_MINUTES}min give-up limit"
|
||||
)
|
||||
result["given_up"].append(job.get("id"))
|
||||
continue
|
||||
demand[family] = demand.get(family, 0) + 1
|
||||
@@ -886,6 +900,6 @@ def redispatch_stalled_jobs(
|
||||
try:
|
||||
result["dispatched"].append(fut.result()["runner_name"])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"Re-dispatch for {job['repo']} job {job.get('id')} failed: {exc!r}")
|
||||
_log_error(f"Re-dispatch for {job['repo']} job {job.get('id')} failed: {exc!r}")
|
||||
result["dispatch_failed"].append(job.get("id"))
|
||||
return result
|
||||
@@ -67,6 +67,18 @@ def _fresh_module(**env):
|
||||
return ephemeral
|
||||
|
||||
|
||||
def _error_messages(out: str) -> list[str]:
|
||||
"""Messages printed as structured ERROR logs (what the error monitor sees)."""
|
||||
msgs = []
|
||||
for line in out.splitlines():
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(entry, dict) and entry.get("severity") == "ERROR":
|
||||
msgs.append(entry["message"])
|
||||
return msgs
|
||||
|
||||
class TestResolveFamily:
|
||||
def test_gpu_label_wins(self):
|
||||
ep = _fresh_module()
|
||||
@@ -583,7 +595,7 @@ class TestRedispatchStalledJobs:
|
||||
|
||||
create.assert_not_called()
|
||||
|
||||
def test_given_up_job_does_not_consume_spare_capacity(self):
|
||||
def test_given_up_job_does_not_consume_spare_capacity(self, capsys):
|
||||
ep = _fresh_module()
|
||||
vms = [("us-central1-a", _make_instance("gha-build-idle", age_minutes=10))]
|
||||
runners = {"gha-build-idle": {"name": "gha-build-idle", "status": "online", "busy": False}}
|
||||
@@ -592,6 +604,11 @@ class TestRedispatchStalledJobs:
|
||||
|
||||
create.assert_not_called()
|
||||
assert result["given_up"] == [1]
|
||||
# Giving up means the safety net failed: it must reach the error monitor.
|
||||
errors = [m for m in _error_messages(capsys.readouterr().out) if "Not re-dispatching" in m]
|
||||
assert errors
|
||||
# No live queue age in the text, so repeat ticks group into one alert pattern.
|
||||
assert all("give-up limit" in m and "90min" not in m for m in errors)
|
||||
|
||||
def test_idle_registered_runner_covers_the_stalled_job(self):
|
||||
ep = _fresh_module()
|
||||
@@ -656,7 +673,7 @@ class TestRedispatchStalledJobs:
|
||||
|
||||
assert create.call_count == 2
|
||||
|
||||
def test_dispatch_failure_is_recorded_and_others_continue(self):
|
||||
def test_dispatch_failure_is_recorded_and_others_continue(self, capsys):
|
||||
ep = _fresh_module()
|
||||
jobs = [_queued_job(1, BUILD_LABELS, age_minutes=10), _queued_job(2, BUILD_LABELS, age_minutes=10)]
|
||||
outcomes = iter([RuntimeError("stockout"), {"runner_name": "gha-build-ok"}])
|
||||
@@ -674,6 +691,7 @@ class TestRedispatchStalledJobs:
|
||||
|
||||
assert result["dispatched"] == ["gha-build-ok"]
|
||||
assert len(result["dispatch_failed"]) == 1
|
||||
assert any("Re-dispatch for" in m and "stockout" in m for m in _error_messages(capsys.readouterr().out))
|
||||
|
||||
|
||||
class TestFamilyOfRunnerName:
|
||||
@@ -685,7 +703,7 @@ class TestFamilyOfRunnerName:
|
||||
|
||||
|
||||
class TestListQueuedSelfHostedJobs:
|
||||
def test_collects_queued_self_hosted_jobs_across_repos(self):
|
||||
def test_collects_queued_self_hosted_jobs_across_repos(self, capsys):
|
||||
ep = _fresh_module()
|
||||
|
||||
def fake_request(method, path, pat, body=None):
|
||||
@@ -713,6 +731,7 @@ class TestListQueuedSelfHostedJobs:
|
||||
jobs = ep.list_queued_self_hosted_jobs("ghp_test")
|
||||
|
||||
assert [(j["repo"], j["id"]) for j in jobs] == [("karaoke-gen", 1)]
|
||||
assert any("Skipping flacfetch in stalled-job scan" in m for m in _error_messages(capsys.readouterr().out))
|
||||
# Run 11 appeared under both statuses but its jobs are fetched once.
|
||||
job_calls = [c for c in req.call_args_list if "/runs/11/jobs" in c.args[1]]
|
||||
assert len(job_calls) == 1
|
||||
@@ -736,7 +755,7 @@ class TestCleanupTriggersRedispatch:
|
||||
assert live == ["gha-build-booting"]
|
||||
assert result["redispatch"] == {"dispatched": []}
|
||||
|
||||
def test_redispatch_error_does_not_break_cleanup(self):
|
||||
def test_redispatch_error_does_not_break_cleanup(self, capsys):
|
||||
ep = _fresh_module()
|
||||
with patch.object(ep, "_list_all_ephemeral_vms", return_value=[]), patch.object(
|
||||
ep, "list_org_runners", return_value=[]
|
||||
@@ -744,6 +763,7 @@ class TestCleanupTriggersRedispatch:
|
||||
result = ep.cleanup_orphans("ghp_test")
|
||||
|
||||
assert result["redispatch"] == {"error": "github down"}
|
||||
assert any("redispatch_stalled_jobs failed" in m for m in _error_messages(capsys.readouterr().out))
|
||||
assert result["deleted_vms"] == []
|
||||
|
||||
def test_kill_switch_disables_redispatch(self):
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.293.0"
|
||||
version = "0.293.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -558,34 +558,50 @@ class TestCaseInsensitiveMatching:
|
||||
|
||||
|
||||
class TestRunnerManagerNoAvailableInstance:
|
||||
"""github-runner-manager's Cloud Run 'no available instance' abort is a burst
|
||||
during runner scale-up; the runners still dispatch (seen 09-20, 09-23, 09-27,
|
||||
10-08), so it must not page."""
|
||||
"""github-runner-manager's Cloud Run abort is a burst during runner scale-up; the
|
||||
stalled-job tick still dispatches the runners (seen 09-20, 09-23, 09-27, 10-08,
|
||||
11-04), so a webhook abort must not page. A scheduler-tick abort still alerts:
|
||||
that tick is the safety net."""
|
||||
|
||||
MSG = (
|
||||
"The request was aborted because there was no available instance. "
|
||||
"Additional troubleshooting documentation can be found at: "
|
||||
"https://cloud.google.com/run/docs/troubleshooting#abort-request"
|
||||
)
|
||||
WAITED_MSG = (
|
||||
"The request was aborted after waiting too long for an available instance. "
|
||||
"Additional troubleshooting documentation can be found at: "
|
||||
"https://cloud.google.com/run/docs/troubleshooting#abort-request"
|
||||
)
|
||||
WEBHOOK = "https://us-central1-nomadkaraoke.cloudfunctions.net/github-runner-manager"
|
||||
SCHEDULER = WEBHOOK + "?action=check_idle"
|
||||
|
||||
def test_ignored_for_runner_manager(self):
|
||||
@pytest.mark.parametrize("msg", [MSG, WAITED_MSG])
|
||||
def test_webhook_abort_ignored(self, msg):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
result = should_ignore("github-runner-manager", self.MSG)
|
||||
result = should_ignore("github-runner-manager", msg, self.WEBHOOK)
|
||||
assert result is not None
|
||||
assert result.pattern_name == "runner_manager_no_available_instance"
|
||||
|
||||
def test_still_alerts_for_other_services(self):
|
||||
@pytest.mark.parametrize("msg", [MSG, WAITED_MSG])
|
||||
def test_scheduler_tick_abort_still_alerts(self, msg):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("github-runner-manager", msg, self.SCHEDULER) is None
|
||||
|
||||
@pytest.mark.parametrize("msg", [MSG, WAITED_MSG])
|
||||
def test_no_url_still_alerts(self, msg):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("github-runner-manager", msg) is None
|
||||
|
||||
@pytest.mark.parametrize("msg", [MSG, WAITED_MSG])
|
||||
def test_still_alerts_for_other_services(self, msg):
|
||||
# On the customer-facing backend the same abort means dropped requests.
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("karaoke-backend", self.MSG) is None
|
||||
|
||||
def test_other_runner_manager_errors_still_alert(self):
|
||||
from backend.services.error_monitor.known_issues import should_ignore
|
||||
|
||||
assert should_ignore("github-runner-manager", "Failed to create runner VM: quota exceeded") is None
|
||||
|
||||
assert should_ignore("karaoke-backend", msg, "https://api.nomadkaraoke.com/github-runner-manager") is None
|
||||
|
||||
class TestAudioSeparationExperimentTracebacks:
|
||||
"""Ad-hoc `python -c` experiments on prod job images (2026-10-31, 11-01) are not
|
||||
|
||||
Reference in new issue
Block a user