mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(runner-manager): retry a repo once on transient GitHub errors before paging (v0.295.1)
A single RemoteDisconnected from GitHub during the stalled-job scan logged at ERROR and became an #alerts message, though the next tick rescans. Network blips and GitHub 5xx/429 now get one retry; a second failure, or any auth/not-found error, still logs at ERROR so a broken safety net pages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012RYLVWJ7FwupZXGBp7cE5X
This commit is contained in:
1 parent
2db5407a5d
commit
e8f624775d
3 files changed
+121
-24
No files matched your search
@@ -14,6 +14,7 @@ This module is invoked from main.py when RUNNER_MODE=ephemeral.
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
@@ -768,33 +769,62 @@ def list_queued_self_hosted_jobs(pat: str) -> list[dict]:
|
||||
"""
|
||||
jobs: list[dict] = []
|
||||
for repo in list_org_repos(pat):
|
||||
try:
|
||||
run_ids: set[int] = set()
|
||||
for status in ("queued", "in_progress"):
|
||||
runs = _github_request(
|
||||
"GET",
|
||||
f"/repos/{GITHUB_ORG}/{repo}/actions/runs?status={status}&per_page=50",
|
||||
pat,
|
||||
)
|
||||
if isinstance(runs, dict):
|
||||
run_ids.update(r["id"] for r in runs.get("workflow_runs", []))
|
||||
for run_id in sorted(run_ids):
|
||||
run_jobs = _github_request(
|
||||
"GET",
|
||||
f"/repos/{GITHUB_ORG}/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page=100",
|
||||
pat,
|
||||
)
|
||||
if not isinstance(run_jobs, dict):
|
||||
# A one-off network blip (GitHub closing the connection, a timeout, a
|
||||
# 5xx) gets one retry before it pages: a single dropped connection is
|
||||
# not the safety net failing, but a repo that fails twice in a row is.
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
jobs.extend(_queued_self_hosted_jobs_in_repo(repo, pat))
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if attempt == 1 and _is_transient_github_error(exc):
|
||||
print(f"Retrying {repo} in stalled-job scan after transient error: {exc!r}", flush=True)
|
||||
continue
|
||||
for job in run_jobs.get("jobs", []):
|
||||
labels = job.get("labels", [])
|
||||
if job.get("status") == "queued" and "self-hosted" in labels and can_serve(labels):
|
||||
jobs.append({**job, "repo": repo})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_log_error(f"Skipping {repo} in stalled-job scan: {exc!r}")
|
||||
_log_error(f"Skipping {repo} in stalled-job scan: {exc!r}")
|
||||
break
|
||||
return jobs
|
||||
|
||||
|
||||
def _queued_self_hosted_jobs_in_repo(repo: str, pat: str) -> list[dict]:
|
||||
"""Queued jobs in one repo that one of our runner families can serve."""
|
||||
jobs: list[dict] = []
|
||||
run_ids: set[int] = set()
|
||||
for status in ("queued", "in_progress"):
|
||||
runs = _github_request(
|
||||
"GET",
|
||||
f"/repos/{GITHUB_ORG}/{repo}/actions/runs?status={status}&per_page=50",
|
||||
pat,
|
||||
)
|
||||
if isinstance(runs, dict):
|
||||
run_ids.update(r["id"] for r in runs.get("workflow_runs", []))
|
||||
for run_id in sorted(run_ids):
|
||||
run_jobs = _github_request(
|
||||
"GET",
|
||||
f"/repos/{GITHUB_ORG}/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page=100",
|
||||
pat,
|
||||
)
|
||||
if not isinstance(run_jobs, dict):
|
||||
continue
|
||||
for job in run_jobs.get("jobs", []):
|
||||
labels = job.get("labels", [])
|
||||
if job.get("status") == "queued" and "self-hosted" in labels and can_serve(labels):
|
||||
jobs.append({**job, "repo": repo})
|
||||
return jobs
|
||||
|
||||
|
||||
def _is_transient_github_error(exc: BaseException) -> bool:
|
||||
"""True for network-level failures and GitHub 5xx/429 worth one retry.
|
||||
|
||||
Auth/permission/not-found errors (other 4xx) are not transient: retrying
|
||||
won't help, so they page straight away.
|
||||
"""
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
return exc.code == 429 or exc.code >= 500
|
||||
# URLError wraps DNS/connect failures; http.client.RemoteDisconnected is a
|
||||
# ConnectionResetError; socket timeouts are TimeoutError (both OSError).
|
||||
return isinstance(exc, (urllib.error.URLError, http.client.HTTPException, OSError))
|
||||
|
||||
|
||||
def _family_of_runner_name(name: str) -> str | None:
|
||||
"""``gha-gpu-windows-abc123`` → ``gpu-windows``; None if not ours."""
|
||||
if not name.startswith("gha-") or "-" not in name[len("gha-"):]:
|
||||
|
||||
@@ -736,6 +736,73 @@ class TestListQueuedSelfHostedJobs:
|
||||
job_calls = [c for c in req.call_args_list if "/runs/11/jobs" in c.args[1]]
|
||||
assert len(job_calls) == 1
|
||||
|
||||
@staticmethod
|
||||
def _scan_with_runs_failures(ep, failures):
|
||||
"""Scan one repo whose first runs-list calls raise ``failures`` in order."""
|
||||
pending = list(failures)
|
||||
|
||||
def fake_request(method, path, pat, body=None):
|
||||
if path.startswith("/orgs/test-org/repos"):
|
||||
return [{"name": "karaoke-gen"}]
|
||||
if "/actions/runs?status=queued" in path and pending:
|
||||
raise pending.pop(0)
|
||||
if "/actions/runs?status=queued" in path:
|
||||
return {"workflow_runs": [{"id": 11}]}
|
||||
if "/actions/runs?status=in_progress" in path:
|
||||
return {"workflow_runs": []}
|
||||
if "/actions/runs/11/jobs" in path:
|
||||
return {"jobs": [{"id": 1, "status": "queued", "labels": BUILD_LABELS}]}
|
||||
raise AssertionError(path)
|
||||
|
||||
with patch.object(ep, "_github_request", side_effect=fake_request) as req:
|
||||
jobs = ep.list_queued_self_hosted_jobs("ghp_test")
|
||||
return jobs, req
|
||||
|
||||
def test_one_dropped_connection_is_retried_without_paging(self, capsys):
|
||||
import http.client
|
||||
|
||||
ep = _fresh_module()
|
||||
jobs, _ = self._scan_with_runs_failures(
|
||||
ep, [http.client.RemoteDisconnected("Remote end closed connection without response")]
|
||||
)
|
||||
|
||||
assert [(j["repo"], j["id"]) for j in jobs] == [("karaoke-gen", 1)]
|
||||
out = capsys.readouterr().out
|
||||
assert _error_messages(out) == []
|
||||
assert "Retrying karaoke-gen in stalled-job scan" in out
|
||||
|
||||
def test_transient_error_twice_in_a_row_pages(self, capsys):
|
||||
import http.client
|
||||
|
||||
ep = _fresh_module()
|
||||
jobs, _ = self._scan_with_runs_failures(
|
||||
ep, [http.client.RemoteDisconnected("closed"), TimeoutError("timed out")]
|
||||
)
|
||||
|
||||
assert jobs == []
|
||||
assert any("Skipping karaoke-gen in stalled-job scan" in m for m in _error_messages(capsys.readouterr().out))
|
||||
|
||||
def test_auth_error_pages_without_retry(self, capsys):
|
||||
import urllib.error
|
||||
|
||||
ep = _fresh_module()
|
||||
forbidden = urllib.error.HTTPError("https://api.github.com/x", 403, "Forbidden", {}, None)
|
||||
jobs, req = self._scan_with_runs_failures(ep, [forbidden])
|
||||
|
||||
assert jobs == []
|
||||
assert any("Skipping karaoke-gen in stalled-job scan" in m for m in _error_messages(capsys.readouterr().out))
|
||||
runs_calls = [c for c in req.call_args_list if "status=queued" in c.args[1]]
|
||||
assert len(runs_calls) == 1
|
||||
|
||||
def test_github_5xx_is_transient(self):
|
||||
import urllib.error
|
||||
|
||||
ep = _fresh_module()
|
||||
assert ep._is_transient_github_error(urllib.error.HTTPError("u", 502, "Bad Gateway", {}, None))
|
||||
assert ep._is_transient_github_error(urllib.error.HTTPError("u", 429, "Too Many", {}, None))
|
||||
assert not ep._is_transient_github_error(urllib.error.HTTPError("u", 404, "Not Found", {}, None))
|
||||
assert not ep._is_transient_github_error(RuntimeError("403 no access"))
|
||||
|
||||
|
||||
class TestCleanupTriggersRedispatch:
|
||||
def test_redispatch_sees_only_surviving_vms(self):
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.295.0"
|
||||
version = "0.295.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user