From e8f624775d28e7016eee8c51b5b5c71b8e2ece8c Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Mon, 9 Nov 2026 15:08:28 -0500 Subject: [PATCH] fix(runner-manager): retry a repo once on transient GitHub errors before paging (v0.295.1) A single RemoteDisconnected from GitHub during the stalled-job scan logged at ERROR and became an #alerts message, though the next tick rescans. Network blips and GitHub 5xx/429 now get one retry; a second failure, or any auth/not-found error, still logs at ERROR so a broken safety net pages. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012RYLVWJ7FwupZXGBp7cE5X --- .../functions/runner_manager/ephemeral.py | 76 +++++++++++++------ .../runner_manager/test_ephemeral.py | 67 ++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 121 insertions(+), 24 deletions(-) diff --git a/infrastructure/functions/runner_manager/ephemeral.py b/infrastructure/functions/runner_manager/ephemeral.py index 810f3f1c..d43de3f0 100644 --- a/infrastructure/functions/runner_manager/ephemeral.py +++ b/infrastructure/functions/runner_manager/ephemeral.py @@ -14,6 +14,7 @@ This module is invoked from main.py when RUNNER_MODE=ephemeral. from __future__ import annotations import concurrent.futures +import http.client import json import os import urllib.error @@ -768,33 +769,62 @@ def list_queued_self_hosted_jobs(pat: str) -> list[dict]: """ jobs: list[dict] = [] for repo in list_org_repos(pat): - try: - run_ids: set[int] = set() - for status in ("queued", "in_progress"): - runs = _github_request( - "GET", - f"/repos/{GITHUB_ORG}/{repo}/actions/runs?status={status}&per_page=50", - pat, - ) - if isinstance(runs, dict): - run_ids.update(r["id"] for r in runs.get("workflow_runs", [])) - for run_id in sorted(run_ids): - run_jobs = _github_request( - "GET", - f"/repos/{GITHUB_ORG}/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page=100", - pat, - ) - if not isinstance(run_jobs, dict): + # A one-off network blip (GitHub closing the connection, a timeout, a + # 5xx) gets one retry before it pages: a single dropped connection is + # not the safety net failing, but a repo that fails twice in a row is. + for attempt in (1, 2): + try: + jobs.extend(_queued_self_hosted_jobs_in_repo(repo, pat)) + break + except Exception as exc: # noqa: BLE001 + if attempt == 1 and _is_transient_github_error(exc): + print(f"Retrying {repo} in stalled-job scan after transient error: {exc!r}", flush=True) continue - for job in run_jobs.get("jobs", []): - labels = job.get("labels", []) - if job.get("status") == "queued" and "self-hosted" in labels and can_serve(labels): - jobs.append({**job, "repo": repo}) - except Exception as exc: # noqa: BLE001 - _log_error(f"Skipping {repo} in stalled-job scan: {exc!r}") + _log_error(f"Skipping {repo} in stalled-job scan: {exc!r}") + break return jobs +def _queued_self_hosted_jobs_in_repo(repo: str, pat: str) -> list[dict]: + """Queued jobs in one repo that one of our runner families can serve.""" + jobs: list[dict] = [] + run_ids: set[int] = set() + for status in ("queued", "in_progress"): + runs = _github_request( + "GET", + f"/repos/{GITHUB_ORG}/{repo}/actions/runs?status={status}&per_page=50", + pat, + ) + if isinstance(runs, dict): + run_ids.update(r["id"] for r in runs.get("workflow_runs", [])) + for run_id in sorted(run_ids): + run_jobs = _github_request( + "GET", + f"/repos/{GITHUB_ORG}/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page=100", + pat, + ) + if not isinstance(run_jobs, dict): + continue + for job in run_jobs.get("jobs", []): + labels = job.get("labels", []) + if job.get("status") == "queued" and "self-hosted" in labels and can_serve(labels): + jobs.append({**job, "repo": repo}) + return jobs + + +def _is_transient_github_error(exc: BaseException) -> bool: + """True for network-level failures and GitHub 5xx/429 worth one retry. + + Auth/permission/not-found errors (other 4xx) are not transient: retrying + won't help, so they page straight away. + """ + if isinstance(exc, urllib.error.HTTPError): + return exc.code == 429 or exc.code >= 500 + # URLError wraps DNS/connect failures; http.client.RemoteDisconnected is a + # ConnectionResetError; socket timeouts are TimeoutError (both OSError). + return isinstance(exc, (urllib.error.URLError, http.client.HTTPException, OSError)) + + def _family_of_runner_name(name: str) -> str | None: """``gha-gpu-windows-abc123`` → ``gpu-windows``; None if not ours.""" if not name.startswith("gha-") or "-" not in name[len("gha-"):]: diff --git a/infrastructure/functions/runner_manager/test_ephemeral.py b/infrastructure/functions/runner_manager/test_ephemeral.py index 899ecfe2..6a0b572c 100644 --- a/infrastructure/functions/runner_manager/test_ephemeral.py +++ b/infrastructure/functions/runner_manager/test_ephemeral.py @@ -736,6 +736,73 @@ class TestListQueuedSelfHostedJobs: job_calls = [c for c in req.call_args_list if "/runs/11/jobs" in c.args[1]] assert len(job_calls) == 1 + @staticmethod + def _scan_with_runs_failures(ep, failures): + """Scan one repo whose first runs-list calls raise ``failures`` in order.""" + pending = list(failures) + + def fake_request(method, path, pat, body=None): + if path.startswith("/orgs/test-org/repos"): + return [{"name": "karaoke-gen"}] + if "/actions/runs?status=queued" in path and pending: + raise pending.pop(0) + if "/actions/runs?status=queued" in path: + return {"workflow_runs": [{"id": 11}]} + if "/actions/runs?status=in_progress" in path: + return {"workflow_runs": []} + if "/actions/runs/11/jobs" in path: + return {"jobs": [{"id": 1, "status": "queued", "labels": BUILD_LABELS}]} + raise AssertionError(path) + + with patch.object(ep, "_github_request", side_effect=fake_request) as req: + jobs = ep.list_queued_self_hosted_jobs("ghp_test") + return jobs, req + + def test_one_dropped_connection_is_retried_without_paging(self, capsys): + import http.client + + ep = _fresh_module() + jobs, _ = self._scan_with_runs_failures( + ep, [http.client.RemoteDisconnected("Remote end closed connection without response")] + ) + + assert [(j["repo"], j["id"]) for j in jobs] == [("karaoke-gen", 1)] + out = capsys.readouterr().out + assert _error_messages(out) == [] + assert "Retrying karaoke-gen in stalled-job scan" in out + + def test_transient_error_twice_in_a_row_pages(self, capsys): + import http.client + + ep = _fresh_module() + jobs, _ = self._scan_with_runs_failures( + ep, [http.client.RemoteDisconnected("closed"), TimeoutError("timed out")] + ) + + assert jobs == [] + assert any("Skipping karaoke-gen in stalled-job scan" in m for m in _error_messages(capsys.readouterr().out)) + + def test_auth_error_pages_without_retry(self, capsys): + import urllib.error + + ep = _fresh_module() + forbidden = urllib.error.HTTPError("https://api.github.com/x", 403, "Forbidden", {}, None) + jobs, req = self._scan_with_runs_failures(ep, [forbidden]) + + assert jobs == [] + assert any("Skipping karaoke-gen in stalled-job scan" in m for m in _error_messages(capsys.readouterr().out)) + runs_calls = [c for c in req.call_args_list if "status=queued" in c.args[1]] + assert len(runs_calls) == 1 + + def test_github_5xx_is_transient(self): + import urllib.error + + ep = _fresh_module() + assert ep._is_transient_github_error(urllib.error.HTTPError("u", 502, "Bad Gateway", {}, None)) + assert ep._is_transient_github_error(urllib.error.HTTPError("u", 429, "Too Many", {}, None)) + assert not ep._is_transient_github_error(urllib.error.HTTPError("u", 404, "Not Found", {}, None)) + assert not ep._is_transient_github_error(RuntimeError("403 no access")) + class TestCleanupTriggersRedispatch: def test_redispatch_sees_only_surviving_vms(self): diff --git a/pyproject.toml b/pyproject.toml index b0dbde04..f9160996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.295.0" +version = "0.295.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"