diff --git a/backend/main.py b/backend/main.py index c547b1f7..23330d9c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -200,10 +200,17 @@ async def lifespan(app: FastAPI): # self-diagnosing (logs the blocking stack; records >=5s stalls in prod). watchdog = None if settings.loop_watchdog_enabled: - from backend.services.loop_watchdog import LoopWatchdog, firestore_stall_recorder + from backend.services.loop_watchdog import ( + LoopWatchdog, + firestore_stall_recorder, + request_tracker, + ) watchdog = LoopWatchdog( recorder=firestore_stall_recorder if settings.environment == "production" else None, + # Stalls nobody waited on (idle, CPU-throttled autoscaled instances) + # stay WARNING so they don't trip the >=10s stall alert. + request_tracker=request_tracker, ) watchdog.start() @@ -292,6 +299,12 @@ app.add_middleware(TenantMiddleware) # backend/middleware/edge_auth.py. app.add_middleware(EdgeAuthMiddleware) +# Counts in-flight requests for the loop watchdog (outermost, so every request that +# waited on a stalled loop counts — even ones the edge lock rejects). +from backend.services.loop_watchdog import RequestTrackingMiddleware # noqa: E402 + +app.add_middleware(RequestTrackingMiddleware) + # Include routers app.include_router(health.router, prefix="/api") app.include_router(jobs.router, prefix="/api") diff --git a/backend/services/loop_watchdog.py b/backend/services/loop_watchdog.py index ad85c597..f4a7d509 100644 --- a/backend/services/loop_watchdog.py +++ b/backend/services/loop_watchdog.py @@ -46,6 +46,55 @@ _MAX_STACK_CHARS = 8000 Recorder = Callable[[dict], None] + +class RequestTracker: + """Counts HTTP requests so the watchdog can tell whether a stall hurt anyone. + + Mutated only on the event-loop thread (by ``RequestTrackingMiddleware``) and + read from the watchdog thread; plain ints are safe to read under the GIL. + + Why (2026-10-12/13 flapping alert): Cloud Run autoscaling starts extra + instances that serve one request and then sit idle, CPU-throttled, while the + SpaCy/NLTK warmup thread trickles along for 10+ minutes. Those idle instances + produced 10-40s "stalls" (empty culprit — the freeze consumed the sampling + schedule) with no request on them, tripping the >=2-in-30min alert. A stall + with no request in flight and none queued behind it affects nobody. + """ + + def __init__(self) -> None: + self.in_flight = 0 + self.started = 0 + + def begin(self) -> None: + self.in_flight += 1 + self.started += 1 + + def end(self) -> None: + self.in_flight = max(0, self.in_flight - 1) + + +# Process-wide tracker: the middleware is attached at import time, the watchdog +# is created later in the lifespan — both share this instance. +request_tracker = RequestTracker() + + +class RequestTrackingMiddleware: + """Pure-ASGI middleware feeding ``request_tracker`` (no per-request allocation + beyond the wrapper call; outermost so even rejected requests count).""" + + def __init__(self, app, tracker: Optional[RequestTracker] = None) -> None: + self.app = app + self.tracker = tracker or request_tracker + + async def __call__(self, scope, receive, send): + if scope.get("type") != "http": + return await self.app(scope, receive, send) + self.tracker.begin() + try: + return await self.app(scope, receive, send) + finally: + self.tracker.end() + # Innermost frames that mean "the loop is idle, waiting for I/O" — not blocked. _IDLE_FUNCS = {"select", "poll", "epoll", "kqueue", "control", "_run_once"} _IDLE_FILES = ("selectors.py", "base_events.py") @@ -91,6 +140,8 @@ class LoopWatchdog: cpu_clock: Callable[[], float] = time.process_time, frozen_gap_s: float = 0.75, frozen_cpu_ratio: float = 0.25, + request_tracker: Optional[RequestTracker] = None, + request_grace_s: float = 0.5, ) -> None: self.tick_s = tick_s self.stall_threshold_s = stall_threshold_s @@ -116,6 +167,16 @@ class LoopWatchdog: self._last_check: Optional[float] = None self._last_cpu = 0.0 self._stall_frozen_s = 0.0 + # With a tracker, a stall only escalates to ERROR (the alerting level) if + # a request was in flight when it began or arrived during it. Requests + # queued in the socket during the freeze are accepted right after the loop + # resumes, so the verdict waits ``request_grace_s`` after resume. + self.request_tracker = request_tracker + self.request_grace_s = request_grace_s + self._stall_in_flight = 0 + self._stall_started_count = 0 + self._resumed_at: Optional[float] = None + self._resumed_duration = 0.0 self._last_tick = clock() self._loop_thread_id: Optional[int] = None @@ -182,7 +243,15 @@ class LoopWatchdog: if self._samples and self._samples[-1]["stack"] == stack: return # identical to the previous sample — nothing new to learn self._samples.append({"at_s": round(age, 1), "culprit": culprit, "stack": stack, "idle": False}) - level = logging.ERROR if age >= self.error_threshold_s else logging.WARNING + # Same impact rule as _finish: an idle instance's stall never logs ERROR. + # (Requests still queued in the socket are invisible until the loop + # resumes, so this can under-call mid-stall; the ended line decides.) + no_impact = self.request_tracker is not None and self._requests_affected() == 0 + level = ( + logging.ERROR + if age >= self.error_threshold_s and not no_impact + else logging.WARNING + ) logger.log( level, "EVENT_LOOP_STALL in progress age_ms=%d culprit=%s\n%s", @@ -215,8 +284,17 @@ class LoopWatchdog: self._samples = [] self._next_sample_idx = 0 self._stall_frozen_s = 0.0 + self._resumed_at = None + if self.request_tracker is not None: + self._stall_in_flight = self.request_tracker.in_flight + self._stall_started_count = self.request_tracker.started else: return + if self._resumed_at is not None: + # Loop already resumed; waiting out the grace window for queued requests. + if self._requests_affected() or now - self._resumed_at >= self.request_grace_s: + self._finish(self._resumed_duration) + return # Only the part of the frozen gap after the last heartbeat belongs to this stall. self._stall_frozen_s += min(frozen, age) @@ -225,6 +303,10 @@ class LoopWatchdog: # The heartbeat that ended the stall fired one tick_s after the # previous one would have, so subtract the nominal sleep. duration = max(0.0, last - self._stall_started_tick - self.tick_s) + if self.request_tracker is not None and not self._requests_affected(): + self._resumed_at = now + self._resumed_duration = duration + return self._finish(duration) return @@ -245,9 +327,19 @@ class LoopWatchdog: self._next_sample_idx += 1 self._sample(age) + def _requests_affected(self) -> int: + """Requests that waited on this stall: in flight at its start + started since.""" + t = self.request_tracker + if t is None: + return 0 + return self._stall_in_flight + max(0, t.started - self._stall_started_count) + def _finish(self, duration: float) -> None: samples = self._samples frozen_s = self._stall_frozen_s + affected = self._requests_affected() + tracked = self.request_tracker is not None + self._resumed_at = None self._in_stall = False self._samples = [] self._stall_frozen_s = 0.0 @@ -268,15 +360,23 @@ class LoopWatchdog: culprits = [s["culprit"] for s in samples if s["culprit"]] # The longest-lived sample is the most representative of the freeze. main_culprit = culprits[-1] if culprits else "" - level = logging.ERROR if duration >= self.error_threshold_s else logging.WARNING + # No request waited on it (idle, typically CPU-throttled instance) → never + # ERROR: the alert counts ERROR lines and nobody saw this freeze. + no_impact = tracked and affected == 0 + level = ( + logging.ERROR + if duration >= self.error_threshold_s and not no_impact + else logging.WARNING + ) logger.log( level, - "EVENT_LOOP_STALL ended duration_ms=%d culprit=%s", - int(duration * 1000), main_culprit, + "EVENT_LOOP_STALL ended duration_ms=%d culprit=%s requests_affected=%s", + int(duration * 1000), main_culprit, affected if tracked else "unknown", extra={ "loop_stall_duration_ms": int(duration * 1000), "loop_stall_culprit": main_culprit, "loop_stall_culprits": culprits, + "loop_stall_requests_affected": affected if tracked else None, }, ) if self.recorder is not None and duration >= self.record_threshold_s: @@ -286,6 +386,7 @@ class LoopWatchdog: "culprit": main_culprit, "culprits": culprits, "stack": samples[-1]["stack"] if samples else "", + "requests_affected": affected if tracked else None, }) except Exception: logger.exception("Failed to record loop stall (non-fatal)") @@ -332,6 +433,7 @@ def firestore_stall_recorder(stall: dict) -> None: "detail": { "duration_ms": stall["duration_ms"], "culprit": stall["culprit"][:300], + "requests_affected": stall.get("requests_affected"), }, "culprits": [c[:300] for c in stall.get("culprits", [])][:5], "stack": stall.get("stack", "")[:_MAX_STACK_CHARS], diff --git a/backend/tests/test_loop_watchdog.py b/backend/tests/test_loop_watchdog.py index 65e59a2e..dfbb0d9e 100644 --- a/backend/tests/test_loop_watchdog.py +++ b/backend/tests/test_loop_watchdog.py @@ -290,3 +290,156 @@ def test_is_idle_frame_detects_parked_selector(): sel.close() a.close() b.close() + + +class TestRequestAwareSeverity: + """Only stalls a request waited on reach ERROR (the alerting level). + + 2026-10-12/13 flapping alert: autoscaled instances served one request, then sat + idle and CPU-throttled during the 10-min SpaCy warmup; their 10-40s "stalls" + had no request on them yet paged twice.""" + + def _wd(self): + from backend.services.loop_watchdog import RequestTracker + + clock = FakeClock() + tracker = RequestTracker() + recorded = [] + wd = LoopWatchdog(clock=clock, cpu_clock=clock, recorder=recorded.append, + request_tracker=tracker) + wd._loop_thread_id = None + return wd, clock, tracker, recorded + + def _stall(self, wd, clock, seconds, during=None, after_resume=None): + wd._last_tick = clock.t + start = clock.t + t = 0.0 + while t < seconds: + t = min(seconds, t + 0.25) + clock.t = start + t + wd.check() + if during and t >= seconds / 2: + during() + during = None + clock.t += 0.25 + wd._last_tick = clock.t # heartbeat resumes + wd.check() + if after_resume: + after_resume() + for _ in range(3): # grace window checks + clock.t += 0.25 + wd._last_tick = clock.t + wd.check() + + def _ended(self, caplog): + return [r for r in caplog.records if "EVENT_LOOP_STALL ended" in r.getMessage()] + + def test_idle_instance_long_stall_is_warning(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + self._stall(wd, clock, 21.0) + ended = self._ended(caplog) + assert len(ended) == 1 + assert ended[0].levelno == logging.WARNING + assert "requests_affected=0" in ended[0].getMessage() + assert recorded[0]["requests_affected"] == 0 # still recorded for forensics + + def test_idle_in_progress_samples_are_warning(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + wd._loop_thread_id = 12345 + with patch("backend.services.loop_watchdog._is_idle_frame", return_value=False), \ + patch("backend.services.loop_watchdog._format_stack", + side_effect=lambda f, n=iter(range(99)): (f"stack{next(n)}", "x.py:1")), \ + patch("backend.services.loop_watchdog.sys._current_frames", return_value={12345: None}): + self._stall(wd, clock, 16.0) + progress = [r for r in caplog.records if "in progress" in r.getMessage()] + assert len(progress) == 3 # 1s, 5s, 15s samples + assert all(r.levelno == logging.WARNING for r in progress) + + def test_in_flight_in_progress_sample_at_15s_is_error(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + wd._loop_thread_id = 12345 + tracker.begin() + with patch("backend.services.loop_watchdog._is_idle_frame", return_value=False), \ + patch("backend.services.loop_watchdog._format_stack", + side_effect=lambda f, n=iter(range(99)): (f"stack{next(n)}", "x.py:1")), \ + patch("backend.services.loop_watchdog.sys._current_frames", return_value={12345: None}): + self._stall(wd, clock, 16.0) + progress = [r for r in caplog.records if "in progress" in r.getMessage()] + assert [r.levelno for r in progress] == [logging.WARNING, logging.WARNING, logging.ERROR] + + def test_in_flight_request_makes_it_error(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + tracker.begin() + self._stall(wd, clock, 12.0) + ended = self._ended(caplog) + assert ended[0].levelno == logging.ERROR + assert "requests_affected=1" in ended[0].getMessage() + + def test_request_queued_during_stall_accepted_after_resume_is_error(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + self._stall(wd, clock, 12.0, after_resume=tracker.begin) + ended = self._ended(caplog) + assert len(ended) == 1 + assert ended[0].levelno == logging.ERROR + + def test_request_finished_mid_stall_still_counts(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + tracker.begin() + self._stall(wd, clock, 12.0, during=tracker.end) + assert self._ended(caplog)[0].levelno == logging.ERROR + + def test_short_idle_stall_logged_once_after_grace(self, caplog): + caplog.set_level(logging.WARNING) + wd, clock, tracker, recorded = self._wd() + self._stall(wd, clock, 2.0) + ended = self._ended(caplog) + assert len(ended) == 1 + assert "duration_ms=2000" in ended[0].getMessage() + assert not wd._in_stall + + def test_without_tracker_severity_unchanged(self, caplog): + caplog.set_level(logging.WARNING) + clock = FakeClock() + wd = LoopWatchdog(clock=clock, cpu_clock=clock) + wd._loop_thread_id = None + self._stall(wd, clock, 12.0) + ended = self._ended(caplog) + assert ended[0].levelno == logging.ERROR + assert "requests_affected=unknown" in ended[0].getMessage() + + +class TestRequestTrackingMiddleware: + @pytest.mark.asyncio + async def test_counts_http_requests_including_errors(self): + from backend.services.loop_watchdog import RequestTracker, RequestTrackingMiddleware + + tracker = RequestTracker() + seen = [] + + async def app(scope, receive, send): + seen.append(tracker.in_flight) + if scope.get("path") == "/boom": + raise RuntimeError("boom") + + mw = RequestTrackingMiddleware(app, tracker=tracker) + await mw({"type": "http", "path": "/"}, None, None) + with pytest.raises(RuntimeError): + await mw({"type": "http", "path": "/boom"}, None, None) + await mw({"type": "lifespan"}, None, None) + assert seen == [1, 1, 0] + assert tracker.in_flight == 0 + assert tracker.started == 2 + + def test_main_app_installs_middleware(self): + from backend.main import app + from backend.services.loop_watchdog import RequestTrackingMiddleware + + classes = [m.cls for m in app.user_middleware] + # add_middleware inserts at index 0 → outermost is first. + assert classes[0] is RequestTrackingMiddleware diff --git a/docs/API.md b/docs/API.md index abb5d0e0..d6635f47 100644 --- a/docs/API.md +++ b/docs/API.md @@ -3002,7 +3002,8 @@ ignored. `device_fingerprint` is the FingerprintJS `visitorId` (same one the mag abuse checks store), `tab_id` is per browser tab, `episode_id` groups one degradation episode (first banner → `banner_recovered`), and `tenant` is derived from the page URL. `server_loop_stall` docs (written by the backend's event-loop watchdog, `source: "server"`) -share the collection; clients can't submit that type (422). +share the collection; clients can't submit that type (422). Their `detail.requests_affected` +counts requests that waited on the stall (0 = idle instance; the field is absent on events written before v0.268.5). Summary: `python scripts/client_events_report.py [--days N] [--all]`. Each event emits a structured `client_event type=…` INFO log line and is diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 64060589..68af7443 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -61,6 +61,7 @@ The "servers unavailable" banner (49 episodes in 2 weeks) was mostly the API's o - **An `await` somewhere in the handler doesn't make its other calls safe** (v0.267.1). `mark_uploads_complete`, `create_job_with_upload_urls` and others awaited `request.json()`, so they passed the guard, yet called `check_dropbox_credentials()` inline. When the Dropbox token had expired, that did an HTTP refresh plus a Secret Manager write and version cleanup on the loop, a 16s freeze. `trigger_gdrive_validation_endpoint` froze 5-10s after every job (OIDC fetch + POST). Both are now behind `asyncio.to_thread`, and the guard's `BLOCKING_CALLS` denylist fails CI if a known-blocking function is called directly in an `async def` route. Add to the denylist when the watchdog names a new culprit. - **Mostly-sync routes that await one trigger** (v0.268.1). `complete_review` (21s corrections `upload_json`), `create_job_from_search` (theme prep), and the `recover_stuck_jobs` / `retry_pending_render_jobs` crons (~260 stalls/week from Firestore scans) each awaited a worker trigger, so the guard passed them. Their bodies now run in a worker thread, and the trigger coroutines go back to the API loop with `backend/utils/loop_bridge.run_on_loop` (`run_coroutine_threadsafe`, 120s timeout), so loop APIs inside the triggers still work. Bodies that call `run_on_loop` run on their own pool (`run_in_body_thread`), not the default executor. Otherwise they could hold every default-pool thread while the coroutines they wait on need one, a deadlock. Watch out for an `async def` helper with no awaits inside, such as `_validate_audio_durations` (2 downloads + 2 decodes) or the YouTube queue's `_process_single_upload` (a whole upload): it's sync work on the loop with an async signature. To rank culprits, group `EVENT_LOOP_STALL in progress` stacks by their first `api/routes/` frame. - **Detection:** `backend/services/loop_watchdog.py` logs `EVENT_LOOP_STALL` with the loop thread's stack and a `culprit=file:line`. Grep for it before guessing. +- **Alert on impact, not on gaps** (v0.268.5). After the CPU-throttle discount, the stall alert still flapped. Cloud Run autoscaling starts extra instances that serve one request, then sit idle and throttled while the startup warmup thread, which spends 10+ minutes loading SpaCy, competes for slices. Every 10-40s ERROR "stall" over 4 days (11 of them) was on an instance with **zero** requests, and all had an empty culprit. A freeze advances the 1/5/15s sample schedule without sampling, so whatever happens after the thaw has no stack. The watchdog now takes a `RequestTracker`, fed by the outermost pure-ASGI `RequestTrackingMiddleware`. A stall reaches ERROR only if a request was in flight at its start or arrived during it. A request that queued during the stall is counted if it starts within the 0.5s grace window after the loop resumes. Check whether anyone waited before chasing a culprit: join `labels.instanceId` between the stall lines and the request logs. --- diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index d77e2593..5ce5937e 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -75,6 +75,8 @@ defense-in-depth beneath the signing-free review path. 3. **Measure user impact:** `python scripts/client_events_report.py --days 7`. It excludes admin/internal/test accounts unless you pass `--all`, and lists `server_loop_stall` culprits. 4. **If there are no `EVENT_LOOP_STALL` lines:** the cause is elsewhere (cold start, deploy swap, Cloudflare). Check `banner_waking` events and revision times. +5. **Alert "≥2 event-loop stalls of 10s+" fired but no user complained:** check `requests_affected=` on the `EVENT_LOOP_STALL ended` lines. Since v0.268.5 a stall is ERROR, the level the alert counts, only if a request was in flight or queued behind it. Before that, idle autoscaled instances that were CPU-throttled during the ~10-minute SpaCy warmup logged 10-40s "stalls" with an empty `culprit=` and paged (2026-10-12/13). Lines with `requests_affected=0` stay at WARNING. + Background: `docs/archive/2026-10-03-backend-loop-freezes-telemetry-plan.md`. ## Emails not arriving / E2E "Timed out waiting for email" (Postmark HTML 403) diff --git a/pyproject.toml b/pyproject.toml index 5c3e903c..b2ee69b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.268.4" +version = "0.268.5" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"