diff --git a/frontend/__tests__/api-fetch.test.ts b/frontend/__tests__/api-fetch.test.ts index 33ef2c4d..9ae9079b 100644 --- a/frontend/__tests__/api-fetch.test.ts +++ b/frontend/__tests__/api-fetch.test.ts @@ -14,11 +14,12 @@ import { STALL_UNAVAILABLE_MS, } from '@/lib/backend-status' -function mockResponse(status: number, body: unknown = {}): Response { +function mockResponse(status: number, body: unknown = {}, headers: Record = {}): Response { return { ok: status >= 200 && status < 300, status, json: async () => body, + headers: { get: (name: string) => headers[name.toLowerCase()] ?? null }, } as unknown as Response } @@ -67,18 +68,23 @@ describe('apiFetch', () => { expect(getBackendStatus()).toBe('online') }) - it('retries a GET on repeated network failure, then throws BackendUnavailableError', async () => { + it('retries a GET on repeated network failure with spaced backoff, then throws BackendUnavailableError', async () => { jest.useFakeTimers() fetchMock.mockRejectedValue(new TypeError('Failed to fetch')) + // Network failures back off 2s then 9s: an edge rate-limit block surfaces + // as an opaque network error (no CORS on the block page) and lasts 10s — + // the final attempt must be able to land after the block expires. const p = apiFetch('/api/jobs/abc') const assertion = expect(p).rejects.toBeInstanceOf(BackendUnavailableError) - await jest.advanceTimersByTimeAsync(600) - await jest.advanceTimersByTimeAsync(1500) + await jest.advanceTimersByTimeAsync(2_000) + await jest.advanceTimersByTimeAsync(9_000) await assertion - expect(fetchMock).toHaveBeenCalledTimes(3) - // A fast total failure (~2s) is NOT a stall, so the banner never appears. + // The 11s in-flight span crosses the stall threshold, so a health probe + // fires through the same fetch mock — count only the real API attempts. + const apiCalls = fetchMock.mock.calls.filter((c) => !String(c[0]).includes('/api/health')) + expect(apiCalls).toHaveLength(3) expect(getBackendStatus()).toBe('online') }) @@ -96,6 +102,42 @@ describe('apiFetch', () => { expect(getBackendStatus()).toBe('online') }) + it('waits out the edge rate-limit window on 429, then retries and succeeds', async () => { + jest.useFakeTimers() + // Route by URL: the >10s in-flight span triggers a health probe through the + // same fetch mock, which must not consume the queued API responses. + const apiResponses = [mockResponse(429), mockResponse(200, { ok: true })] + fetchMock.mockImplementation((url: unknown) => { + if (String(url).includes('/api/health')) return Promise.resolve(mockResponse(200)) + return Promise.resolve(apiResponses.shift() ?? mockResponse(200, { ok: true })) + }) + const apiCalls = () => fetchMock.mock.calls.filter((c) => !String(c[0]).includes('/api/health')).length + + const p = apiFetch('/api/jobs/abc') + // Must NOT retry inside the 10s mitigation window (that both fails and + // prolongs the block) — no second API attempt for the first 10s... + await jest.advanceTimersByTimeAsync(10_000) + expect(apiCalls()).toBe(1) + // ...then the retry fires just past the window. + await jest.advanceTimersByTimeAsync(1_000) + const res = await p + expect(res.status).toBe(200) + expect(apiCalls()).toBe(2) + }) + + it('honors Retry-After on 429 when the edge provides it', async () => { + jest.useFakeTimers() + fetchMock + .mockResolvedValueOnce(mockResponse(429, undefined, { 'retry-after': '3' })) + .mockResolvedValueOnce(mockResponse(200, { ok: true })) + + const p = apiFetch('/api/jobs/abc') + await jest.advanceTimersByTimeAsync(3_600) + const res = await p + expect(res.status).toBe(200) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + it('recovers transparently when a retried GET succeeds on a later attempt', async () => { jest.useFakeTimers() fetchMock @@ -103,7 +145,7 @@ describe('apiFetch', () => { .mockResolvedValueOnce(mockResponse(200, { ok: true })) const p = apiFetch('/api/jobs/abc') - await jest.advanceTimersByTimeAsync(600) + await jest.advanceTimersByTimeAsync(2_000) const res = await p expect(res.status).toBe(200) diff --git a/frontend/app/[locale]/app/jobs/[[...slug]]/client.tsx b/frontend/app/[locale]/app/jobs/[[...slug]]/client.tsx index 0bc97531..ddeb9613 100644 --- a/frontend/app/[locale]/app/jobs/[[...slug]]/client.tsx +++ b/frontend/app/[locale]/app/jobs/[[...slug]]/client.tsx @@ -3,7 +3,7 @@ import { useEffect, useState, useCallback, useRef } from "react" import { reportDegradationEvent } from "@/lib/degradation-events" import { useAuth } from "@/lib/auth" -import { api, Job, createLyricsReviewApiClient, lyricsReviewApi } from "@/lib/api" +import { api, BackendUnavailableError, Job, createLyricsReviewApiClient, lyricsReviewApi } from "@/lib/api" import { Spinner } from "@/components/ui/spinner" import { Button } from "@/components/ui/button" import { ArrowLeft, AlertCircle } from "lucide-react" @@ -25,6 +25,7 @@ type AccessState = | { status: "loading" } | { status: "not_authenticated" } | { status: "not_authorized"; reason: string } + | { status: "backend_unavailable" } | { status: "job_not_found" } | { status: "wrong_state"; currentState: string; expectedStates: string[] } | { status: "invalid_route" } @@ -87,6 +88,7 @@ export function JobRouterClient() { const { user, isLoading: authLoading, hasHydrated } = useAuth() const [accessState, setAccessState] = useState({ status: "loading" }) + const [accessRetryNonce, setAccessRetryNonce] = useState(0) useEffect(() => { async function checkAccess() { @@ -152,6 +154,13 @@ export function JobRouterClient() { // Job not found or API error if (error && typeof error === "object" && "status" in error && error.status === 404) { setAccessState({ status: "job_not_found" }) + } else if (error instanceof BackendUnavailableError) { + // The job fetch never got a real answer (origin blip, or the request + // was blocked at the edge — e.g. a many-tab burst tripping the rate + // limit). This is NOT an authorization verdict; showing "Access + // denied" here (as we used to) told users they lacked permission for + // their own jobs. Offer a retry instead. + setAccessState({ status: "backend_unavailable" }) } else { setAccessState({ status: "not_authorized", @@ -162,7 +171,7 @@ export function JobRouterClient() { } checkAccess() - }, [inLocalMode, jobId, routeType, user, authLoading, hasHydrated, isReplay]) + }, [inLocalMode, jobId, routeType, user, authLoading, hasHydrated, isReplay, accessRetryNonce]) // Loading state if (accessState.status === "loading") { @@ -215,6 +224,32 @@ export function JobRouterClient() { ) } + // Backend unreachable while checking access — a retryable condition, not a verdict + if (accessState.status === "backend_unavailable") { + return ( +
+
+ +

Trouble reaching our servers

+

+ This is usually temporary — your job is safe. Please try again in a moment. +

+
+ + +
+
+
+ ) + } + // Not authorized if (accessState.status === "not_authorized") { return ( diff --git a/frontend/lib/api.ts b/frontend/lib/api.ts index a4948f1a..fd8fd8f6 100644 --- a/frontend/lib/api.ts +++ b/frontend/lib/api.ts @@ -516,9 +516,18 @@ const HARD_TIMEOUT_MS = 45_000; const GET_MAX_ATTEMPTS = 3; /** Backoff between GET retries (ms), indexed by prior-attempt number. */ const RETRY_BACKOFF_MS = [600, 1500]; +/** Backoff for network-level failures (fetch rejected). A cross-origin request + * blocked by Cloudflare's rate limit surfaces as an opaque network error (its + * block page has no CORS headers), and the mitigation window is 10s — quick + * 0.6/1.5s retries all land inside the block AND add fuel to the count. Space + * the retries so the last attempt can land after a block window has expired. */ +const NETWORK_RETRY_BACKOFF_MS = [2_000, 9_000]; /** Edge/origin status codes that mean "backend briefly unreachable", not a real - * application error: standard 502/503/504 plus Cloudflare's origin-reachability 52x. */ -const TRANSIENT_STATUS = new Set([502, 503, 504, 520, 521, 522, 523, 524]); + * application error: standard 502/503/504 plus Cloudflare's origin-reachability + * 52x, and 429 (edge rate-limit block, when the response is readable). */ +const TRANSIENT_STATUS = new Set([429, 502, 503, 504, 520, 521, 522, 523, 524]); +/** Wait before retrying a 429: at least the edge mitigation window. */ +const RATE_LIMITED_RETRY_MS = 10_500; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -658,7 +667,7 @@ export async function apiFetch( if (callerSignal?.aborted) throw err; lastError = err; if (attempt < maxAttempts) { - await sleep(RETRY_BACKOFF_MS[Math.min(attempt - 1, RETRY_BACKOFF_MS.length - 1)]); + await sleep(NETWORK_RETRY_BACKOFF_MS[Math.min(attempt - 1, NETWORK_RETRY_BACKOFF_MS.length - 1)]); continue; } throw new BackendUnavailableError(err); @@ -667,7 +676,19 @@ export async function apiFetch( if (TRANSIENT_STATUS.has(response.status)) { if (allowRetry && attempt < maxAttempts) { - await sleep(RETRY_BACKOFF_MS[Math.min(attempt - 1, RETRY_BACKOFF_MS.length - 1)]); + if (response.status === 429) { + // Rate-limited at the edge: honor Retry-After when present, else + // wait out the full mitigation window — retrying sooner both fails + // and prolongs the block. + const retryAfterS = Number(response.headers.get('retry-after')); + await sleep( + Number.isFinite(retryAfterS) && retryAfterS > 0 + ? retryAfterS * 1000 + 500 + : RATE_LIMITED_RETRY_MS, + ); + } else { + await sleep(RETRY_BACKOFF_MS[Math.min(attempt - 1, RETRY_BACKOFF_MS.length - 1)]); + } continue; } if (isGet) { diff --git a/infrastructure/config.py b/infrastructure/config.py index b449ee84..f5da1a4b 100644 --- a/infrastructure/config.py +++ b/infrastructure/config.py @@ -182,9 +182,17 @@ class CloudflareConfig: # Free plan constraints (enforced by the Cloudflare API): period must be 10s, # and mitigation_timeout must equal the period (10s). A single flood control # rule — the WAF path block (below) is the primary defense against the - # scanner class; this catches volumetric abuse. ~50 req / 10s = 5 req/s per - # IP, comfortably above legit page-load bursts to a control-plane API. - RATE_LIMIT_REQUESTS = 50 + # scanner class; this catches volumetric abuse. + # + # Sized for the legitimate worst case from ONE IP (2026-09-22 incident): an + # admin reloading 16 review tabs at once fires ~10 API calls per tab within + # seconds. At the old 50/10s the edge blocked a chunk of them — and because + # CF block pages carry no CORS headers, the browser saw opaque network + # failures ("Access denied" / "temporarily unavailable" screens) while the + # origin was fully healthy. 200/10s (20 req/s/IP) still stops volumetric + # abuse; OPTIONS preflights and /api/health are excluded from counting (see + # create_rate_limit_ruleset). + RATE_LIMIT_REQUESTS = 200 RATE_LIMIT_PERIOD_SECONDS = 10 RATE_LIMIT_MITIGATION_SECONDS = 10 diff --git a/infrastructure/modules/edge_security.py b/infrastructure/modules/edge_security.py index 7a0e2fcd..69c791a0 100644 --- a/infrastructure/modules/edge_security.py +++ b/infrastructure/modules/edge_security.py @@ -250,10 +250,20 @@ def create_waf_rulesets(zone_id: str, hosts: list[str]) -> dict[str, cloudflare. # --------------------------------------------------------------------------- # def create_rate_limit_ruleset(zone_id: str, hosts: list[str]) -> cloudflare.Ruleset: host_expr = _host_match_expression(hosts) - # Don't rate-limit OIDC-authed scheduler hits to /api/internal/*. + # Exclusions from rate-limit counting: + # - /api/internal/*: OIDC-authed scheduler hits. + # - OPTIONS: CORS preflights double every browser API call's edge count, and + # a blocked preflight kills the real request even when it would have + # passed. Preflights are cheap and carry no abuse value. + # - /api/health: the frontend's connectivity probe must reflect ORIGIN + # health. If probes get rate-limited alongside a burst, every open tab + # shows the outage banner for an origin that is perfectly healthy + # (2026-09-22 16-tab incident). expression = ( f'{host_expr} and ' - f'(not starts_with(http.request.uri.path, "{CloudflareConfig.INTERNAL_PATH_PREFIX}"))' + f'(not starts_with(http.request.uri.path, "{CloudflareConfig.INTERNAL_PATH_PREFIX}")) and ' + f'(http.request.method ne "OPTIONS") and ' + f'(http.request.uri.path ne "/api/health")' ) return cloudflare.Ruleset( "edge-ratelimit", diff --git a/pyproject.toml b/pyproject.toml index 2b67a8ba..2d13e6e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.232.1" +version = "0.233.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"