From 9b6c4f236e544fd36eb63ce4cf8f52d0876cc732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 27 Aug 2026 05:07:17 -0400 Subject: [PATCH] fix: Cloud Tasks queue region from GPU workers + REVIEW_COMPLETE lost-trigger recovery (v0.202.1) (#948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Follow-up to #947, found by live-testing auto-approval in prod (job `b8bda9c2` β€” the first real fully-auto-approved job). **Bug:** the GPU `audio-separation-job` sets `GCP_REGION` to its GPU region, but every Cloud Tasks queue lives in `us-central1`. Any queue enqueue from inside that worker fails with `404 Queue does not exist` β€” which silently dropped the auto-approval render trigger (job auto-approved perfectly, then sat at `review_complete` forever). This is also a latent pre-existing bug for any other enqueue attempted from the GPU job. **Fixes:** 1. New `settings.cloud_tasks_region` (default `us-central1`, overridable via `CLOUD_TASKS_REGION`) used for all Cloud Tasks queue paths. `gcp_region` is still used for Cloud Run Job locations (correct). 2. `recover-stuck-jobs` cron gains a sweep for `REVIEW_COMPLETE` jobs stalled >10 min with no `render_progress` β€” re-triggers the render worker (idempotent via worker-generation fence). This closes the lost-trigger orphan gap for BOTH the human complete-review fire-and-forget task and the auto-approval path, and will auto-rescue `b8bda9c2` on the first tick after deploy. ## Testing - New unit tests: queue path uses `us-central1` even when `GCP_REGION=us-east4`; stall detector (stalled / fresh / render-underway). - Note: `test_internal_api.py` auth tests fail identically on pristine `main` in this local env (green in CI) β€” pre-existing, unrelated. @coderabbitai ignore πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 --- backend/api/routes/internal.py | 62 ++++++++++++++++- backend/config.py | 5 ++ backend/services/worker_service.py | 6 +- .../tests/test_recover_stuck_jobs_repark.py | 50 ++++++++++++-- .../tests/test_worker_service_queue_region.py | 68 +++++++++++++++++++ pyproject.toml | 2 +- scripts/review_capture.py | 6 +- 7 files changed, 185 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_worker_service_queue_region.py diff --git a/backend/api/routes/internal.py b/backend/api/routes/internal.py index 55716b59..d1dd1b44 100644 --- a/backend/api/routes/internal.py +++ b/backend/api/routes/internal.py @@ -901,10 +901,41 @@ async def recover_stuck_jobs( if _repark_stalled_render(job_manager, job_id): reparked.append(job_id) + # --- REVIEW_COMPLETE with no render started: lost render trigger --- + # Both the human complete-review endpoint (fire-and-forget task) and the + # auto-approval executor can lose the render trigger (e.g. Cloud Tasks + # enqueue failure). Without this, the job sits at review_complete forever + # with no error and no Retry button. Re-trigger is idempotent (worker + # generation fence + status validation). + # REVIEW_COMPLETE is a transient state (render normally starts in seconds), + # so the population is tiny β€” scan it all (no limit, which could otherwise + # return only fresh jobs and starve stalled ones) and cap re-triggers per + # tick to bound the blast radius. + render_retriggered = [] + rc_query = jobs_ref.where( + filter=FieldFilter("status", "==", JobStatus.REVIEW_COMPLETE.value) + ).stream() + for doc in rc_query: + if len(render_retriggered) >= RENDER_RETRIGGERS_PER_TICK: + break + job_id = (doc.to_dict() or {}).get("job_id", doc.id) + job = job_manager.get_job(job_id) + if not job: + continue + if not _review_complete_stalled(job): + continue + logger.warning(f"[job:{job_id}] REVIEW_COMPLETE stalled >10 min with no render β€” re-triggering") + try: + if await worker_service.trigger_render_video_worker(job_id): + render_retriggered.append(job_id) + except Exception as e: + logger.warning(f"[job:{job_id}] render re-trigger failed: {e}") + logger.info( f"RECOVER_STUCK_JOBS complete: recovered={len(recovered)} " f"download_parked={len(download_parked)} download_retried={len(download_retried)} " - f"download_timed_out={len(download_timed_out)} reparked={len(reparked)}" + f"download_timed_out={len(download_timed_out)} reparked={len(reparked)} " + f"render_retriggered={len(render_retriggered)}" ) add_span_event("recovery_complete", { "recovered_count": len(recovered), @@ -923,9 +954,38 @@ async def recover_stuck_jobs( "download_timed_out_jobs": download_timed_out, "reparked_render_jobs": reparked, "reparked_render_count": len(reparked), + "render_retriggered_jobs": render_retriggered, + "render_retriggered_count": len(render_retriggered), } +REVIEW_COMPLETE_STALL_SECONDS = 10 * 60 # render normally starts within seconds +RENDER_RETRIGGERS_PER_TICK = 10 # bound the blast radius per 5-min tick + + +def _review_complete_stalled(job) -> bool: + """True when a REVIEW_COMPLETE job has sat >10 min with no render underway. + + ``render_progress`` is cleared on review completion and (re)written by the + render worker β€” its absence plus a stale ``updated_at`` means the trigger + was lost. + """ + if (job.state_data or {}).get("render_progress"): + return False + updated_at = getattr(job, "updated_at", None) + if updated_at is None: + return False + from datetime import datetime, timezone as _tz + now = datetime.now(_tz.utc) + if updated_at.tzinfo is None: + now = datetime.utcnow() + try: + age = (now - updated_at).total_seconds() + except TypeError: + return False + return age > REVIEW_COMPLETE_STALL_SECONDS + + # Torrent sources whose downloads are worth auto-retrying over a long window β€” # they fail transiently when a rare release has few/intermittent seeders or the # private tracker is briefly unreachable. Non-torrent sources (YouTube/Spotify/ diff --git a/backend/config.py b/backend/config.py index 023473e6..548f4f24 100644 --- a/backend/config.py +++ b/backend/config.py @@ -163,6 +163,11 @@ class Settings(BaseSettings): # When disabled (default), workers are triggered via direct HTTP (for development) enable_cloud_tasks: bool = os.getenv("ENABLE_CLOUD_TASKS", "false").lower() in ("true", "1", "yes") gcp_region: str = os.getenv("GCP_REGION", "us-central1") + # ALL Cloud Tasks queues live in us-central1, but workers running in other + # regions (the GPU audio-separation-job sets GCP_REGION to its GPU region) + # must still enqueue to the real queue location β€” building queue paths from + # gcp_region 404s from those workers ("Queue does not exist"). + cloud_tasks_region: str = os.getenv("CLOUD_TASKS_REGION", "us-central1") # Cloud Run Jobs (for long-running video encoding) # When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index cd338174..680a7c52 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -249,7 +249,7 @@ class WorkerService: logger.error("GOOGLE_CLOUD_PROJECT not set, cannot enqueue Cloud Task") return False - location = self.settings.gcp_region # Must match queue location + location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region # Build queue path parent = self.tasks_client.queue_path(project, location, queue_name) @@ -754,7 +754,7 @@ class WorkerService: logger.error("GOOGLE_CLOUD_PROJECT not set") return False - location = self.settings.gcp_region + location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region # Build queue path parent = self.tasks_client.queue_path(project, location, queue_name) @@ -843,7 +843,7 @@ class WorkerService: logger.error("GOOGLE_CLOUD_PROJECT not set") return False - location = self.settings.gcp_region + location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region # Build queue path parent = self.tasks_client.queue_path(project, location, queue_name) diff --git a/backend/tests/test_recover_stuck_jobs_repark.py b/backend/tests/test_recover_stuck_jobs_repark.py index 5f4dbcb4..f8c66c1e 100644 --- a/backend/tests/test_recover_stuck_jobs_repark.py +++ b/backend/tests/test_recover_stuck_jobs_repark.py @@ -55,16 +55,19 @@ def _job(job_id, status, *, minutes_stale=0, source_name=None, download_retry=No ) -def _mock_jm(*, downloading=None, pending=None, rendering=None, jobs=None): - """Wire a JobManager mock for the endpoint's 3 status queries. +def _mock_jm(*, downloading=None, pending=None, rendering=None, review_complete=None, jobs=None): + """Wire a JobManager mock for the endpoint's 4 status queries. - downloading/rendering use .where().stream(); pending uses .where().limit().stream(). - `jobs` maps job_id -> Job returned by get_job(). + downloading/rendering/review_complete use .where().stream(); pending uses + .where().limit().stream(). `jobs` maps job_id -> Job returned by get_job(). """ mock_jm = MagicMock() where = mock_jm.firestore.db.collection.return_value.where.return_value - # Two non-limited .stream() calls in order: downloading_audio, rendering_video - where.stream.side_effect = [iter(downloading or []), iter(rendering or [])] + # Non-limited .stream() calls in order: downloading_audio, rendering_video, + # review_complete (lost-render-trigger sweep) + where.stream.side_effect = [ + iter(downloading or []), iter(rendering or []), iter(review_complete or []), + ] # One .limit().stream() call: download_pending_retry where.limit.return_value.stream.return_value = iter(pending or []) mock_jm.get_job.side_effect = lambda jid: (jobs or {}).get(jid) @@ -175,3 +178,38 @@ def test_fresh_render_not_reparked(client): resp, _ = _run(client, mock_jm) assert resp.json()["reparked_render_count"] == 0 mock_jm.transition_to_state.assert_not_called() + + +def test_stalled_review_complete_render_is_retriggered(client): + """REVIEW_COMPLETE >10 min with no render_progress = lost render trigger + (e.g. Cloud Tasks enqueue failed from the GPU worker) -> re-trigger.""" + job = _job("rc1", JobStatus.REVIEW_COMPLETE, minutes_stale=30) + mock_jm = _mock_jm( + review_complete=[_doc("rc1", JobStatus.REVIEW_COMPLETE)], + jobs={"rc1": job}, + ) + mock_ws = MagicMock() + mock_ws.trigger_audio_download_worker = AsyncMock(return_value=True) + mock_ws.trigger_render_video_worker = AsyncMock(return_value=True) + with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \ + patch("backend.services.worker_service.get_worker_service", return_value=mock_ws): + resp = client.post("/api/internal/recover-stuck-jobs") + assert resp.status_code == 200 + assert resp.json()["render_retriggered_jobs"] == ["rc1"] + mock_ws.trigger_render_video_worker.assert_awaited_once_with("rc1") + + +def test_fresh_review_complete_is_left_alone(client): + job = _job("rc2", JobStatus.REVIEW_COMPLETE, minutes_stale=1) + mock_jm = _mock_jm( + review_complete=[_doc("rc2", JobStatus.REVIEW_COMPLETE)], + jobs={"rc2": job}, + ) + mock_ws = MagicMock() + mock_ws.trigger_render_video_worker = AsyncMock(return_value=True) + with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \ + patch("backend.services.worker_service.get_worker_service", return_value=mock_ws): + resp = client.post("/api/internal/recover-stuck-jobs") + assert resp.status_code == 200 + assert resp.json()["render_retriggered_jobs"] == [] + mock_ws.trigger_render_video_worker.assert_not_awaited() diff --git a/backend/tests/test_worker_service_queue_region.py b/backend/tests/test_worker_service_queue_region.py new file mode 100644 index 00000000..1ffafe63 --- /dev/null +++ b/backend/tests/test_worker_service_queue_region.py @@ -0,0 +1,68 @@ +"""Cloud Tasks queue paths must use cloud_tasks_region, not gcp_region. + +The GPU audio-separation-job runs with GCP_REGION set to its GPU region, but +every Cloud Tasks queue lives in us-central1 β€” building queue paths from +gcp_region 404s ("Queue does not exist") from that worker, which silently +dropped the auto-approval render trigger (job b8bda9c2, 2026-08-27). +""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + + +@pytest.mark.asyncio +async def test_enqueue_uses_cloud_tasks_region_not_gcp_region(monkeypatch) -> None: + monkeypatch.setenv("ENABLE_CLOUD_TASKS", "true") + monkeypatch.setenv("GCP_REGION", "us-east4") # simulate the GPU job env + monkeypatch.delenv("CLOUD_TASKS_REGION", raising=False) + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + + from backend.config import Settings + from backend.services.worker_service import WorkerService + + with patch("backend.services.worker_service.get_settings", return_value=Settings()): + service = WorkerService() + tasks_client = MagicMock() + tasks_client.queue_path.side_effect = ( + lambda project, location, queue: f"projects/{project}/locations/{location}/queues/{queue}" + ) + service._tasks_client = tasks_client + + await service._enqueue_cloud_task("render-video", "job1") + + args = tasks_client.queue_path.call_args.args + assert args[1] == "us-central1", f"queue location must be us-central1, got {args[1]}" + + +def test_settings_cloud_tasks_region_default(monkeypatch) -> None: + monkeypatch.delenv("CLOUD_TASKS_REGION", raising=False) + from backend.config import Settings + assert Settings().cloud_tasks_region == "us-central1" + + +# --- REVIEW_COMPLETE stall detection (lost render trigger) --- + +def _job(minutes_old: float, render_progress: bool = False) -> SimpleNamespace: + return SimpleNamespace( + state_data={"render_progress": {"stage": "encoding"}} if render_progress else {}, + updated_at=datetime.now(timezone.utc) - timedelta(minutes=minutes_old), + ) + + +def test_review_complete_stalled_after_ten_minutes() -> None: + from backend.api.routes.internal import _review_complete_stalled + assert _review_complete_stalled(_job(minutes_old=15)) is True + + +def test_review_complete_fresh_is_not_stalled() -> None: + from backend.api.routes.internal import _review_complete_stalled + assert _review_complete_stalled(_job(minutes_old=2)) is False + + +def test_review_complete_with_render_underway_is_not_stalled() -> None: + from backend.api.routes.internal import _review_complete_stalled + assert _review_complete_stalled(_job(minutes_old=30, render_progress=True)) is False diff --git a/pyproject.toml b/pyproject.toml index a5246e98..01164a5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.202.0" +version = "0.202.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/review_capture.py b/scripts/review_capture.py index 7b83a573..47375865 100644 --- a/scripts/review_capture.py +++ b/scripts/review_capture.py @@ -157,11 +157,11 @@ def render_markdown(g: Dict[str, Any], captured_at: str) -> str: A("") # --- What actually changed --- - A("## What was changed in review (raw transcription β†’ final)") + A("## Raw transcription β†’ shipped final (includes auto-applied AI corrections β€” NOT purely human edits)") A("") if not diff.has_changes: - A("**No lyric edits** β€” the reviewer accepted the transcription as-is. " - "(Strong signal this class may be auto-approvable.)") + A("**No changes rawβ†’final** β€” neither the auto-applied AI nor the reviewer " + "changed anything. (Strong signal this class may be auto-approvable.)") else: A(f"**{diff.total_changes} changes**: " f"{len(diff.replacements)} word swaps, {len(diff.text_edits)} in-place text, "