fix: Cloud Tasks queue region from GPU workers + REVIEW_COMPLETE lost-trigger recovery (v0.202.1) (#948)

## Summary

Follow-up to #947, found by live-testing auto-approval in prod (job
`b8bda9c2` — the first real fully-auto-approved job).

**Bug:** the GPU `audio-separation-job` sets `GCP_REGION` to its GPU
region, but every Cloud Tasks queue lives in `us-central1`. Any queue
enqueue from inside that worker fails with `404 Queue does not exist` —
which silently dropped the auto-approval render trigger (job
auto-approved perfectly, then sat at `review_complete` forever). This is
also a latent pre-existing bug for any other enqueue attempted from the
GPU job.

**Fixes:**
1. New `settings.cloud_tasks_region` (default `us-central1`, overridable
via `CLOUD_TASKS_REGION`) used for all Cloud Tasks queue paths.
`gcp_region` is still used for Cloud Run Job locations (correct).
2. `recover-stuck-jobs` cron gains a sweep for `REVIEW_COMPLETE` jobs
stalled >10 min with no `render_progress` — re-triggers the render
worker (idempotent via worker-generation fence). This closes the
lost-trigger orphan gap for BOTH the human complete-review
fire-and-forget task and the auto-approval path, and will auto-rescue
`b8bda9c2` on the first tick after deploy.

## Testing
- New unit tests: queue path uses `us-central1` even when
`GCP_REGION=us-east4`; stall detector (stalled / fresh /
render-underway).
- Note: `test_internal_api.py` auth tests fail identically on pristine
`main` in this local env (green in CI) — pre-existing, unrelated.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Fable 5 authored and GitHub committed 2026-08-27 05:07:17 -04:00
1 parent f01213f5cb
commit 9b6c4f236e
7 files changed
+185 -14

No files matched your search

+61 -1
View File
@@ -901,10 +901,41 @@ async def recover_stuck_jobs(
if _repark_stalled_render(job_manager, job_id):
reparked.append(job_id)
# --- REVIEW_COMPLETE with no render started: lost render trigger ---
# Both the human complete-review endpoint (fire-and-forget task) and the
# auto-approval executor can lose the render trigger (e.g. Cloud Tasks
# enqueue failure). Without this, the job sits at review_complete forever
# with no error and no Retry button. Re-trigger is idempotent (worker
# generation fence + status validation).
# REVIEW_COMPLETE is a transient state (render normally starts in seconds),
# so the population is tiny — scan it all (no limit, which could otherwise
# return only fresh jobs and starve stalled ones) and cap re-triggers per
# tick to bound the blast radius.
render_retriggered = []
rc_query = jobs_ref.where(
filter=FieldFilter("status", "==", JobStatus.REVIEW_COMPLETE.value)
).stream()
for doc in rc_query:
if len(render_retriggered) >= RENDER_RETRIGGERS_PER_TICK:
break
job_id = (doc.to_dict() or {}).get("job_id", doc.id)
job = job_manager.get_job(job_id)
if not job:
continue
if not _review_complete_stalled(job):
continue
logger.warning(f"[job:{job_id}] REVIEW_COMPLETE stalled >10 min with no render — re-triggering")
try:
if await worker_service.trigger_render_video_worker(job_id):
render_retriggered.append(job_id)
except Exception as e:
logger.warning(f"[job:{job_id}] render re-trigger failed: {e}")
logger.info(
f"RECOVER_STUCK_JOBS complete: recovered={len(recovered)} "
f"download_parked={len(download_parked)} download_retried={len(download_retried)} "
f"download_timed_out={len(download_timed_out)} reparked={len(reparked)}"
f"download_timed_out={len(download_timed_out)} reparked={len(reparked)} "
f"render_retriggered={len(render_retriggered)}"
)
add_span_event("recovery_complete", {
"recovered_count": len(recovered),
@@ -923,9 +954,38 @@ async def recover_stuck_jobs(
"download_timed_out_jobs": download_timed_out,
"reparked_render_jobs": reparked,
"reparked_render_count": len(reparked),
"render_retriggered_jobs": render_retriggered,
"render_retriggered_count": len(render_retriggered),
}
REVIEW_COMPLETE_STALL_SECONDS = 10 * 60 # render normally starts within seconds
RENDER_RETRIGGERS_PER_TICK = 10 # bound the blast radius per 5-min tick
def _review_complete_stalled(job) -> bool:
"""True when a REVIEW_COMPLETE job has sat >10 min with no render underway.
``render_progress`` is cleared on review completion and (re)written by the
render worker — its absence plus a stale ``updated_at`` means the trigger
was lost.
"""
if (job.state_data or {}).get("render_progress"):
return False
updated_at = getattr(job, "updated_at", None)
if updated_at is None:
return False
from datetime import datetime, timezone as _tz
now = datetime.now(_tz.utc)
if updated_at.tzinfo is None:
now = datetime.utcnow()
try:
age = (now - updated_at).total_seconds()
except TypeError:
return False
return age > REVIEW_COMPLETE_STALL_SECONDS
# Torrent sources whose downloads are worth auto-retrying over a long window —
# they fail transiently when a rare release has few/intermittent seeders or the
# private tracker is briefly unreachable. Non-torrent sources (YouTube/Spotify/
+5
View File
@@ -163,6 +163,11 @@ class Settings(BaseSettings):
# When disabled (default), workers are triggered via direct HTTP (for development)
enable_cloud_tasks: bool = os.getenv("ENABLE_CLOUD_TASKS", "false").lower() in ("true", "1", "yes")
gcp_region: str = os.getenv("GCP_REGION", "us-central1")
# ALL Cloud Tasks queues live in us-central1, but workers running in other
# regions (the GPU audio-separation-job sets GCP_REGION to its GPU region)
# must still enqueue to the real queue location — building queue paths from
# gcp_region 404s from those workers ("Queue does not exist").
cloud_tasks_region: str = os.getenv("CLOUD_TASKS_REGION", "us-central1")
# Cloud Run Jobs (for long-running video encoding)
# When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs
+3 -3
View File
@@ -249,7 +249,7 @@ class WorkerService:
logger.error("GOOGLE_CLOUD_PROJECT not set, cannot enqueue Cloud Task")
return False
location = self.settings.gcp_region # Must match queue location
location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region
# Build queue path
parent = self.tasks_client.queue_path(project, location, queue_name)
@@ -754,7 +754,7 @@ class WorkerService:
logger.error("GOOGLE_CLOUD_PROJECT not set")
return False
location = self.settings.gcp_region
location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region
# Build queue path
parent = self.tasks_client.queue_path(project, location, queue_name)
@@ -843,7 +843,7 @@ class WorkerService:
logger.error("GOOGLE_CLOUD_PROJECT not set")
return False
location = self.settings.gcp_region
location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region
# Build queue path
parent = self.tasks_client.queue_path(project, location, queue_name)
@@ -55,16 +55,19 @@ def _job(job_id, status, *, minutes_stale=0, source_name=None, download_retry=No
)
def _mock_jm(*, downloading=None, pending=None, rendering=None, jobs=None):
"""Wire a JobManager mock for the endpoint's 3 status queries.
def _mock_jm(*, downloading=None, pending=None, rendering=None, review_complete=None, jobs=None):
"""Wire a JobManager mock for the endpoint's 4 status queries.
downloading/rendering use .where().stream(); pending uses .where().limit().stream().
`jobs` maps job_id -> Job returned by get_job().
downloading/rendering/review_complete use .where().stream(); pending uses
.where().limit().stream(). `jobs` maps job_id -> Job returned by get_job().
"""
mock_jm = MagicMock()
where = mock_jm.firestore.db.collection.return_value.where.return_value
# Two non-limited .stream() calls in order: downloading_audio, rendering_video
where.stream.side_effect = [iter(downloading or []), iter(rendering or [])]
# Non-limited .stream() calls in order: downloading_audio, rendering_video,
# review_complete (lost-render-trigger sweep)
where.stream.side_effect = [
iter(downloading or []), iter(rendering or []), iter(review_complete or []),
]
# One .limit().stream() call: download_pending_retry
where.limit.return_value.stream.return_value = iter(pending or [])
mock_jm.get_job.side_effect = lambda jid: (jobs or {}).get(jid)
@@ -175,3 +178,38 @@ def test_fresh_render_not_reparked(client):
resp, _ = _run(client, mock_jm)
assert resp.json()["reparked_render_count"] == 0
mock_jm.transition_to_state.assert_not_called()
def test_stalled_review_complete_render_is_retriggered(client):
"""REVIEW_COMPLETE >10 min with no render_progress = lost render trigger
(e.g. Cloud Tasks enqueue failed from the GPU worker) -> re-trigger."""
job = _job("rc1", JobStatus.REVIEW_COMPLETE, minutes_stale=30)
mock_jm = _mock_jm(
review_complete=[_doc("rc1", JobStatus.REVIEW_COMPLETE)],
jobs={"rc1": job},
)
mock_ws = MagicMock()
mock_ws.trigger_audio_download_worker = AsyncMock(return_value=True)
mock_ws.trigger_render_video_worker = AsyncMock(return_value=True)
with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \
patch("backend.services.worker_service.get_worker_service", return_value=mock_ws):
resp = client.post("/api/internal/recover-stuck-jobs")
assert resp.status_code == 200
assert resp.json()["render_retriggered_jobs"] == ["rc1"]
mock_ws.trigger_render_video_worker.assert_awaited_once_with("rc1")
def test_fresh_review_complete_is_left_alone(client):
job = _job("rc2", JobStatus.REVIEW_COMPLETE, minutes_stale=1)
mock_jm = _mock_jm(
review_complete=[_doc("rc2", JobStatus.REVIEW_COMPLETE)],
jobs={"rc2": job},
)
mock_ws = MagicMock()
mock_ws.trigger_render_video_worker = AsyncMock(return_value=True)
with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \
patch("backend.services.worker_service.get_worker_service", return_value=mock_ws):
resp = client.post("/api/internal/recover-stuck-jobs")
assert resp.status_code == 200
assert resp.json()["render_retriggered_jobs"] == []
mock_ws.trigger_render_video_worker.assert_not_awaited()
@@ -0,0 +1,68 @@
"""Cloud Tasks queue paths must use cloud_tasks_region, not gcp_region.
The GPU audio-separation-job runs with GCP_REGION set to its GPU region, but
every Cloud Tasks queue lives in us-central1 — building queue paths from
gcp_region 404s ("Queue does not exist") from that worker, which silently
dropped the auto-approval render trigger (job b8bda9c2, 2026-08-27).
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
@pytest.mark.asyncio
async def test_enqueue_uses_cloud_tasks_region_not_gcp_region(monkeypatch) -> None:
monkeypatch.setenv("ENABLE_CLOUD_TASKS", "true")
monkeypatch.setenv("GCP_REGION", "us-east4") # simulate the GPU job env
monkeypatch.delenv("CLOUD_TASKS_REGION", raising=False)
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project")
from backend.config import Settings
from backend.services.worker_service import WorkerService
with patch("backend.services.worker_service.get_settings", return_value=Settings()):
service = WorkerService()
tasks_client = MagicMock()
tasks_client.queue_path.side_effect = (
lambda project, location, queue: f"projects/{project}/locations/{location}/queues/{queue}"
)
service._tasks_client = tasks_client
await service._enqueue_cloud_task("render-video", "job1")
args = tasks_client.queue_path.call_args.args
assert args[1] == "us-central1", f"queue location must be us-central1, got {args[1]}"
def test_settings_cloud_tasks_region_default(monkeypatch) -> None:
monkeypatch.delenv("CLOUD_TASKS_REGION", raising=False)
from backend.config import Settings
assert Settings().cloud_tasks_region == "us-central1"
# --- REVIEW_COMPLETE stall detection (lost render trigger) ---
def _job(minutes_old: float, render_progress: bool = False) -> SimpleNamespace:
return SimpleNamespace(
state_data={"render_progress": {"stage": "encoding"}} if render_progress else {},
updated_at=datetime.now(timezone.utc) - timedelta(minutes=minutes_old),
)
def test_review_complete_stalled_after_ten_minutes() -> None:
from backend.api.routes.internal import _review_complete_stalled
assert _review_complete_stalled(_job(minutes_old=15)) is True
def test_review_complete_fresh_is_not_stalled() -> None:
from backend.api.routes.internal import _review_complete_stalled
assert _review_complete_stalled(_job(minutes_old=2)) is False
def test_review_complete_with_render_underway_is_not_stalled() -> None:
from backend.api.routes.internal import _review_complete_stalled
assert _review_complete_stalled(_job(minutes_old=30, render_progress=True)) is False
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.202.0"
version = "0.202.1"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+3 -3
View File
@@ -157,11 +157,11 @@ def render_markdown(g: Dict[str, Any], captured_at: str) -> str:
A("")
# --- What actually changed ---
A("## What was changed in review (raw transcription → final)")
A("## Raw transcription → shipped final (includes auto-applied AI corrections — NOT purely human edits)")
A("")
if not diff.has_changes:
A("**No lyric edits** — the reviewer accepted the transcription as-is. "
"(Strong signal this class may be auto-approvable.)")
A("**No changes raw→final** — neither the auto-applied AI nor the reviewer "
"changed anything. (Strong signal this class may be auto-approvable.)")
else:
A(f"**{diff.total_changes} changes**: "
f"{len(diff.replacements)} word swaps, {len(diff.text_edits)} in-place text, "