mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: Cloud Tasks queue region from GPU workers + REVIEW_COMPLETE lost-trigger recovery (v0.202.1) (#948)
## Summary Follow-up to #947, found by live-testing auto-approval in prod (job `b8bda9c2` — the first real fully-auto-approved job). **Bug:** the GPU `audio-separation-job` sets `GCP_REGION` to its GPU region, but every Cloud Tasks queue lives in `us-central1`. Any queue enqueue from inside that worker fails with `404 Queue does not exist` — which silently dropped the auto-approval render trigger (job auto-approved perfectly, then sat at `review_complete` forever). This is also a latent pre-existing bug for any other enqueue attempted from the GPU job. **Fixes:** 1. New `settings.cloud_tasks_region` (default `us-central1`, overridable via `CLOUD_TASKS_REGION`) used for all Cloud Tasks queue paths. `gcp_region` is still used for Cloud Run Job locations (correct). 2. `recover-stuck-jobs` cron gains a sweep for `REVIEW_COMPLETE` jobs stalled >10 min with no `render_progress` — re-triggers the render worker (idempotent via worker-generation fence). This closes the lost-trigger orphan gap for BOTH the human complete-review fire-and-forget task and the auto-approval path, and will auto-rescue `b8bda9c2` on the first tick after deploy. ## Testing - New unit tests: queue path uses `us-central1` even when `GCP_REGION=us-east4`; stall detector (stalled / fresh / render-underway). - Note: `test_internal_api.py` auth tests fail identically on pristine `main` in this local env (green in CI) — pre-existing, unrelated. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f01213f5cb
commit
9b6c4f236e
7 files changed
+185
-14
No files matched your search
@@ -901,10 +901,41 @@ async def recover_stuck_jobs(
|
||||
if _repark_stalled_render(job_manager, job_id):
|
||||
reparked.append(job_id)
|
||||
|
||||
# --- REVIEW_COMPLETE with no render started: lost render trigger ---
|
||||
# Both the human complete-review endpoint (fire-and-forget task) and the
|
||||
# auto-approval executor can lose the render trigger (e.g. Cloud Tasks
|
||||
# enqueue failure). Without this, the job sits at review_complete forever
|
||||
# with no error and no Retry button. Re-trigger is idempotent (worker
|
||||
# generation fence + status validation).
|
||||
# REVIEW_COMPLETE is a transient state (render normally starts in seconds),
|
||||
# so the population is tiny — scan it all (no limit, which could otherwise
|
||||
# return only fresh jobs and starve stalled ones) and cap re-triggers per
|
||||
# tick to bound the blast radius.
|
||||
render_retriggered = []
|
||||
rc_query = jobs_ref.where(
|
||||
filter=FieldFilter("status", "==", JobStatus.REVIEW_COMPLETE.value)
|
||||
).stream()
|
||||
for doc in rc_query:
|
||||
if len(render_retriggered) >= RENDER_RETRIGGERS_PER_TICK:
|
||||
break
|
||||
job_id = (doc.to_dict() or {}).get("job_id", doc.id)
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
continue
|
||||
if not _review_complete_stalled(job):
|
||||
continue
|
||||
logger.warning(f"[job:{job_id}] REVIEW_COMPLETE stalled >10 min with no render — re-triggering")
|
||||
try:
|
||||
if await worker_service.trigger_render_video_worker(job_id):
|
||||
render_retriggered.append(job_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"[job:{job_id}] render re-trigger failed: {e}")
|
||||
|
||||
logger.info(
|
||||
f"RECOVER_STUCK_JOBS complete: recovered={len(recovered)} "
|
||||
f"download_parked={len(download_parked)} download_retried={len(download_retried)} "
|
||||
f"download_timed_out={len(download_timed_out)} reparked={len(reparked)}"
|
||||
f"download_timed_out={len(download_timed_out)} reparked={len(reparked)} "
|
||||
f"render_retriggered={len(render_retriggered)}"
|
||||
)
|
||||
add_span_event("recovery_complete", {
|
||||
"recovered_count": len(recovered),
|
||||
@@ -923,9 +954,38 @@ async def recover_stuck_jobs(
|
||||
"download_timed_out_jobs": download_timed_out,
|
||||
"reparked_render_jobs": reparked,
|
||||
"reparked_render_count": len(reparked),
|
||||
"render_retriggered_jobs": render_retriggered,
|
||||
"render_retriggered_count": len(render_retriggered),
|
||||
}
|
||||
|
||||
|
||||
REVIEW_COMPLETE_STALL_SECONDS = 10 * 60 # render normally starts within seconds
|
||||
RENDER_RETRIGGERS_PER_TICK = 10 # bound the blast radius per 5-min tick
|
||||
|
||||
|
||||
def _review_complete_stalled(job) -> bool:
|
||||
"""True when a REVIEW_COMPLETE job has sat >10 min with no render underway.
|
||||
|
||||
``render_progress`` is cleared on review completion and (re)written by the
|
||||
render worker — its absence plus a stale ``updated_at`` means the trigger
|
||||
was lost.
|
||||
"""
|
||||
if (job.state_data or {}).get("render_progress"):
|
||||
return False
|
||||
updated_at = getattr(job, "updated_at", None)
|
||||
if updated_at is None:
|
||||
return False
|
||||
from datetime import datetime, timezone as _tz
|
||||
now = datetime.now(_tz.utc)
|
||||
if updated_at.tzinfo is None:
|
||||
now = datetime.utcnow()
|
||||
try:
|
||||
age = (now - updated_at).total_seconds()
|
||||
except TypeError:
|
||||
return False
|
||||
return age > REVIEW_COMPLETE_STALL_SECONDS
|
||||
|
||||
|
||||
# Torrent sources whose downloads are worth auto-retrying over a long window —
|
||||
# they fail transiently when a rare release has few/intermittent seeders or the
|
||||
# private tracker is briefly unreachable. Non-torrent sources (YouTube/Spotify/
|
||||
|
||||
@@ -163,6 +163,11 @@ class Settings(BaseSettings):
|
||||
# When disabled (default), workers are triggered via direct HTTP (for development)
|
||||
enable_cloud_tasks: bool = os.getenv("ENABLE_CLOUD_TASKS", "false").lower() in ("true", "1", "yes")
|
||||
gcp_region: str = os.getenv("GCP_REGION", "us-central1")
|
||||
# ALL Cloud Tasks queues live in us-central1, but workers running in other
|
||||
# regions (the GPU audio-separation-job sets GCP_REGION to its GPU region)
|
||||
# must still enqueue to the real queue location — building queue paths from
|
||||
# gcp_region 404s from those workers ("Queue does not exist").
|
||||
cloud_tasks_region: str = os.getenv("CLOUD_TASKS_REGION", "us-central1")
|
||||
|
||||
# Cloud Run Jobs (for long-running video encoding)
|
||||
# When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs
|
||||
|
||||
@@ -249,7 +249,7 @@ class WorkerService:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT not set, cannot enqueue Cloud Task")
|
||||
return False
|
||||
|
||||
location = self.settings.gcp_region # Must match queue location
|
||||
location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region
|
||||
|
||||
# Build queue path
|
||||
parent = self.tasks_client.queue_path(project, location, queue_name)
|
||||
@@ -754,7 +754,7 @@ class WorkerService:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT not set")
|
||||
return False
|
||||
|
||||
location = self.settings.gcp_region
|
||||
location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region
|
||||
|
||||
# Build queue path
|
||||
parent = self.tasks_client.queue_path(project, location, queue_name)
|
||||
@@ -843,7 +843,7 @@ class WorkerService:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT not set")
|
||||
return False
|
||||
|
||||
location = self.settings.gcp_region
|
||||
location = self.settings.cloud_tasks_region # queues all live here, NOT gcp_region
|
||||
|
||||
# Build queue path
|
||||
parent = self.tasks_client.queue_path(project, location, queue_name)
|
||||
|
||||
@@ -55,16 +55,19 @@ def _job(job_id, status, *, minutes_stale=0, source_name=None, download_retry=No
|
||||
)
|
||||
|
||||
|
||||
def _mock_jm(*, downloading=None, pending=None, rendering=None, jobs=None):
|
||||
"""Wire a JobManager mock for the endpoint's 3 status queries.
|
||||
def _mock_jm(*, downloading=None, pending=None, rendering=None, review_complete=None, jobs=None):
|
||||
"""Wire a JobManager mock for the endpoint's 4 status queries.
|
||||
|
||||
downloading/rendering use .where().stream(); pending uses .where().limit().stream().
|
||||
`jobs` maps job_id -> Job returned by get_job().
|
||||
downloading/rendering/review_complete use .where().stream(); pending uses
|
||||
.where().limit().stream(). `jobs` maps job_id -> Job returned by get_job().
|
||||
"""
|
||||
mock_jm = MagicMock()
|
||||
where = mock_jm.firestore.db.collection.return_value.where.return_value
|
||||
# Two non-limited .stream() calls in order: downloading_audio, rendering_video
|
||||
where.stream.side_effect = [iter(downloading or []), iter(rendering or [])]
|
||||
# Non-limited .stream() calls in order: downloading_audio, rendering_video,
|
||||
# review_complete (lost-render-trigger sweep)
|
||||
where.stream.side_effect = [
|
||||
iter(downloading or []), iter(rendering or []), iter(review_complete or []),
|
||||
]
|
||||
# One .limit().stream() call: download_pending_retry
|
||||
where.limit.return_value.stream.return_value = iter(pending or [])
|
||||
mock_jm.get_job.side_effect = lambda jid: (jobs or {}).get(jid)
|
||||
@@ -175,3 +178,38 @@ def test_fresh_render_not_reparked(client):
|
||||
resp, _ = _run(client, mock_jm)
|
||||
assert resp.json()["reparked_render_count"] == 0
|
||||
mock_jm.transition_to_state.assert_not_called()
|
||||
|
||||
|
||||
def test_stalled_review_complete_render_is_retriggered(client):
|
||||
"""REVIEW_COMPLETE >10 min with no render_progress = lost render trigger
|
||||
(e.g. Cloud Tasks enqueue failed from the GPU worker) -> re-trigger."""
|
||||
job = _job("rc1", JobStatus.REVIEW_COMPLETE, minutes_stale=30)
|
||||
mock_jm = _mock_jm(
|
||||
review_complete=[_doc("rc1", JobStatus.REVIEW_COMPLETE)],
|
||||
jobs={"rc1": job},
|
||||
)
|
||||
mock_ws = MagicMock()
|
||||
mock_ws.trigger_audio_download_worker = AsyncMock(return_value=True)
|
||||
mock_ws.trigger_render_video_worker = AsyncMock(return_value=True)
|
||||
with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \
|
||||
patch("backend.services.worker_service.get_worker_service", return_value=mock_ws):
|
||||
resp = client.post("/api/internal/recover-stuck-jobs")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["render_retriggered_jobs"] == ["rc1"]
|
||||
mock_ws.trigger_render_video_worker.assert_awaited_once_with("rc1")
|
||||
|
||||
|
||||
def test_fresh_review_complete_is_left_alone(client):
|
||||
job = _job("rc2", JobStatus.REVIEW_COMPLETE, minutes_stale=1)
|
||||
mock_jm = _mock_jm(
|
||||
review_complete=[_doc("rc2", JobStatus.REVIEW_COMPLETE)],
|
||||
jobs={"rc2": job},
|
||||
)
|
||||
mock_ws = MagicMock()
|
||||
mock_ws.trigger_render_video_worker = AsyncMock(return_value=True)
|
||||
with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \
|
||||
patch("backend.services.worker_service.get_worker_service", return_value=mock_ws):
|
||||
resp = client.post("/api/internal/recover-stuck-jobs")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["render_retriggered_jobs"] == []
|
||||
mock_ws.trigger_render_video_worker.assert_not_awaited()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Cloud Tasks queue paths must use cloud_tasks_region, not gcp_region.
|
||||
|
||||
The GPU audio-separation-job runs with GCP_REGION set to its GPU region, but
|
||||
every Cloud Tasks queue lives in us-central1 — building queue paths from
|
||||
gcp_region 404s ("Queue does not exist") from that worker, which silently
|
||||
dropped the auto-approval render trigger (job b8bda9c2, 2026-08-27).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enqueue_uses_cloud_tasks_region_not_gcp_region(monkeypatch) -> None:
|
||||
monkeypatch.setenv("ENABLE_CLOUD_TASKS", "true")
|
||||
monkeypatch.setenv("GCP_REGION", "us-east4") # simulate the GPU job env
|
||||
monkeypatch.delenv("CLOUD_TASKS_REGION", raising=False)
|
||||
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project")
|
||||
|
||||
from backend.config import Settings
|
||||
from backend.services.worker_service import WorkerService
|
||||
|
||||
with patch("backend.services.worker_service.get_settings", return_value=Settings()):
|
||||
service = WorkerService()
|
||||
tasks_client = MagicMock()
|
||||
tasks_client.queue_path.side_effect = (
|
||||
lambda project, location, queue: f"projects/{project}/locations/{location}/queues/{queue}"
|
||||
)
|
||||
service._tasks_client = tasks_client
|
||||
|
||||
await service._enqueue_cloud_task("render-video", "job1")
|
||||
|
||||
args = tasks_client.queue_path.call_args.args
|
||||
assert args[1] == "us-central1", f"queue location must be us-central1, got {args[1]}"
|
||||
|
||||
|
||||
def test_settings_cloud_tasks_region_default(monkeypatch) -> None:
|
||||
monkeypatch.delenv("CLOUD_TASKS_REGION", raising=False)
|
||||
from backend.config import Settings
|
||||
assert Settings().cloud_tasks_region == "us-central1"
|
||||
|
||||
|
||||
# --- REVIEW_COMPLETE stall detection (lost render trigger) ---
|
||||
|
||||
def _job(minutes_old: float, render_progress: bool = False) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
state_data={"render_progress": {"stage": "encoding"}} if render_progress else {},
|
||||
updated_at=datetime.now(timezone.utc) - timedelta(minutes=minutes_old),
|
||||
)
|
||||
|
||||
|
||||
def test_review_complete_stalled_after_ten_minutes() -> None:
|
||||
from backend.api.routes.internal import _review_complete_stalled
|
||||
assert _review_complete_stalled(_job(minutes_old=15)) is True
|
||||
|
||||
|
||||
def test_review_complete_fresh_is_not_stalled() -> None:
|
||||
from backend.api.routes.internal import _review_complete_stalled
|
||||
assert _review_complete_stalled(_job(minutes_old=2)) is False
|
||||
|
||||
|
||||
def test_review_complete_with_render_underway_is_not_stalled() -> None:
|
||||
from backend.api.routes.internal import _review_complete_stalled
|
||||
assert _review_complete_stalled(_job(minutes_old=30, render_progress=True)) is False
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.202.0"
|
||||
version = "0.202.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -157,11 +157,11 @@ def render_markdown(g: Dict[str, Any], captured_at: str) -> str:
|
||||
A("")
|
||||
|
||||
# --- What actually changed ---
|
||||
A("## What was changed in review (raw transcription → final)")
|
||||
A("## Raw transcription → shipped final (includes auto-applied AI corrections — NOT purely human edits)")
|
||||
A("")
|
||||
if not diff.has_changes:
|
||||
A("**No lyric edits** — the reviewer accepted the transcription as-is. "
|
||||
"(Strong signal this class may be auto-approvable.)")
|
||||
A("**No changes raw→final** — neither the auto-applied AI nor the reviewer "
|
||||
"changed anything. (Strong signal this class may be auto-approvable.)")
|
||||
else:
|
||||
A(f"**{diff.total_changes} changes**: "
|
||||
f"{len(diff.replacements)} word swaps, {len(diff.text_edits)} in-place text, "
|
||||
|
||||
Reference in new issue
Block a user