fix: worker Cloud Run Jobs never dispatch video/render into the API process (v0.278.12) (#1192)

## Problem: a regression from #1191, caught by #alerts triage
At 07:03 UTC on 2026-10-19, two new freeze alerts fired: **52s with 29
requests affected** (culprit
`firestore_service.append_log_to_subcollection`, during CDG packaging)
and **81s with 33 requests** (culprit `dropbox_service.upload_file`).
The stacks show `video_worker.generate_video` running **inside
karaoke-backend**.

Cause: in today's daily E2E run (job 5a5e5181), #1191's park/resume
worked as designed (parked 06:55:04, resumed 06:57:50). But the resume
runs in `audio-separation-job`, which doesn't set
`USE_CLOUD_RUN_JOBS_FOR_VIDEO`. So `trigger_video_worker` fell back to
Cloud Tasks, which runs it as a `BackgroundTask` on the API, putting the
whole video worker on the API's event loop. The same latent bug already
applied to the audio worker's auto-approval → render trigger.

## Fix
- `backend/config.py`: inside a Cloud Run Job (`CLOUD_RUN_JOB` is set by
the platform), `use_cloud_run_jobs_for_video` and
`use_cloud_run_jobs_for_render` default to **true**. An explicit env
value still wins. This covers every worker job (audio, lyrics, download,
...) with no infra change, through the normal image deploy.
- Error monitor normalizer: `requests_affected=N` →
`requests_affected=<N>`, so a stall culprit is one alert pattern instead
of one per request count.

## Testing
- New `backend/tests/test_config_cloud_run_job_dispatch.py`: inside a
job → true; outside → unchanged; explicit env wins.
- New normalizer test.
- `backend/tests` + `tests/unit/services/error_monitor`: 6871 passed.
- Local CodeRabbit review: 0 findings.

## Verify after deploy
The next parked-then-resumed job logs `Started Cloud Run Job
video-encoding-job` from audio-separation-job, and there are no
`EVENT_LOOP_STALL` lines on karaoke-backend with `video_worker` in the
stack.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_019pmG2XAgtUguEJWLFpkKHN

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 5.5 authored and GitHub committed 2026-10-19 13:17:49 -04:00
1 parent 0d0605d7b5
commit 1344a086da
6 files changed
+85 -8

No files matched your search

+12 -2
View File
@@ -228,7 +228,15 @@ class Settings(BaseSettings):
# When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs
# instead of Cloud Tasks. This supports encoding times >30 minutes (up to 24 hours).
# Default is false - Cloud Tasks is sufficient for most videos (15-20 min).
use_cloud_run_jobs_for_video: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_VIDEO", "false").lower() in ("true", "1", "yes")
# Inside a Cloud Run Job (Cloud Run sets CLOUD_RUN_JOB) both flags default to
# true: a worker job that triggers the next stage (audio worker resuming a
# parked encode, auto-approval starting a render) must never fall back to
# Cloud Tasks -> a BackgroundTask on the API service. That runs the whole
# video worker on the API's event loop (2026-10-19: 52s + 81s freezes,
# ~30 requests each). Explicit env values still win.
use_cloud_run_jobs_for_video: bool = os.getenv(
"USE_CLOUD_RUN_JOBS_FOR_VIDEO", "true" if os.getenv("CLOUD_RUN_JOB") else "false"
).lower() in ("true", "1", "yes")
# When enabled AND enable_cloud_tasks is true, the post-review render worker
# also runs as a Cloud Run Job (reusing video-encoding-job with an args
@@ -237,7 +245,9 @@ class Settings(BaseSettings):
# 10s after SIGTERM, far less than a render takes, so the job froze at
# rendering_video until the 45-min stuck-job sweep (incident 2026-09-13,
# job 41e06b90; same class as incident 2026-03-08 for the video worker).
use_cloud_run_jobs_for_render: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_RENDER", "false").lower() in ("true", "1", "yes")
use_cloud_run_jobs_for_render: bool = os.getenv(
"USE_CLOUD_RUN_JOBS_FOR_RENDER", "true" if os.getenv("CLOUD_RUN_JOB") else "false"
).lower() in ("true", "1", "yes")
# GCE Encoding Worker (for high-performance video encoding)
# When enabled, video encoding is offloaded to a dedicated C4 GCE instance
+7 -1
View File
@@ -102,7 +102,13 @@ _NORMALIZERS: list[tuple[re.Pattern, str]] = [
re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"),
"<ID>",
),
# 12. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
# 12. Request counts in loop-stall lines (EVENT_LOOP_STALL ... requests_affected=29):
# small numbers that would otherwise make every stall a "new" pattern.
(
re.compile(r"\brequests_affected=\d+"),
"requests_affected=<N>",
),
# 13. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
(
re.compile(r"\b\d{4,}\b"),
"<NUM>",
@@ -7,6 +7,7 @@ Increment attempt counter, create()-only single-use / idempotency markers,
set(merge=True) preserving the precomputed eval, and the FieldFilter queries
behind the signup / show-credit caps. Run with: scripts/run-emulator-tests.sh
"""
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
@@ -156,10 +157,17 @@ class TestShowCredit:
prefix = uuid.uuid4().hex
def attempt(i):
try:
return _grant(svc, ref, email, f"{prefix}-{i}", max_per_24h=3).granted
except (kps.ShowCreditCapReached, kps.TransactionContention):
return False
# Retry on contention like a real caller would. Treating contention
# as "not granted" let all 8 lose on the emulator (granted == 0), a
# flake that blocked unrelated PRs (#1191, #1192 on 2026-10-19).
for _ in range(20):
try:
return _grant(svc, ref, email, f"{prefix}-{i}", max_per_24h=3).granted
except kps.ShowCreditCapReached:
return False
except kps.TransactionContention:
time.sleep(0.05)
return False
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(attempt, range(8)))
@@ -0,0 +1,43 @@
"""Worker Cloud Run Jobs must dispatch follow-on video/render work as Cloud Run
Jobs, never via Cloud Tasks -> a BackgroundTask on the API service (which runs
the video worker on the API's event loop: 52s + 81s freezes, 2026-10-19)."""
import importlib
import pytest
def _settings(monkeypatch, **env):
for key in ("CLOUD_RUN_JOB", "USE_CLOUD_RUN_JOBS_FOR_VIDEO", "USE_CLOUD_RUN_JOBS_FOR_RENDER"):
monkeypatch.delenv(key, raising=False)
for key, value in env.items():
monkeypatch.setenv(key, value)
import backend.config as config
return importlib.reload(config).Settings()
@pytest.fixture(autouse=True)
def _restore_config(monkeypatch):
yield
monkeypatch.undo()
import backend.config as config
importlib.reload(config)
def test_inside_cloud_run_job_defaults_to_cloud_run_jobs(monkeypatch):
s = _settings(monkeypatch, CLOUD_RUN_JOB="audio-separation-job")
assert s.use_cloud_run_jobs_for_video is True
assert s.use_cloud_run_jobs_for_render is True
def test_outside_cloud_run_job_default_unchanged(monkeypatch):
s = _settings(monkeypatch)
assert s.use_cloud_run_jobs_for_video is False
assert s.use_cloud_run_jobs_for_render is False
def test_explicit_env_still_wins_inside_a_job(monkeypatch):
s = _settings(monkeypatch, CLOUD_RUN_JOB="x", USE_CLOUD_RUN_JOBS_FOR_VIDEO="false",
USE_CLOUD_RUN_JOBS_FOR_RENDER="false")
assert s.use_cloud_run_jobs_for_video is False
assert s.use_cloud_run_jobs_for_render is False
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.278.11"
version = "0.278.12"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
@@ -442,3 +442,13 @@ class TestComputePatternHash:
result = compute_pattern_hash("", "")
assert isinstance(result, str)
assert len(result) == 64
def test_loop_stall_request_counts_collapse_to_one_pattern():
"""Each distinct requests_affected count used to create a new alert pattern."""
from backend.services.error_monitor.normalizer import normalize_message
a = normalize_message("EVENT_LOOP_STALL ended duration_ms=52484 culprit=x.py:539 in f requests_affected=29")
b = normalize_message("EVENT_LOOP_STALL ended duration_ms=81183 culprit=x.py:539 in f requests_affected=3")
assert a == b
assert "requests_affected=<N>" in a