mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: worker Cloud Run Jobs never dispatch video/render into the API process (v0.278.12) (#1192)
## Problem: a regression from #1191, caught by #alerts triage At 07:03 UTC on 2026-10-19, two new freeze alerts fired: **52s with 29 requests affected** (culprit `firestore_service.append_log_to_subcollection`, during CDG packaging) and **81s with 33 requests** (culprit `dropbox_service.upload_file`). The stacks show `video_worker.generate_video` running **inside karaoke-backend**. Cause: in today's daily E2E run (job 5a5e5181), #1191's park/resume worked as designed (parked 06:55:04, resumed 06:57:50). But the resume runs in `audio-separation-job`, which doesn't set `USE_CLOUD_RUN_JOBS_FOR_VIDEO`. So `trigger_video_worker` fell back to Cloud Tasks, which runs it as a `BackgroundTask` on the API, putting the whole video worker on the API's event loop. The same latent bug already applied to the audio worker's auto-approval → render trigger. ## Fix - `backend/config.py`: inside a Cloud Run Job (`CLOUD_RUN_JOB` is set by the platform), `use_cloud_run_jobs_for_video` and `use_cloud_run_jobs_for_render` default to **true**. An explicit env value still wins. This covers every worker job (audio, lyrics, download, ...) with no infra change, through the normal image deploy. - Error monitor normalizer: `requests_affected=N` → `requests_affected=<N>`, so a stall culprit is one alert pattern instead of one per request count. ## Testing - New `backend/tests/test_config_cloud_run_job_dispatch.py`: inside a job → true; outside → unchanged; explicit env wins. - New normalizer test. - `backend/tests` + `tests/unit/services/error_monitor`: 6871 passed. - Local CodeRabbit review: 0 findings. ## Verify after deploy The next parked-then-resumed job logs `Started Cloud Run Job video-encoding-job` from audio-separation-job, and there are no `EVENT_LOOP_STALL` lines on karaoke-backend with `video_worker` in the stack. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_019pmG2XAgtUguEJWLFpkKHN --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0d0605d7b5
commit
1344a086da
6 files changed
+85
-8
No files matched your search
+12
-2
@@ -228,7 +228,15 @@ class Settings(BaseSettings):
|
||||
# When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs
|
||||
# instead of Cloud Tasks. This supports encoding times >30 minutes (up to 24 hours).
|
||||
# Default is false - Cloud Tasks is sufficient for most videos (15-20 min).
|
||||
use_cloud_run_jobs_for_video: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_VIDEO", "false").lower() in ("true", "1", "yes")
|
||||
# Inside a Cloud Run Job (Cloud Run sets CLOUD_RUN_JOB) both flags default to
|
||||
# true: a worker job that triggers the next stage (audio worker resuming a
|
||||
# parked encode, auto-approval starting a render) must never fall back to
|
||||
# Cloud Tasks -> a BackgroundTask on the API service. That runs the whole
|
||||
# video worker on the API's event loop (2026-10-19: 52s + 81s freezes,
|
||||
# ~30 requests each). Explicit env values still win.
|
||||
use_cloud_run_jobs_for_video: bool = os.getenv(
|
||||
"USE_CLOUD_RUN_JOBS_FOR_VIDEO", "true" if os.getenv("CLOUD_RUN_JOB") else "false"
|
||||
).lower() in ("true", "1", "yes")
|
||||
|
||||
# When enabled AND enable_cloud_tasks is true, the post-review render worker
|
||||
# also runs as a Cloud Run Job (reusing video-encoding-job with an args
|
||||
@@ -237,7 +245,9 @@ class Settings(BaseSettings):
|
||||
# 10s after SIGTERM, far less than a render takes, so the job froze at
|
||||
# rendering_video until the 45-min stuck-job sweep (incident 2026-09-13,
|
||||
# job 41e06b90; same class as incident 2026-03-08 for the video worker).
|
||||
use_cloud_run_jobs_for_render: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_RENDER", "false").lower() in ("true", "1", "yes")
|
||||
use_cloud_run_jobs_for_render: bool = os.getenv(
|
||||
"USE_CLOUD_RUN_JOBS_FOR_RENDER", "true" if os.getenv("CLOUD_RUN_JOB") else "false"
|
||||
).lower() in ("true", "1", "yes")
|
||||
|
||||
# GCE Encoding Worker (for high-performance video encoding)
|
||||
# When enabled, video encoding is offloaded to a dedicated C4 GCE instance
|
||||
|
||||
@@ -102,7 +102,13 @@ _NORMALIZERS: list[tuple[re.Pattern, str]] = [
|
||||
re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"),
|
||||
"<ID>",
|
||||
),
|
||||
# 12. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
|
||||
# 12. Request counts in loop-stall lines (EVENT_LOOP_STALL ... requests_affected=29):
|
||||
# small numbers that would otherwise make every stall a "new" pattern.
|
||||
(
|
||||
re.compile(r"\brequests_affected=\d+"),
|
||||
"requests_affected=<N>",
|
||||
),
|
||||
# 13. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements
|
||||
(
|
||||
re.compile(r"\b\d{4,}\b"),
|
||||
"<NUM>",
|
||||
|
||||
@@ -7,6 +7,7 @@ Increment attempt counter, create()-only single-use / idempotency markers,
|
||||
set(merge=True) preserving the precomputed eval, and the FieldFilter queries
|
||||
behind the signup / show-credit caps. Run with: scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -156,10 +157,17 @@ class TestShowCredit:
|
||||
prefix = uuid.uuid4().hex
|
||||
|
||||
def attempt(i):
|
||||
try:
|
||||
return _grant(svc, ref, email, f"{prefix}-{i}", max_per_24h=3).granted
|
||||
except (kps.ShowCreditCapReached, kps.TransactionContention):
|
||||
return False
|
||||
# Retry on contention like a real caller would. Treating contention
|
||||
# as "not granted" let all 8 lose on the emulator (granted == 0), a
|
||||
# flake that blocked unrelated PRs (#1191, #1192 on 2026-10-19).
|
||||
for _ in range(20):
|
||||
try:
|
||||
return _grant(svc, ref, email, f"{prefix}-{i}", max_per_24h=3).granted
|
||||
except kps.ShowCreditCapReached:
|
||||
return False
|
||||
except kps.TransactionContention:
|
||||
time.sleep(0.05)
|
||||
return False
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(attempt, range(8)))
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Worker Cloud Run Jobs must dispatch follow-on video/render work as Cloud Run
|
||||
Jobs, never via Cloud Tasks -> a BackgroundTask on the API service (which runs
|
||||
the video worker on the API's event loop: 52s + 81s freezes, 2026-10-19)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _settings(monkeypatch, **env):
|
||||
for key in ("CLOUD_RUN_JOB", "USE_CLOUD_RUN_JOBS_FOR_VIDEO", "USE_CLOUD_RUN_JOBS_FOR_RENDER"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
for key, value in env.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
import backend.config as config
|
||||
return importlib.reload(config).Settings()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_config(monkeypatch):
|
||||
yield
|
||||
monkeypatch.undo()
|
||||
import backend.config as config
|
||||
importlib.reload(config)
|
||||
|
||||
|
||||
def test_inside_cloud_run_job_defaults_to_cloud_run_jobs(monkeypatch):
|
||||
s = _settings(monkeypatch, CLOUD_RUN_JOB="audio-separation-job")
|
||||
assert s.use_cloud_run_jobs_for_video is True
|
||||
assert s.use_cloud_run_jobs_for_render is True
|
||||
|
||||
|
||||
def test_outside_cloud_run_job_default_unchanged(monkeypatch):
|
||||
s = _settings(monkeypatch)
|
||||
assert s.use_cloud_run_jobs_for_video is False
|
||||
assert s.use_cloud_run_jobs_for_render is False
|
||||
|
||||
|
||||
def test_explicit_env_still_wins_inside_a_job(monkeypatch):
|
||||
s = _settings(monkeypatch, CLOUD_RUN_JOB="x", USE_CLOUD_RUN_JOBS_FOR_VIDEO="false",
|
||||
USE_CLOUD_RUN_JOBS_FOR_RENDER="false")
|
||||
assert s.use_cloud_run_jobs_for_video is False
|
||||
assert s.use_cloud_run_jobs_for_render is False
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.278.11"
|
||||
version = "0.278.12"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -442,3 +442,13 @@ class TestComputePatternHash:
|
||||
result = compute_pattern_hash("", "")
|
||||
assert isinstance(result, str)
|
||||
assert len(result) == 64
|
||||
|
||||
|
||||
def test_loop_stall_request_counts_collapse_to_one_pattern():
|
||||
"""Each distinct requests_affected count used to create a new alert pattern."""
|
||||
from backend.services.error_monitor.normalizer import normalize_message
|
||||
|
||||
a = normalize_message("EVENT_LOOP_STALL ended duration_ms=52484 culprit=x.py:539 in f requests_affected=29")
|
||||
b = normalize_message("EVENT_LOOP_STALL ended duration_ms=81183 culprit=x.py:539 in f requests_affected=3")
|
||||
assert a == b
|
||||
assert "requests_affected=<N>" in a
|
||||
Reference in new issue
Block a user