diff --git a/backend/config.py b/backend/config.py index 8a48066c..165b07d0 100644 --- a/backend/config.py +++ b/backend/config.py @@ -228,7 +228,15 @@ class Settings(BaseSettings): # When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs # instead of Cloud Tasks. This supports encoding times >30 minutes (up to 24 hours). # Default is false - Cloud Tasks is sufficient for most videos (15-20 min). - use_cloud_run_jobs_for_video: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_VIDEO", "false").lower() in ("true", "1", "yes") + # Inside a Cloud Run Job (Cloud Run sets CLOUD_RUN_JOB) both flags default to + # true: a worker job that triggers the next stage (audio worker resuming a + # parked encode, auto-approval starting a render) must never fall back to + # Cloud Tasks -> a BackgroundTask on the API service. That runs the whole + # video worker on the API's event loop (2026-10-19: 52s + 81s freezes, + # ~30 requests each). Explicit env values still win. + use_cloud_run_jobs_for_video: bool = os.getenv( + "USE_CLOUD_RUN_JOBS_FOR_VIDEO", "true" if os.getenv("CLOUD_RUN_JOB") else "false" + ).lower() in ("true", "1", "yes") # When enabled AND enable_cloud_tasks is true, the post-review render worker # also runs as a Cloud Run Job (reusing video-encoding-job with an args @@ -237,7 +245,9 @@ class Settings(BaseSettings): # 10s after SIGTERM, far less than a render takes, so the job froze at # rendering_video until the 45-min stuck-job sweep (incident 2026-09-13, # job 41e06b90; same class as incident 2026-03-08 for the video worker). - use_cloud_run_jobs_for_render: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_RENDER", "false").lower() in ("true", "1", "yes") + use_cloud_run_jobs_for_render: bool = os.getenv( + "USE_CLOUD_RUN_JOBS_FOR_RENDER", "true" if os.getenv("CLOUD_RUN_JOB") else "false" + ).lower() in ("true", "1", "yes") # GCE Encoding Worker (for high-performance video encoding) # When enabled, video encoding is offloaded to a dedicated C4 GCE instance diff --git a/backend/services/error_monitor/normalizer.py b/backend/services/error_monitor/normalizer.py index 3038b4ab..d3f26851 100644 --- a/backend/services/error_monitor/normalizer.py +++ b/backend/services/error_monitor/normalizer.py @@ -102,7 +102,13 @@ _NORMALIZERS: list[tuple[re.Pattern, str]] = [ re.compile(r"\b(?=[a-fA-F0-9]*[a-fA-F][a-fA-F0-9]*\d|[a-fA-F0-9]*\d[a-fA-F0-9]*[a-fA-F])[a-fA-F0-9]{8,}\b"), "", ), - # 12. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements + # 12. Request counts in loop-stall lines (EVENT_LOOP_STALL ... requests_affected=29): + # small numbers that would otherwise make every stall a "new" pattern. + ( + re.compile(r"\brequests_affected=\d+"), + "requests_affected=", + ), + # 13. Large numbers (4+ digits) — must come after epoch/IP/UUID replacements ( re.compile(r"\b\d{4,}\b"), "", diff --git a/backend/tests/emulator/test_kjbox_partner_service.py b/backend/tests/emulator/test_kjbox_partner_service.py index b1ea57b9..015ef790 100644 --- a/backend/tests/emulator/test_kjbox_partner_service.py +++ b/backend/tests/emulator/test_kjbox_partner_service.py @@ -7,6 +7,7 @@ Increment attempt counter, create()-only single-use / idempotency markers, set(merge=True) preserving the precomputed eval, and the FieldFilter queries behind the signup / show-credit caps. Run with: scripts/run-emulator-tests.sh """ +import time import uuid from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone @@ -156,10 +157,17 @@ class TestShowCredit: prefix = uuid.uuid4().hex def attempt(i): - try: - return _grant(svc, ref, email, f"{prefix}-{i}", max_per_24h=3).granted - except (kps.ShowCreditCapReached, kps.TransactionContention): - return False + # Retry on contention like a real caller would. Treating contention + # as "not granted" let all 8 lose on the emulator (granted == 0), a + # flake that blocked unrelated PRs (#1191, #1192 on 2026-10-19). + for _ in range(20): + try: + return _grant(svc, ref, email, f"{prefix}-{i}", max_per_24h=3).granted + except kps.ShowCreditCapReached: + return False + except kps.TransactionContention: + time.sleep(0.05) + return False with ThreadPoolExecutor(max_workers=8) as pool: results = list(pool.map(attempt, range(8))) diff --git a/backend/tests/test_config_cloud_run_job_dispatch.py b/backend/tests/test_config_cloud_run_job_dispatch.py new file mode 100644 index 00000000..f9d30c8e --- /dev/null +++ b/backend/tests/test_config_cloud_run_job_dispatch.py @@ -0,0 +1,43 @@ +"""Worker Cloud Run Jobs must dispatch follow-on video/render work as Cloud Run +Jobs, never via Cloud Tasks -> a BackgroundTask on the API service (which runs +the video worker on the API's event loop: 52s + 81s freezes, 2026-10-19).""" + +import importlib + +import pytest + + +def _settings(monkeypatch, **env): + for key in ("CLOUD_RUN_JOB", "USE_CLOUD_RUN_JOBS_FOR_VIDEO", "USE_CLOUD_RUN_JOBS_FOR_RENDER"): + monkeypatch.delenv(key, raising=False) + for key, value in env.items(): + monkeypatch.setenv(key, value) + import backend.config as config + return importlib.reload(config).Settings() + + +@pytest.fixture(autouse=True) +def _restore_config(monkeypatch): + yield + monkeypatch.undo() + import backend.config as config + importlib.reload(config) + + +def test_inside_cloud_run_job_defaults_to_cloud_run_jobs(monkeypatch): + s = _settings(monkeypatch, CLOUD_RUN_JOB="audio-separation-job") + assert s.use_cloud_run_jobs_for_video is True + assert s.use_cloud_run_jobs_for_render is True + + +def test_outside_cloud_run_job_default_unchanged(monkeypatch): + s = _settings(monkeypatch) + assert s.use_cloud_run_jobs_for_video is False + assert s.use_cloud_run_jobs_for_render is False + + +def test_explicit_env_still_wins_inside_a_job(monkeypatch): + s = _settings(monkeypatch, CLOUD_RUN_JOB="x", USE_CLOUD_RUN_JOBS_FOR_VIDEO="false", + USE_CLOUD_RUN_JOBS_FOR_RENDER="false") + assert s.use_cloud_run_jobs_for_video is False + assert s.use_cloud_run_jobs_for_render is False diff --git a/pyproject.toml b/pyproject.toml index a6eca6d6..1c458334 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.278.11" +version = "0.278.12" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/services/error_monitor/test_normalizer.py b/tests/unit/services/error_monitor/test_normalizer.py index bb335a0f..6378fd7d 100644 --- a/tests/unit/services/error_monitor/test_normalizer.py +++ b/tests/unit/services/error_monitor/test_normalizer.py @@ -442,3 +442,13 @@ class TestComputePatternHash: result = compute_pattern_hash("", "") assert isinstance(result, str) assert len(result) == 64 + + +def test_loop_stall_request_counts_collapse_to_one_pattern(): + """Each distinct requests_affected count used to create a new alert pattern.""" + from backend.services.error_monitor.normalizer import normalize_message + + a = normalize_message("EVENT_LOOP_STALL ended duration_ms=52484 culprit=x.py:539 in f requests_affected=29") + b = normalize_message("EVENT_LOOP_STALL ended duration_ms=81183 culprit=x.py:539 in f requests_affected=3") + assert a == b + assert "requests_affected=" in a