From 9014754bda7bf340feba8b51030c42a197c1e59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Fri, 23 Oct 2026 16:37:49 -0400 Subject: [PATCH] feat(error-monitor): watch voice pool jobs (v0.283.2) (#1219) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The Phase 3d voice pool runs for ~1 week unattended. Failures must reach Discord #alerts (actionable-signal rule), and the error monitor only watches Cloud Run jobs listed in `MONITORED_CLOUD_RUN_JOBS`. ## Changes Added to `MONITORED_CLOUD_RUN_JOBS` (`backend/services/error_monitor/config.py`): - `voice-pool-select`, `voice-pool-fetch`, `voice-pool-analyse` (karaoke-decide, us-central1) - `vocal-range-gen` (karaoke-decide, us-central1; logs one ERROR from task 0 only when every attempted job fails) - `voice-pool-separate-job` (gen GPU job, us-east4) The monitor's log query filters on `resource.type="cloud_run_job"` + `resource.labels.job_name` only, with no region/location filter, so us-east4 is covered the same way as `audio-separation-job`. No docs list the monitored jobs, so none needed updating. ## Testing - Updated the pinned count/contents test (`tests/unit/services/error_monitor/test_config.py`) - Added a test that the built log filter includes the new jobs and has no location scope - error-monitor tests: 414 passed - Local CodeRabbit review: no findings - Version 0.283.1 -> 0.283.2 @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX Co-authored-by: Claude Opus 5.5 --- backend/services/error_monitor/config.py | 9 +++++++++ backend/tests/test_error_monitor_decide_resources.py | 11 +++++++++++ pyproject.toml | 2 +- tests/unit/services/error_monitor/test_config.py | 10 +++++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/services/error_monitor/config.py b/backend/services/error_monitor/config.py index 18d9aafb..eb54566f 100644 --- a/backend/services/error_monitor/config.py +++ b/backend/services/error_monitor/config.py @@ -61,6 +61,15 @@ MONITORED_CLOUD_RUN_JOBS: list[str] = [ "lb-refresh", # karaoke-decide song search index build (eval-gated) "song-index-build", + # karaoke-decide Phase 3d voice pool (us-central1): select -> fetch -> analyse + "voice-pool-select", + "voice-pool-fetch", + "voice-pool-analyse", + # karaoke-decide vocal-range generation (us-central1); logs one ERROR from task 0 + # only when every attempted job fails + "vocal-range-gen", + # gen GPU separation job for the voice pool (us-east4; no region filter in the log query) + "voice-pool-separate-job", ] # Gen2 Cloud Functions log as cloud_run_revision, not cloud_function. diff --git a/backend/tests/test_error_monitor_decide_resources.py b/backend/tests/test_error_monitor_decide_resources.py index 330fdec2..37b87c01 100644 --- a/backend/tests/test_error_monitor_decide_resources.py +++ b/backend/tests/test_error_monitor_decide_resources.py @@ -10,3 +10,14 @@ def test_decide_search_resources_monitored(): def test_monitor_log_query_covers_them(): assert "karaoke-decide-search" in monitor.RESOURCE_TYPE_FILTERS["cloud_run_revision"][1] assert "song-index-build" in monitor.RESOURCE_TYPE_FILTERS["cloud_run_job"][1] + + +def test_voice_pool_jobs_monitored_across_regions(): + """voice-pool-* jobs (decide us-central1, gen GPU us-east4) must reach #alerts.""" + jobs = monitor.RESOURCE_TYPE_FILTERS["cloud_run_job"][1] + for job in ("voice-pool-select", "voice-pool-fetch", "voice-pool-analyse", "voice-pool-separate-job", "vocal-range-gen"): + assert job in jobs + # The query filters by job_name only (no region/location), so us-east4 is covered. + flt = monitor._build_log_filter("cloud_run_job", jobs, 60) + assert "voice-pool-separate-job" in flt + assert "location" not in flt diff --git a/pyproject.toml b/pyproject.toml index d786a268..63e408be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.283.1" +version = "0.283.2" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/services/error_monitor/test_config.py b/tests/unit/services/error_monitor/test_config.py index 329d0bc6..21c9fd09 100644 --- a/tests/unit/services/error_monitor/test_config.py +++ b/tests/unit/services/error_monitor/test_config.py @@ -26,7 +26,7 @@ class TestMonitoredServiceLists: def test_cloud_run_jobs_count(self): from backend.services.error_monitor import config - assert len(config.MONITORED_CLOUD_RUN_JOBS) == 7 + assert len(config.MONITORED_CLOUD_RUN_JOBS) == 12 def test_cloud_run_jobs_contains_expected(self): from backend.services.error_monitor import config @@ -37,6 +37,14 @@ class TestMonitoredServiceLists: assert "audio-download-job" in config.MONITORED_CLOUD_RUN_JOBS assert "mb-refresh" in config.MONITORED_CLOUD_RUN_JOBS assert "lb-refresh" in config.MONITORED_CLOUD_RUN_JOBS + for job in ( + "voice-pool-select", + "voice-pool-fetch", + "voice-pool-analyse", + "voice-pool-separate-job", + "vocal-range-gen", + ): + assert job in config.MONITORED_CLOUD_RUN_JOBS def test_gen2_cloud_functions_monitored_as_cloud_run_services(self): # Gen2 Cloud Functions log under resource.type=cloud_run_revision,