mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(error-monitor): watch voice pool jobs (v0.283.2) (#1219)
## Why The Phase 3d voice pool runs for ~1 week unattended. Failures must reach Discord #alerts (actionable-signal rule), and the error monitor only watches Cloud Run jobs listed in `MONITORED_CLOUD_RUN_JOBS`. ## Changes Added to `MONITORED_CLOUD_RUN_JOBS` (`backend/services/error_monitor/config.py`): - `voice-pool-select`, `voice-pool-fetch`, `voice-pool-analyse` (karaoke-decide, us-central1) - `vocal-range-gen` (karaoke-decide, us-central1; logs one ERROR from task 0 only when every attempted job fails) - `voice-pool-separate-job` (gen GPU job, us-east4) The monitor's log query filters on `resource.type="cloud_run_job"` + `resource.labels.job_name` only, with no region/location filter, so us-east4 is covered the same way as `audio-separation-job`. No docs list the monitored jobs, so none needed updating. ## Testing - Updated the pinned count/contents test (`tests/unit/services/error_monitor/test_config.py`) - Added a test that the built log filter includes the new jobs and has no location scope - error-monitor tests: 414 passed - Local CodeRabbit review: no findings - Version 0.283.1 -> 0.283.2 @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
eab20db79c
commit
9014754bda
4 files changed
+30
-2
No files matched your search
@@ -61,6 +61,15 @@ MONITORED_CLOUD_RUN_JOBS: list[str] = [
|
||||
"lb-refresh",
|
||||
# karaoke-decide song search index build (eval-gated)
|
||||
"song-index-build",
|
||||
# karaoke-decide Phase 3d voice pool (us-central1): select -> fetch -> analyse
|
||||
"voice-pool-select",
|
||||
"voice-pool-fetch",
|
||||
"voice-pool-analyse",
|
||||
# karaoke-decide vocal-range generation (us-central1); logs one ERROR from task 0
|
||||
# only when every attempted job fails
|
||||
"vocal-range-gen",
|
||||
# gen GPU separation job for the voice pool (us-east4; no region filter in the log query)
|
||||
"voice-pool-separate-job",
|
||||
]
|
||||
|
||||
# Gen2 Cloud Functions log as cloud_run_revision, not cloud_function.
|
||||
|
||||
@@ -10,3 +10,14 @@ def test_decide_search_resources_monitored():
|
||||
def test_monitor_log_query_covers_them():
|
||||
assert "karaoke-decide-search" in monitor.RESOURCE_TYPE_FILTERS["cloud_run_revision"][1]
|
||||
assert "song-index-build" in monitor.RESOURCE_TYPE_FILTERS["cloud_run_job"][1]
|
||||
|
||||
|
||||
def test_voice_pool_jobs_monitored_across_regions():
|
||||
"""voice-pool-* jobs (decide us-central1, gen GPU us-east4) must reach #alerts."""
|
||||
jobs = monitor.RESOURCE_TYPE_FILTERS["cloud_run_job"][1]
|
||||
for job in ("voice-pool-select", "voice-pool-fetch", "voice-pool-analyse", "voice-pool-separate-job", "vocal-range-gen"):
|
||||
assert job in jobs
|
||||
# The query filters by job_name only (no region/location), so us-east4 is covered.
|
||||
flt = monitor._build_log_filter("cloud_run_job", jobs, 60)
|
||||
assert "voice-pool-separate-job" in flt
|
||||
assert "location" not in flt
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.283.1"
|
||||
version = "0.283.2"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -26,7 +26,7 @@ class TestMonitoredServiceLists:
|
||||
def test_cloud_run_jobs_count(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert len(config.MONITORED_CLOUD_RUN_JOBS) == 7
|
||||
assert len(config.MONITORED_CLOUD_RUN_JOBS) == 12
|
||||
|
||||
def test_cloud_run_jobs_contains_expected(self):
|
||||
from backend.services.error_monitor import config
|
||||
@@ -37,6 +37,14 @@ class TestMonitoredServiceLists:
|
||||
assert "audio-download-job" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
assert "mb-refresh" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
assert "lb-refresh" in config.MONITORED_CLOUD_RUN_JOBS
|
||||
for job in (
|
||||
"voice-pool-select",
|
||||
"voice-pool-fetch",
|
||||
"voice-pool-analyse",
|
||||
"voice-pool-separate-job",
|
||||
"vocal-range-gen",
|
||||
):
|
||||
assert job in config.MONITORED_CLOUD_RUN_JOBS
|
||||
|
||||
def test_gen2_cloud_functions_monitored_as_cloud_run_services(self):
|
||||
# Gen2 Cloud Functions log under resource.type=cloud_run_revision,
|
||||
|
||||
Reference in new issue
Block a user