diff --git a/backend/services/expo_push_service.py b/backend/services/expo_push_service.py index 05b9a19b..5a18d633 100644 --- a/backend/services/expo_push_service.py +++ b/backend/services/expo_push_service.py @@ -28,6 +28,9 @@ from backend.services.user_service import USERS_COLLECTION, UserService, get_use logger = logging.getLogger(__name__) EXPO_PUSH_URL = "https://exp.host/--/api/v2/push/send" +# Android notification channel the app creates for gen's job pushes (kjbox uses +# "tonight"). Without it Android files them under Expo's default channel. +ANDROID_CHANNEL_ID = "jobs" EXPO_BATCH_SIZE = 100 TOKENS_FIELD = "expo_push_tokens" VALUES_FIELD = "expo_push_token_values" @@ -154,7 +157,8 @@ class ExpoPushService: for i in range(0, len(tokens), EXPO_BATCH_SIZE): batch = tokens[i:i + EXPO_BATCH_SIZE] messages = [ - {"to": tok, "title": title, "body": body, "sound": "default", "data": {"url": url, "kind": kind}} + {"to": tok, "title": title, "body": body, "sound": "default", + "channelId": ANDROID_CHANNEL_ID, "data": {"url": url, "kind": kind}} for tok in batch ] try: diff --git a/backend/services/job_manager.py b/backend/services/job_manager.py index a4478f8c..3bff5d80 100644 --- a/backend/services/job_manager.py +++ b/backend/services/job_manager.py @@ -8,10 +8,12 @@ This module handles the complete job lifecycle including: - Progress tracking and timeline events - Error handling and retries """ +import asyncio import logging +import threading import uuid from datetime import datetime, timezone -from typing import Optional, Dict, Any, List +from typing import Awaitable, Callable, Optional, Dict, Any, List, TypeVar from backend.config import settings from backend.exceptions import InvalidStateTransitionError, InsufficientCreditsError @@ -41,6 +43,140 @@ PREP_PHASE_STATUSES = frozenset({ FAILURE_PUSH_SENT_KEY = 'failure_push_sent' +# ---------------------------------------------------------------- notification delivery +# +# Emails, pushes and idle-reminder scheduling are fire-and-forget: a state change +# never waits for them. In the API server that is a task on the server's loop, +# which lives on. A Cloud Run Job worker, though, runs its whole job inside +# ``asyncio.run(...)``, which cancels every pending task when the job coroutine +# returns, so a push scheduled by its last fail_job / transition never went out +# (silently: CancelledError is not an Exception). Every send is tracked here and +# worker entry points run their job through ``with_notification_drain`` (or await +# ``drain_notifications``) so the sends finish, within a bounded wait, before the +# loop closes. Module-level, not per instance: one worker builds several JobManagers. + +# How long a worker's drain waits for its notification tasks before giving up. +# This bounds the coroutine wait only: a provider call already running in a +# thread (asyncio.to_thread) can't be cancelled, and on Python 3.11 (prod) +# asyncio.run's default-executor shutdown, and interpreter exit, wait for it +# without limit. Exit time is really bounded by each call's own timeout (Web +# Push WEBPUSH_TIMEOUT_S, Expo's httpx timeout, the Firestore/Postmark clients' +# deadlines), so every provider call must keep one. +NOTIFICATION_DRAIN_TIMEOUT_S = 10.0 + +_pending_notification_tasks: "set[asyncio.Task]" = set() +_pending_notification_threads: "set[threading.Thread]" = set() +_pending_threads_lock = threading.Lock() +# Job ids whose failure push is scheduled but not yet sent (see _failure_already_notified). +_failure_push_in_flight: "set[str]" = set() + +T = TypeVar("T") + + +def _schedule_notification( + make_coro: Callable[[], Awaitable[Any]], + label: str, + on_done: Optional[Callable[[], None]] = None, +) -> None: + """Run ``make_coro()`` in the background and track it until it finishes. + + Inside a running loop (API request, worker job) it becomes a task on that loop; + with no loop (sync code, e.g. a worker's ``asyncio.to_thread`` helper) it runs on + its own daemon thread. Never blocks the caller, never raises from the send. + ``on_done`` runs once the send has finished, failed or been cancelled (even a + task cancelled before its first step, whose coroutine body never runs). + """ + async def guarded(): + try: + await make_coro() + except Exception as e: + logger.error(f"Notification send failed ({label}): {e}") + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop is not None: + task = loop.create_task(guarded(), name=f"notification: {label}") + _pending_notification_tasks.add(task) + task.add_done_callback(_pending_notification_tasks.discard) + if on_done is not None: + task.add_done_callback(lambda _task: on_done()) + return + + def run_in_thread(): + try: + asyncio.run(guarded()) + finally: + if on_done is not None: + on_done() + with _pending_threads_lock: + _pending_notification_threads.discard(threading.current_thread()) + + thread = threading.Thread(target=run_in_thread, daemon=True, name=f"notification: {label}") + with _pending_threads_lock: + _pending_notification_threads.add(thread) + thread.start() + + +async def drain_notifications(timeout_s: Optional[float] = None) -> bool: + """Wait for this process's pending notification sends, at most ``timeout_s``. + + Call before a worker's event loop closes. Returns True when everything was sent + (or failed and was logged); on timeout logs a WARNING, cancels what is left and + returns False. Never raises from a send. Cancelling a task doesn't stop a + provider call already running in a thread (see NOTIFICATION_DRAIN_TIMEOUT_S). + """ + if timeout_s is None: + timeout_s = NOTIFICATION_DRAIN_TIMEOUT_S + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_s + me = asyncio.current_task() + while True: + # A send may schedule another, so re-check until nothing is left. + tasks = [t for t in list(_pending_notification_tasks) + if t.get_loop() is loop and not t.done() and t is not me] + with _pending_threads_lock: + threads = [th for th in _pending_notification_threads if th.is_alive()] + if not tasks and not threads: + return True + remaining = deadline - loop.time() + if remaining <= 0: + break + if tasks: + await asyncio.wait(tasks, timeout=remaining) + else: + await asyncio.sleep(min(0.05, remaining)) + + logger.warning( + f"Gave up waiting for {len(tasks)} notification send(s) and {len(threads)} " + f"notification thread(s) after {timeout_s:.0f}s; exiting without them: " + f"{[t.get_name() for t in tasks] + [th.name for th in threads]}" + ) + for t in tasks: + t.cancel() + return False + + +async def with_notification_drain(job: Awaitable[T]) -> T: + """Await a worker's job coroutine, then drain its notifications (also on error). + + Use as ``asyncio.run(with_notification_drain(process_x(job_id)))`` in a Cloud + Run Job entry point. The drain never raises and its wait is capped at + NOTIFICATION_DRAIN_TIMEOUT_S, so it can't change the job's result. A provider + call already running in a thread still finishes before the process exits; + its own timeout bounds that (see NOTIFICATION_DRAIN_TIMEOUT_S). + """ + try: + return await job + finally: + try: + await drain_notifications() + except Exception as e: + logger.warning(f"Notification drain failed: {e}") + + def _cloud_run_retry_pending(state_data: Optional[Dict[str, Any]]) -> bool: """True if the audio-download worker recorded an unexpired Cloud Run auto-retry marker.""" pending = (state_data or {}).get('cloud_run_retry_pending') @@ -685,8 +821,6 @@ class JobManager: job_id: Job ID new_status: New job status """ - import asyncio - try: # Get the job to access user info job = self.get_job(job_id) @@ -723,9 +857,14 @@ class JobManager: if suppress_customer_notifications(job): logger.info(f"Job {job_id}: failure push suppressed (quiet re-render)") return - self._send_push_notification(job, "failed") - # Lets a second worker failing the same run skip a duplicate push. - self.firestore.update_job(job_id, {f'state_data.{FAILURE_PUSH_SENT_KEY}': True}) + # The run's marker (lets a second worker failing the same run skip a + # duplicate push) is written once the push was attempted, not when it + # is scheduled, so a send that never ran can't hide a later failure. + self._send_push_notification( + job, "failed", + after_send=lambda: self.firestore.update_job( + job_id, {f'state_data.{FAILURE_PUSH_SENT_KEY}': True}), + ) except Exception as e: # Never let notification failures affect job processing @@ -736,11 +875,8 @@ class JobManager: Schedule sending a job completion email. For made-for-you orders, also transfers ownership from admin to customer. - Uses asyncio to fire-and-forget the email sending. + Fire-and-forget (see _schedule_notification). """ - import asyncio - import threading - try: from backend.services.job_notification_service import get_job_notification_service @@ -791,19 +927,7 @@ class JobManager: youtube_queued=youtube_queued, ) - # Try to get existing event loop, create new one if none exists - try: - loop = asyncio.get_running_loop() - # If we're in an async context, create a task - loop.create_task(send_email()) - except RuntimeError: - # No event loop - we're likely in a sync context - # Use daemon thread to avoid blocking job completion - def run_in_thread(): - asyncio.run(send_email()) - thread = threading.Thread(target=run_in_thread, daemon=True) - thread.start() - + _schedule_notification(send_email, f"completion email {job.job_id}") logger.info(f"Scheduled completion email for job {job.job_id}") except Exception as e: @@ -818,9 +942,6 @@ class JobManager: - AWAITING_DURATION_CONFIRM: 15 minutes (900s) — user needs to confirm extra cost - All other blocking states: default delay (IDLE_REMINDER_DELAY_SECONDS) """ - import asyncio - import threading - try: # Record when we entered blocking state (for idle detection) blocking_entered_at = datetime.utcnow().isoformat() @@ -859,17 +980,7 @@ class JobManager: else: await worker_service.schedule_idle_reminder(job.job_id) - # Try to get existing event loop, create new one if none exists - try: - loop = asyncio.get_running_loop() - loop.create_task(schedule_reminder()) - except RuntimeError: - # No event loop - we're in a sync context - # Use daemon thread to avoid blocking job processing - def run_in_thread(): - asyncio.run(schedule_reminder()) - thread = threading.Thread(target=run_in_thread, daemon=True) - thread.start() + _schedule_notification(schedule_reminder, f"idle reminder {job.job_id}") logger.info(f"Scheduled idle reminder for job {job.job_id} ({action_type})") @@ -903,7 +1014,9 @@ class JobManager: path.format(id=job.job_id), ) - def _send_push_notification(self, job: Job, action_type: str) -> None: + def _send_push_notification( + self, job: Job, action_type: str, after_send: Optional[Callable[[], None]] = None, + ) -> None: """ Send push notifications (Web Push and mobile Expo push) for job state changes. @@ -913,10 +1026,9 @@ class JobManager: job: Job object action_type: "complete", "lyrics", "instrumental", "audio_edit", "duration_confirm" or "failed" + after_send: Optional blocking callback, run (in a worker thread) once + both channels were attempted. """ - import asyncio - import threading - try: from backend.services.push_notification_service import get_push_notification_service @@ -936,6 +1048,14 @@ class JobManager: } async def send_notification(): + await send_channels() + if after_send is not None: + try: + await asyncio.to_thread(after_send) + except Exception as e: # best-effort bookkeeping + logger.warning(f"Post-push update failed for job {job.job_id}: {e}") + + async def send_channels(): # Each channel is isolated so one failing cannot stop the other. if web_enabled: try: @@ -956,17 +1076,17 @@ class JobManager: except Exception as e: logger.error(f"Expo push failed for job {job.job_id}: {e}") - # Try to get existing event loop, create new one if none exists + on_done = None + if action_type == "failed": + # Until the send finishes (and its marker is written), this process + # treats the run's failure as notified: see _failure_already_notified. + _failure_push_in_flight.add(job.job_id) + on_done = lambda: _failure_push_in_flight.discard(job.job_id) # noqa: E731 try: - loop = asyncio.get_running_loop() - loop.create_task(send_notification()) - except RuntimeError: - # No event loop - we're in a sync context - def run_in_thread(): - asyncio.run(send_notification()) - thread = threading.Thread(target=run_in_thread, daemon=True) - thread.start() - + _schedule_notification(send_notification, f"{action_type} push {job.job_id}", on_done) + except Exception: + _failure_push_in_flight.discard(job.job_id) + raise logger.debug(f"Scheduled push notification for job {job.job_id} ({action_type})") except Exception as e: @@ -1564,8 +1684,12 @@ class JobManager: is FAILED without the marker (e.g. a Cloud Run auto-retry attempt, which runs while the job is still FAILED and whose first failure skipped the push) still pushes. A manual retry moves the job out of FAILED, so a later - failure pushes again. Best-effort: a read error counts as "not notified". + failure pushes again. While this process's failure push for the job is still + being sent (the marker is written only after it), it also counts as notified. + Best-effort: a read error counts as "not notified". """ + if job_id in _failure_push_in_flight: + return True # this process is sending the run's failure push right now try: job = self.get_job(job_id) return bool( diff --git a/backend/services/push_notification_service.py b/backend/services/push_notification_service.py index c239cb05..36341c77 100644 --- a/backend/services/push_notification_service.py +++ b/backend/services/push_notification_service.py @@ -19,6 +19,9 @@ from backend.i18n import t logger = logging.getLogger(__name__) +# Per-request timeout for a Web Push send (seconds). +WEBPUSH_TIMEOUT_S = 10 + class SubscriptionGoneError(Exception): """Raised when a push subscription is no longer valid (410 Gone).""" @@ -201,7 +204,10 @@ class PushNotificationService: vapid_private_key=self.vapid_private_key, vapid_claims={ "sub": self.settings.vapid_subject - } + }, + # Bounded, so a hung push service can't hold a worker + # thread (and a Cloud Run Job's exit) indefinitely. + timeout=WEBPUSH_TIMEOUT_S, ) success_count += 1 logger.debug(f"Push sent to {sub.get('device_name', 'unknown device')}") diff --git a/backend/tests/test_expo_push.py b/backend/tests/test_expo_push.py index b9bca04b..6a5df8e5 100644 --- a/backend/tests/test_expo_push.py +++ b/backend/tests/test_expo_push.py @@ -180,7 +180,7 @@ class TestSend: url, msgs = calls[0] assert url == "https://exp.host/--/api/v2/push/send" assert msgs == [{"to": TOKEN, "title": "T", "body": "B", "sound": "default", - "data": {"url": "/job/1", "kind": "ready"}}] + "channelId": "jobs", "data": {"url": "/job/1", "kind": "ready"}}] def test_device_not_registered_removes_token(self, db, user_service): _seed_tokens(db, A, [TOKEN, TOKEN2]) @@ -203,12 +203,16 @@ class TestSend: _seed_tokens(db, A, tokens) sizes = [] + channels = set() + def post(url, json, **kw): sizes.append(len(json)) + channels.update(m.get("channelId") for m in json) return _Resp([{"status": "ok"}] * len(json)) assert _service(db, user_service, post).send(A, "ready", "T", "B", "/x") == 250 assert sizes == [100, 100, 50] + assert channels == {"jobs"} # every message, every batch, on the app's "jobs" channel def test_http_failure_never_raises(self, db, user_service): _seed_tokens(db, A, [TOKEN]) diff --git a/backend/tests/test_worker_job_push_env.py b/backend/tests/test_worker_job_push_env.py new file mode 100644 index 00000000..05042bc4 --- /dev/null +++ b/backend/tests/test_worker_job_push_env.py @@ -0,0 +1,114 @@ +"""Worker Cloud Run Jobs carry the API service's Web Push config. + +PushNotificationService.is_enabled() needs ENABLE_PUSH_NOTIFICATIONS=true and the +VAPID keys. CI sets them on the API service only (`gcloud run deploy` in ci.yml); +the Pulumi job definitions had none, so every Web Push a worker job triggered +(failure, audio edit, "video ready") was silently skipped. Source-level checks, +like test_worker_service_cpu_jobs_region: infrastructure/ isn't importable here. +""" +import ast +import re +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +CLOUD_RUN = (REPO_ROOT / "infrastructure" / "modules" / "cloud_run.py").read_text() +CI = (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text() + +# Jobs whose workers change job state (and so notify), with the worker each runs. +NOTIFYING_JOBS = { + "create_lyrics_transcription_job": "backend.workers.lyrics_worker", + "create_audio_separation_job": "backend.workers.audio_worker", + "create_audio_download_job": "backend.workers.audio_download_worker", + "create_bulk_search_job": "backend.workers.bulk_search_worker", + # also runs backend.workers.render_video_worker via an args override + "create_video_encoding_job": "backend.workers.video_worker", +} + + +def _function_source(name: str) -> str: + tree = ast.parse(CLOUD_RUN) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == name: + return ast.get_source_segment(CLOUD_RUN, node) + raise AssertionError(f"{name} not found in infrastructure/modules/cloud_run.py") + + +def _push_secrets() -> dict: + tree = ast.parse(CLOUD_RUN) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == "PUSH_NOTIFICATION_SECRETS" for t in node.targets + ): + return ast.literal_eval(node.value) + raise AssertionError("PUSH_NOTIFICATION_SECRETS not found in cloud_run.py") + + +def _api_deploy_step() -> str: + """ci.yml text of the `gcloud run deploy karaoke-backend` command's step.""" + start = CI.find("gcloud run deploy karaoke-backend") + assert start != -1, "could not find `gcloud run deploy karaoke-backend` in ci.yml" + end = CI.find("\n - name:", start) + return CI[start:end if end != -1 else len(CI)] + + +def _ci_flag_list(flag: str) -> set: + m = re.search(rf'{flag} "([^"]+)"', _api_deploy_step()) + assert m, f"could not find {flag} in the karaoke-backend deploy step of ci.yml" + return set(m.group(1).split(",")) + + +@pytest.mark.parametrize("job_fn,worker", sorted(NOTIFYING_JOBS.items())) +def test_notifying_job_gets_push_envs(job_fn, worker): + src = _function_source(job_fn) + assert worker in src, f"{job_fn} no longer runs {worker}" + assert "*_push_notification_envs()" in src, f"{job_fn} lacks the Web Push env (ENABLE_PUSH_NOTIFICATIONS + VAPID keys)" + + +def test_push_envs_enable_push_and_mount_vapid_secrets(): + src = _function_source("_push_notification_envs") + assert re.search(r'name="ENABLE_PUSH_NOTIFICATIONS",\s*value="true"', src) + assert "secret_key_ref" in src and 'version="latest"' in src + # push_notification_service reads these via settings.get_secret, env first. + assert _push_secrets() == {"VAPID_PUBLIC_KEY": "vapid-public-key", "VAPID_PRIVATE_KEY": "vapid-private-key"} + + +def test_job_push_config_matches_the_api_service_deploy(): + assert "ENABLE_PUSH_NOTIFICATIONS=true" in _ci_flag_list("--set-env-vars") + api_secrets = _ci_flag_list("--set-secrets") + for env_name, secret_id in _push_secrets().items(): + assert f"{env_name}={secret_id}:latest" in api_secrets + + +def _job_function_for(job_name: str) -> str: + tree = ast.parse(CLOUD_RUN) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name.startswith("create_"): + src = ast.get_source_segment(CLOUD_RUN, node) + if f'name="{job_name}"' in src: + return src + raise AssertionError(f"no cloud_run.py job function creates {job_name}") + + +CI_IMAGE_PINNED_JOBS = sorted(set(re.findall(r"gcloud run jobs update ([a-z0-9-]+)", CI))) + + +def test_ci_pins_the_expected_jobs(): + assert CI_IMAGE_PINNED_JOBS == [ + "audio-separation-job", "lyrics-transcription-job", "storage-retention-job", + "video-encoding-job", "voice-pool-separate-job", + ] + + +@pytest.mark.parametrize("job_name", CI_IMAGE_PINNED_JOBS) +def test_ci_pinned_job_ignores_image_changes(job_name): + """CI pins these jobs to a SHA image after each deploy; a pulumi up (e.g. a + targeted one for the push env) must not roll them back to `:latest`.""" + assert "opts=_ci_pinned_image_opts()" in _job_function_for(job_name) + + +def test_ci_pinned_image_path_is_the_job_container_image(): + assert re.search( + r'^CI_PINNED_IMAGE_PATH = "template\.template\.containers\[0\]\.image"$', CLOUD_RUN, re.MULTILINE) + assert "ignore_changes=[CI_PINNED_IMAGE_PATH]" in _function_source("_ci_pinned_image_opts") diff --git a/backend/tests/test_worker_notification_delivery.py b/backend/tests/test_worker_notification_delivery.py new file mode 100644 index 00000000..8a867f56 --- /dev/null +++ b/backend/tests/test_worker_notification_delivery.py @@ -0,0 +1,311 @@ +"""Notifications sent from Cloud Run Job workers must be delivered before the worker exits. + +A worker runs its main coroutine with ``asyncio.run(...)``. JobManager used to send +pushes/emails with a bare ``loop.create_task(...)``; the worker then returned, and +``asyncio.run`` cancelled the pending task on exit, so the Web Push + Expo send +(and completion email / idle-reminder scheduling) never happened, silently +(CancelledError is not an Exception). Each worker's ``main()`` now drains the +pending notifications before the loop closes, with a bounded wait. + +Reuses the ``triggers`` fixture from test_expo_push: real JobManager notification +code, real Expo + Web Push services over the in-memory Firestore fake; only Expo +HTTP and pywebpush are faked. +""" +import asyncio +import importlib +import logging +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from backend.models.job import JobStatus +from backend.services import job_manager as jm_module +from backend.services.job_manager import JobManager, drain_notifications +from backend.tests.test_expo_push import A, db, triggers, user_service # noqa: F401 (fixtures) + + +# (module, coroutine main() runs, argv, return value meaning "clean exit") +WORKERS = [ + ("backend.workers.audio_download_worker", "process_audio_download", ["--job-id", "job1"], True), + ("backend.workers.audio_worker", "process_audio_separation", ["--job-id", "job1"], True), + ("backend.workers.lyrics_worker", "process_lyrics_transcription", ["--job-id", "job1"], True), + ("backend.workers.video_worker", "generate_video", ["--job-id", "job1"], True), + ("backend.workers.render_video_worker", "process_render_video", ["--job-id", "job1"], True), + ("backend.workers.bulk_search_worker", "process_batch", ["--batch-id", "b1"], 1), +] + + +def _run_worker_main(module_name, fn_name, argv, result, body): + """Run the worker's real CLI ``main()`` with its job coroutine replaced by + ``body()`` (sync, like the worker's own job_manager calls) + return ``result``.""" + module = importlib.import_module(module_name) + + async def fake_job(_id): + body() + return result # returns at once: nothing else awaits after the notification + + with patch.object(module, fn_name, fake_job), \ + patch.object(sys, "argv", ["worker", *argv]), \ + pytest.raises(SystemExit) as exit_info: + module.main() + assert exit_info.value.code == 0 + + +@pytest.mark.parametrize("module_name,fn_name,argv,result", WORKERS, ids=[w[0].rsplit(".", 1)[1] for w in WORKERS]) +class TestWorkerFailurePushIsDelivered: + def test_fail_job_push_delivered_before_exit(self, triggers, module_name, fn_name, argv, result): + pushes, webpush = triggers( + JobStatus.DOWNLOADING_AUDIO, + action=lambda mgr: _run_worker_main( + module_name, fn_name, argv, result, + lambda: mgr.fail_job("job1", "boom", {"stage": "x"})), + ) + assert [(p["kind"], p["url"]) for p in pushes] == [("problems", "/job/job1")] + webpush.assert_called_once() + + def test_mark_job_failed_push_delivered_before_exit(self, triggers, module_name, fn_name, argv, result): + pushes, webpush = triggers( + JobStatus.RENDERING_VIDEO, + action=lambda mgr: _run_worker_main( + module_name, fn_name, argv, result, + lambda: mgr.mark_job_failed("job1", "boom", {"stage": "x"})), + ) + assert [p["kind"] for p in pushes] == ["problems"] + webpush.assert_called_once() + + +class TestWorkerTransitionPushIsDelivered: + @pytest.mark.parametrize("from_status,to_status,kind,url", [ + (JobStatus.GENERATING_SCREENS, JobStatus.AWAITING_REVIEW, "review", "/review/job1"), + (JobStatus.DOWNLOADING, JobStatus.AWAITING_AUDIO_EDIT, "review", "/job/job1"), + (JobStatus.ENCODING, JobStatus.COMPLETE, "ready", "/job/job1"), + ]) + def test_transition_push_delivered_before_exit(self, triggers, from_status, to_status, kind, url): + pushes, webpush = triggers( + from_status, + action=lambda mgr: _run_worker_main( + "backend.workers.video_worker", "generate_video", ["--job-id", "job1"], True, + lambda: mgr.transition_to_state("job1", new_status=to_status)), + ) + assert [(p["kind"], p["url"]) for p in pushes] == [(kind, url)] + webpush.assert_called_once() + + +def _network_call(done: list): + """Stand-in for a provider call that records only once it COMPLETES, after + yielding to the loop like real I/O. (An AsyncMock counts as awaited as soon + as it starts, and asyncio.run still runs a fresh task's first step before + cancelling it, which would hide the bug.)""" + async def call(*args, **kwargs): + await asyncio.sleep(0.01) + done.append((args, kwargs)) + return call + + +def _manager_with_job(job): + mgr = JobManager.__new__(JobManager) + mgr.get_job = lambda job_id: job + mgr.firestore = MagicMock() + return mgr + + +class TestEmailAndReminderAreDelivered: + """The completion email and idle-reminder scheduling use the same fire-and-forget + path, so they were exposed to the same cancellation.""" + + def test_completion_email_sent_before_worker_exits(self): + from backend.tests.test_expo_push import _job + job = _job(JobStatus.COMPLETE) + mgr = _manager_with_job(job) + notifier = MagicMock() + sent = [] + notifier.send_job_completion_email = _network_call(sent) + with patch("backend.services.job_notification_service.get_job_notification_service", return_value=notifier), \ + patch("backend.services.delivery_guard.completion_email_hold_reason", return_value=None): + _run_worker_main("backend.workers.video_worker", "generate_video", ["--job-id", "job1"], True, + lambda: mgr._schedule_completion_email(job)) + assert len(sent) == 1 + assert sent[0][1]["job_id"] == "job1" + + def test_idle_reminder_scheduled_before_worker_exits(self): + from backend.tests.test_expo_push import _job + job = _job(JobStatus.AWAITING_AUDIO_EDIT) + mgr = _manager_with_job(job) + worker_service = MagicMock() + scheduled = [] + worker_service.schedule_idle_reminder = _network_call(scheduled) + with patch("backend.services.worker_service.get_worker_service", return_value=worker_service): + _run_worker_main("backend.workers.audio_download_worker", "process_audio_download", + ["--job-id", "job1"], True, + lambda: mgr._schedule_idle_reminder(job, JobStatus.AWAITING_AUDIO_EDIT)) + assert scheduled == [(("job1",), {})] + + +class TestFailurePushSentFlag: + def test_flag_written_only_after_the_push_was_attempted(self, triggers): + seen = {} + + def action(mgr): + async def worker(): + mgr.fail_job("job1", "boom", {"stage": "x"}) + seen["at_schedule"] = mgr.get_job("job1").state_data.get("failure_push_sent") + await drain_notifications() + seen["after_send"] = mgr.get_job("job1").state_data.get("failure_push_sent") + asyncio.run(worker()) + + pushes, _ = triggers(JobStatus.DOWNLOADING_AUDIO, action=action) + assert len(pushes) == 1 + assert not seen["at_schedule"] + assert seen["after_send"] is True + + def test_cancelled_send_leaves_flag_unset(self, triggers): + """A run whose push never went out must not stop a later failure's push.""" + def action(mgr): + async def worker(): + mgr.fail_job("job1", "boom", {"stage": "x"}) + # no drain: asyncio.run cancels the pending send on exit + asyncio.run(worker()) + assert not mgr.get_job("job1").state_data.get("failure_push_sent") + + triggers(JobStatus.DOWNLOADING_AUDIO, action=action) + + def test_send_cancelled_before_its_first_step_clears_in_flight(self, triggers): + """A task cancelled before it ever ran (its body never starts, so no finally + inside it runs) must not leave the job marked in flight in this process, + or every later failure push for the job would be skipped.""" + def action(mgr): + async def worker(): + mgr.fail_job("job1", "first", {"stage": "x"}) + assert "job1" in jm_module._failure_push_in_flight + for task in list(jm_module._pending_notification_tasks): + task.cancel() # before the loop gave it a first step + await asyncio.sleep(0) + await asyncio.sleep(0) + assert "job1" not in jm_module._failure_push_in_flight + assert not mgr.get_job("job1").state_data.get("failure_push_sent") + # The next failure of this (still unnotified) run pushes. + mgr.fail_job("job1", "second", {"stage": "x"}) + await drain_notifications() + asyncio.run(worker()) + + pushes, webpush = triggers(JobStatus.DOWNLOADING_AUDIO, action=action) + assert len(pushes) == 1 + webpush.assert_called_once() + assert "job1" not in jm_module._failure_push_in_flight + + def test_thread_path_clears_in_flight(self, triggers): + """No running loop (sync worker helper): the send runs on a thread, which + clears the in-flight mark when it ends.""" + pushes, _ = triggers(JobStatus.DOWNLOADING_AUDIO, + action=lambda mgr: mgr.fail_job("job1", "boom", {"stage": "x"})) + assert len(pushes) == 1 + assert "job1" not in jm_module._failure_push_in_flight + + def test_two_failures_in_one_worker_push_once_even_before_the_send(self, triggers): + def action(mgr): + async def worker(): + mgr.fail_job("job1", "first", {"stage": "x"}) + mgr.mark_job_failed("job1", "second", {"stage": "x"}) # before the first send ran + await drain_notifications() + asyncio.run(worker()) + + pushes, webpush = triggers(JobStatus.DOWNLOADING_AUDIO, action=action) + assert len(pushes) == 1 + webpush.assert_called_once() + + def test_manual_retry_then_second_failure_pushes_again(self, triggers): + def fail_in_worker(mgr, message): + async def worker(): + mgr.fail_job("job1", message, {"stage": "audio_download"}) + await drain_notifications() + asyncio.run(worker()) + + def action(mgr): + fail_in_worker(mgr, "first failure") + assert mgr.get_job("job1").state_data.get("failure_push_sent") is True + # Manual retry moves the job out of FAILED... + assert mgr.transition_to_state("job1", new_status=JobStatus.DOWNLOADING_AUDIO, notify=False) + # ...so the retried run failing again is a new problem worth a push. + fail_in_worker(mgr, "second failure") + + pushes, webpush = triggers(JobStatus.DOWNLOADING_AUDIO, action=action) + assert [p["kind"] for p in pushes] == ["problems", "problems"] + assert webpush.call_count == 2 + + +class TestDrainBounds: + def test_hung_provider_cannot_keep_the_worker_alive(self, caplog): + async def hang(): + await asyncio.sleep(3600) + + async def worker(): + jm_module._schedule_notification(hang, "hung test send") + loop = asyncio.get_running_loop() + t0 = loop.time() + delivered = await drain_notifications(timeout_s=0.2) + return delivered, loop.time() - t0 + + with caplog.at_level(logging.WARNING, logger=jm_module.logger.name): + delivered, waited = asyncio.run(worker()) + assert delivered is False + assert waited < 2 + assert any(r.levelno == logging.WARNING and "notification" in r.getMessage().lower() + for r in caplog.records) + + def test_drain_with_nothing_pending_returns_at_once(self): + assert asyncio.run(drain_notifications(timeout_s=0.01)) is True + + def test_failing_send_does_not_raise_from_drain(self): + async def boom(): + raise RuntimeError("provider down") + + async def worker(): + jm_module._schedule_notification(boom, "failing test send") + return await drain_notifications(timeout_s=1) + + assert asyncio.run(worker()) is True + + def test_worker_result_kept_when_drain_times_out(self): + async def hang(): + await asyncio.sleep(3600) + + async def job(): + jm_module._schedule_notification(hang, "hung test send") + return "result" + + with patch.object(jm_module, "NOTIFICATION_DRAIN_TIMEOUT_S", 0.1): + assert asyncio.run(jm_module.with_notification_drain(job())) == "result" + + def test_drain_runs_when_the_worker_raises(self): + sent = [] + + async def send(): + await asyncio.sleep(0) + sent.append(True) + + async def job(): + jm_module._schedule_notification(send, "test send") + raise ValueError("worker crashed") + + with pytest.raises(ValueError): + asyncio.run(jm_module.with_notification_drain(job())) + assert sent == [True] + + +class TestApiServerStaysFireAndForget: + def test_trigger_returns_before_the_send_runs(self, triggers): + """In the API server the loop lives on: scheduling must not wait for the send.""" + seen = {} + + def action(mgr): + async def handler(): + mgr._trigger_state_notifications("job1", JobStatus.COMPLETE) + from backend.services.fake_workers import outbox + seen["right_after"] = len(outbox.list_pushes(A)) + await drain_notifications() + asyncio.run(handler()) + + pushes, _ = triggers(JobStatus.COMPLETE, action=action) + assert seen["right_after"] == 0 + assert len(pushes) == 1 diff --git a/backend/workers/audio_download_worker.py b/backend/workers/audio_download_worker.py index cfc28db3..52912b6e 100644 --- a/backend/workers/audio_download_worker.py +++ b/backend/workers/audio_download_worker.py @@ -26,7 +26,7 @@ import time from datetime import datetime, timedelta, timezone from backend.models.job import JobStatus -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, with_notification_drain from backend.services.storage_service import StorageService from backend.config import get_settings from backend.services.fake_workers import fake_workers_enabled @@ -711,7 +711,9 @@ def main(): logger.info(f"Starting audio download worker for job {job_id}") try: - success = asyncio.run(process_audio_download(job_id)) + # Drain: push/email sends scheduled by the job's last state change must + # finish before asyncio.run cancels pending tasks on exit. + success = asyncio.run(with_notification_drain(process_audio_download(job_id))) if success: logger.info(f"Audio download completed successfully for job {job_id}") sys.exit(0) diff --git a/backend/workers/audio_worker.py b/backend/workers/audio_worker.py index 3de19c71..582dff49 100644 --- a/backend/workers/audio_worker.py +++ b/backend/workers/audio_worker.py @@ -23,7 +23,7 @@ import tempfile import time from typing import Optional, Dict, Any -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, with_notification_drain from backend.services.input_persistence import ensure_job_inputs_persisted from backend.services.storage_service import StorageService from backend.services.audio_search_service import DownloadError @@ -811,7 +811,9 @@ def main(): # Run the async worker function try: - success = asyncio.run(process_audio_separation(job_id)) + # Drain: push/email sends scheduled by the job's last state change must + # finish before asyncio.run cancels pending tasks on exit. + success = asyncio.run(with_notification_drain(process_audio_separation(job_id))) if success: logger.info(f"Audio separation completed successfully for job {job_id}") sys.exit(0) diff --git a/backend/workers/bulk_search_worker.py b/backend/workers/bulk_search_worker.py index 9b8a059e..2eaeb8a7 100644 --- a/backend/workers/bulk_search_worker.py +++ b/backend/workers/bulk_search_worker.py @@ -27,7 +27,7 @@ from backend.services.audio_search_service import ( NoResultsError, pick_auto_selection, ) -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, with_notification_drain from backend.services.pricing import duration_to_credits, is_blocked from backend.services.user_service import get_user_service from backend.services.worker_service import get_worker_service @@ -404,7 +404,9 @@ def main(): ) logger.info("Starting bulk search worker for batch %s", args.batch_id) try: - count = asyncio.run(process_batch(args.batch_id)) + # Drain: push/email sends scheduled by the job's last state change must + # finish before asyncio.run cancels pending tasks on exit. + count = asyncio.run(with_notification_drain(process_batch(args.batch_id))) logger.info("Bulk search worker finished batch %s (%d jobs)", args.batch_id, count) sys.exit(0) except Exception as e: # noqa: BLE001 diff --git a/backend/workers/lyrics_worker.py b/backend/workers/lyrics_worker.py index b3631b9b..e123e07c 100644 --- a/backend/workers/lyrics_worker.py +++ b/backend/workers/lyrics_worker.py @@ -29,7 +29,7 @@ from typing import Optional, Dict, Any from pathlib import Path from backend.models.job import JobStatus -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, with_notification_drain from backend.services.input_persistence import ensure_job_inputs_persisted from backend.services.storage_service import StorageService from backend.services.audio_search_service import DownloadError @@ -1033,7 +1033,9 @@ def main(): # Run the async worker function try: - success = asyncio.run(process_lyrics_transcription(job_id)) + # Drain: push/email sends scheduled by the job's last state change must + # finish before asyncio.run cancels pending tasks on exit. + success = asyncio.run(with_notification_drain(process_lyrics_transcription(job_id))) if success: logger.info(f"Lyrics transcription completed successfully for job {job_id}") sys.exit(0) diff --git a/backend/workers/render_video_worker.py b/backend/workers/render_video_worker.py index 16e0be14..8c9340c5 100644 --- a/backend/workers/render_video_worker.py +++ b/backend/workers/render_video_worker.py @@ -38,7 +38,7 @@ from dataclasses import dataclass from backend.models.job import JobStatus from backend.exceptions import InvalidStateTransitionError from backend.utils.audio_filenames import local_audio_filename -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, with_notification_drain from backend.services.input_persistence import ensure_job_inputs_persisted from backend.services.input_recovery import recover_missing_inputs from backend.services.storage_service import StorageService @@ -1017,7 +1017,9 @@ def main(): logger.info(f"Starting render video worker CLI for job {job_id}") try: - success = asyncio.run(process_render_video(job_id)) + # Drain: push/email sends scheduled by the job's last state change must + # finish before asyncio.run cancels pending tasks on exit. + success = asyncio.run(with_notification_drain(process_render_video(job_id))) if success: logger.info(f"Render video completed successfully for job {job_id}") else: diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index 3b1dc8cd..adbe9e02 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -36,7 +36,7 @@ from google.cloud.firestore_v1 import DELETE_FIELD from backend.models.job import JobStatus from backend.exceptions import InvalidStateTransitionError -from backend.services.job_manager import JobManager +from backend.services.job_manager import JobManager, with_notification_drain from backend.services.input_persistence import ensure_job_inputs_persisted from backend.services.input_recovery import recover_missing_inputs from backend.services.storage_service import StorageService @@ -2003,7 +2003,9 @@ def main(): # Run the async worker function try: - success = asyncio.run(generate_video(job_id)) + # Drain: push/email sends scheduled by the job's last state change must + # finish before asyncio.run cancels pending tasks on exit. + success = asyncio.run(with_notification_drain(generate_video(job_id))) if success: logger.info(f"Video generation completed successfully for job {job_id}") sys.exit(0) diff --git a/infrastructure/modules/cloud_run.py b/infrastructure/modules/cloud_run.py index 6f804294..9162d6e9 100644 --- a/infrastructure/modules/cloud_run.py +++ b/infrastructure/modules/cloud_run.py @@ -42,6 +42,55 @@ def _job_resource_name(base: str, region: str) -> str: return base if region == REGION else f"{base}-{region}" +# CI owns these jobs' container image: after each deploy it runs +# `gcloud run jobs update --image /...:` (ci.yml, +# "Update Cloud Run Jobs with new image"). The image here (`:latest`) only +# matters on create; ignoring it keeps a `pulumi up` (even a targeted one for an +# env change) from rolling a job back from CI's SHA pin to `:latest`. +CI_PINNED_IMAGE_PATH = "template.template.containers[0].image" + + +def _ci_pinned_image_opts() -> pulumi.ResourceOptions: + """Resource options for a job whose image CI pins (fresh object per resource).""" + return pulumi.ResourceOptions(ignore_changes=[CI_PINNED_IMAGE_PATH]) + + +# Cloud Run Jobs whose workers change job state and so send the user's push +# notifications (fail_job / mark_job_failed, AWAITING_* and COMPLETE transitions) +# get the API service's push config, which CI sets on `gcloud run deploy` +# (ci.yml: ENABLE_PUSH_NOTIFICATIONS=true in --set-env-vars; +# VAPID_PUBLIC_KEY=vapid-public-key:latest, VAPID_PRIVATE_KEY=vapid-private-key:latest +# in --set-secrets). Without it PushNotificationService.is_enabled() is False in +# the job and every worker-sent Web Push is silently skipped. The backend SA's +# project-level secretAccessor covers both secrets. +PUSH_NOTIFICATION_SECRETS = { + "VAPID_PUBLIC_KEY": "vapid-public-key", + "VAPID_PRIVATE_KEY": "vapid-private-key", +} + + +def _push_notification_envs() -> list: + """Env vars a worker job needs to send Web Push like the API service does.""" + return [ + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="ENABLE_PUSH_NOTIFICATIONS", + value="true", + ), + *[ + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name=env_name, + value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs( + secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs( + secret=f"projects/{PROJECT_ID}/secrets/{secret_id}", + version="latest", + ), + ), + ) + for env_name, secret_id in PUSH_NOTIFICATION_SECRETS.items() + ], + ] + + def create_domain_mapping() -> cloudrun.DomainMapping: """ Create the Cloud Run domain mapping for api.nomadkaraoke.com. @@ -217,6 +266,8 @@ def create_lyrics_transcription_job( ), ), ), + # Web Push from this worker (see _push_notification_envs). + *_push_notification_envs(), ], ) ], @@ -225,6 +276,7 @@ def create_lyrics_transcription_job( max_retries=2, ), ), + opts=_ci_pinned_image_opts(), # CI owns the image (see CI_PINNED_IMAGE_PATH) ) return lyrics_transcription_job @@ -312,6 +364,8 @@ def create_audio_separation_job( name="GOOGLE_CLOUD_PROJECT", value=PROJECT_ID, ), + # Web Push from this worker (see _push_notification_envs). + *_push_notification_envs(), ], ) ], @@ -324,6 +378,7 @@ def create_audio_separation_job( max_retries=2, ), ), + opts=_ci_pinned_image_opts(), # CI owns the image (see CI_PINNED_IMAGE_PATH) ) return audio_separation_job @@ -382,6 +437,7 @@ def create_voice_pool_separate_job( max_retries=0, # decide re-runs; finished songs are skipped via existing stems ), ), + opts=_ci_pinned_image_opts(), # CI owns the image (see CI_PINNED_IMAGE_PATH) ) @@ -496,6 +552,8 @@ def create_audio_download_job( ), ), ), + # Web Push from this worker (see _push_notification_envs). + *_push_notification_envs(), ], ) ], @@ -600,6 +658,8 @@ def create_bulk_search_job( name="GOOGLE_CLOUD_PROJECT", value=PROJECT_ID, ), + # Web Push from this worker (see _push_notification_envs). + *_push_notification_envs(), ], ) ], @@ -653,6 +713,7 @@ def create_storage_retention_job( max_retries=0, ), ), + opts=_ci_pinned_image_opts(), # CI owns the image (see CI_PINNED_IMAGE_PATH) ) @@ -841,6 +902,8 @@ def create_video_encoding_job( name="USE_CLOUD_RUN_JOBS_FOR_RENDER", value="true", ), + # Web Push from this worker (see _push_notification_envs). + *_push_notification_envs(), ], ) ], @@ -849,6 +912,7 @@ def create_video_encoding_job( max_retries=2, ), ), + opts=_ci_pinned_image_opts(), # CI owns the image (see CI_PINNED_IMAGE_PATH) ) return video_encoding_job diff --git a/pyproject.toml b/pyproject.toml index 4bb98339..6b50f1a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.284.0" +version = "0.284.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"