diff --git a/backend/api/dependencies.py b/backend/api/dependencies.py index 5725c3be..44a314e8 100644 --- a/backend/api/dependencies.py +++ b/backend/api/dependencies.py @@ -1,6 +1,7 @@ """ FastAPI dependencies for authentication and authorization. """ +import asyncio import logging import secrets from datetime import datetime, timedelta, UTC @@ -89,7 +90,8 @@ async def require_auth( ) # Validate token using the full method - auth_result = auth_service.validate_token_full(token_str) + # Sync Firestore reads: run off the event loop (every authed request passes here). + auth_result = await asyncio.to_thread(auth_service.validate_token_full, token_str) # Get request_id from middleware for correlation request_id = getattr(request.state, "request_id", None) @@ -122,7 +124,7 @@ async def require_auth( if not getattr(request.state, "tenant_config", None): from backend.services.tenant_service import get_tenant_service tenant_service = get_tenant_service() - tenant_config = tenant_service.get_tenant_config(auth_result.tenant_id) + tenant_config = await asyncio.to_thread(tenant_service.get_tenant_config, auth_result.tenant_id) if tenant_config and tenant_config.is_active: request.state.tenant_config = tenant_config @@ -165,7 +167,7 @@ async def require_auth_legacy( detail="Authentication required. Provide token via Authorization header or ?token= parameter" ) - is_valid, user_type, remaining_uses, message = auth_service.validate_token(token_str) + is_valid, user_type, remaining_uses, message = await asyncio.to_thread(auth_service.validate_token, token_str) if not is_valid: raise HTTPException( @@ -284,10 +286,10 @@ def require_review_auth_factory(job_id_param: str = "job_id"): full_token = token if full_token: - auth_result = auth_service.validate_token_full(full_token) + auth_result = await asyncio.to_thread(auth_service.validate_token_full, full_token) if auth_result.is_valid: # For full auth, also verify job ownership - job = job_manager.get_job(job_id) + job = await asyncio.to_thread(job_manager.get_job, job_id) if not job: raise HTTPException(status_code=404, detail=f"Job {job_id} not found") @@ -319,7 +321,7 @@ def require_review_auth_factory(job_id_param: str = "job_id"): # Try review token if review_token: - job = job_manager.get_job(job_id) + job = await asyncio.to_thread(job_manager.get_job, job_id) if not job: raise HTTPException(status_code=404, detail=f"Job {job_id} not found") diff --git a/backend/api/routes/jobs.py b/backend/api/routes/jobs.py index eeba1efc..615a5e26 100644 --- a/backend/api/routes/jobs.py +++ b/backend/api/routes/jobs.py @@ -3093,14 +3093,14 @@ async def change_visibility( """ locale = get_locale_from_request(http_request) job_manager = JobManager() - job = job_manager.get_job(job_id) + job = await asyncio.to_thread(job_manager.get_job, job_id) if not job: raise HTTPException(status_code=404, detail=t(locale, "jobs.notFound")) user_email = auth_result.user_email or "unknown" is_admin = auth_result.is_admin - from backend.services.visibility_change_service import VisibilityChangeService + from backend.services.visibility_change_service import VisibilityChangeInProgress, VisibilityChangeService service = VisibilityChangeService(job_manager) # Validate (skip job-exists check since we already handled it) @@ -3130,6 +3130,8 @@ async def change_visibility( except RerenderError as e: # Old track whose finals were archived and can't be regenerated right now. raise HTTPException(status_code=e.status_code, detail=str(e)) + except VisibilityChangeInProgress as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error changing visibility for job {job_id}: {e}", exc_info=True) raise HTTPException(status_code=500, detail=f"Failed to change visibility: {str(e)}") @@ -3248,7 +3250,7 @@ async def rerender_with_current_theme( if reason: raise HTTPException(status_code=400, detail=reason) - tenant = get_tenant_service().get_tenant_config(job.tenant_id) + tenant = await asyncio.to_thread(get_tenant_service().get_tenant_config, job.tenant_id) if tenant is None or not tenant.is_active: raise HTTPException(status_code=400, detail="This track's portal is no longer active.") # Re-rendering is free work, so (like tenant billing) require the caller to diff --git a/backend/middleware/tenant.py b/backend/middleware/tenant.py index 90a60baa..33ae5e06 100644 --- a/backend/middleware/tenant.py +++ b/backend/middleware/tenant.py @@ -13,6 +13,7 @@ The middleware is non-blocking - if no tenant is detected, the request proceeds as a default Nomad Karaoke request (tenant_id = None). """ +import asyncio import logging import os from typing import Optional @@ -66,7 +67,8 @@ class TenantMiddleware(BaseHTTPMiddleware): tenant_config: Optional[TenantConfig] = None if tenant_id: tenant_service = get_tenant_service() - tenant_config = tenant_service.get_tenant_config(tenant_id) + # get_tenant_config can hit GCS on a cache miss: keep it off the event loop. + tenant_config = await asyncio.to_thread(tenant_service.get_tenant_config, tenant_id) if tenant_config and not tenant_config.is_active: # Tenant exists but is inactive - treat as default diff --git a/backend/services/firestore_service.py b/backend/services/firestore_service.py index bbd0d14b..3eb91a11 100644 --- a/backend/services/firestore_service.py +++ b/backend/services/firestore_service.py @@ -12,6 +12,7 @@ from google.cloud.firestore_v1 import FieldFilter from backend.config import settings from backend.models.job import Job, JobStatus, TimelineEvent from backend.models.worker_log import WorkerLogEntry +from backend.services.log_write_offload import submit_or_run from backend.models.review_session import ReviewSession from backend.models.audio_edit_session import AudioEditSession @@ -152,6 +153,26 @@ class FirestoreService: return _claim(self.db.transaction(), doc_ref) + def try_set_job_state_flag(self, job_id: str, key: str) -> bool: + """Atomically set ``state_data.`` to True if it isn't already truthy. + + The inverse of ``claim_job_state_flag``: returns True for exactly one of + several concurrent callers (a guard flag such as visibility_change_in_progress). + """ + doc_ref = self.db.collection(self.collection).document(job_id) + + @firestore.transactional + def _set(transaction, ref) -> bool: + snapshot = ref.get(transaction=transaction) + if not snapshot.exists: + return False + if ((snapshot.to_dict() or {}).get("state_data") or {}).get(key): + return False + transaction.update(ref, {f"state_data.{key}": True, "updated_at": datetime.utcnow()}) + return True + + return _set(self.db.transaction(), doc_ref) + def update_job_status( self, job_id: str, @@ -534,9 +555,19 @@ class FirestoreService: # Get subcollection reference logs_ref = self.db.collection(self.collection).document(job_id).collection("logs") - # Add document with auto-generated ID or use log_entry.id + # Add document with auto-generated ID or use log_entry.id. + # On the API event loop this hands off to a background thread + # (a sync Firestore write there freezes every request on the instance). doc_ref = logs_ref.document(log_entry.id) - doc_ref.set(log_entry.to_dict()) + data = log_entry.to_dict() + + def _write(): + try: + doc_ref.set(data) + except Exception as e: + logger.debug(f"Error appending log to subcollection for job {job_id}: {e}") + + submit_or_run(_write) # Don't log every append - too spammy except Exception as e: diff --git a/backend/services/log_write_offload.py b/backend/services/log_write_offload.py new file mode 100644 index 00000000..94a4c551 --- /dev/null +++ b/backend/services/log_write_offload.py @@ -0,0 +1,100 @@ +""" +Keep job-log Firestore writes off the API event loop. + +Job logs (jobs/{id}/logs/*) are written synchronously by the JobLogHandler / +JobLogger / job_logging handlers. When that code runs on the API's event loop (the +screens worker and preview encodes run as in-process background tasks), every log +line is a 0.1-5s Firestore round-trip that freezes every request on the instance: +the loop watchdog attributed 45 stalls in a week to append_log_to_subcollection. + +`submit_or_run` hands the write to one background thread when the caller is on a +running event loop in the API process, and runs it inline everywhere else (worker +threads, Cloud Run Jobs, scripts), so ordering-sensitive callers are unaffected. +Logs are read back ordered by their own timestamp, so the hand-off doesn't reorder +them. The queue is bounded: under a Firestore outage we drop job-log lines rather +than grow memory. +""" +import asyncio +import atexit +import logging +import os +import queue +import threading +import time +from typing import Callable + +logger = logging.getLogger(__name__) + +_MAX_PENDING = 5000 +_DROP_WARNING_INTERVAL_S = 60.0 + +_queue: "queue.Queue[Callable[[], None]]" = queue.Queue(maxsize=_MAX_PENDING) +_worker: threading.Thread | None = None +_worker_lock = threading.Lock() +_dropped = 0 +_last_drop_warning = 0.0 + + +def _on_event_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def _should_offload() -> bool: + # A Cloud Run Job runs a single job; blocking its own loop affects nobody else, + # and a process that exits right after its last log line must not lose it. + return _on_event_loop() and not os.environ.get("CLOUD_RUN_JOB") + + +def _run_worker() -> None: + while True: + write = _queue.get() + try: + write() + except Exception as e: # the write itself already swallows; belt and braces + logger.debug(f"Offloaded job-log write failed: {e}") + finally: + _queue.task_done() + + +def _ensure_worker() -> None: + global _worker + if _worker is not None and _worker.is_alive(): + return + with _worker_lock: + if _worker is None or not _worker.is_alive(): + _worker = threading.Thread(target=_run_worker, name="job-log-writer", daemon=True) + _worker.start() + + +def submit_or_run(write: Callable[[], None]) -> None: + """Run `write` on the background writer if called on the API loop, else inline.""" + global _dropped, _last_drop_warning + if not _should_offload(): + write() + return + _ensure_worker() + try: + _queue.put_nowait(write) + except queue.Full: + _dropped += 1 + now = time.monotonic() + if now - _last_drop_warning >= _DROP_WARNING_INTERVAL_S: + _last_drop_warning = now + logger.warning(f"Job-log writer queue full ({_MAX_PENDING}); dropped {_dropped} job-log lines so far") + + +def drain(timeout: float = 10.0) -> bool: + """Wait (up to `timeout` s) for queued writes to finish. True if fully drained.""" + deadline = time.monotonic() + timeout + while _queue.unfinished_tasks: + if _worker is None or not _worker.is_alive() or time.monotonic() >= deadline: + return False + time.sleep(0.01) + return True + + +atexit.register(drain) diff --git a/backend/services/visibility_change_service.py b/backend/services/visibility_change_service.py index e2fded33..ddf47b9f 100644 --- a/backend/services/visibility_change_service.py +++ b/backend/services/visibility_change_service.py @@ -5,6 +5,7 @@ Handles changing a job's visibility between public and private: - Public -> Private: redistribute existing finals to private destination (fast, ~1-2 min) - Private -> Public: reset styles, regenerate screens, re-render, re-encode (slow, ~15-30 min) """ +import asyncio import logging from datetime import datetime, timezone from typing import Optional @@ -18,6 +19,10 @@ from backend.services.storage_service import StorageService logger = logging.getLogger(__name__) +class VisibilityChangeInProgress(Exception): + """Another visibility change for this job claimed it first.""" + + class VisibilityChangeService: """Service for changing job visibility after completion.""" @@ -53,7 +58,9 @@ class VisibilityChangeService: return None - async def change_to_private(self, job_id: str, job, user_email: str, regenerate_if_missing: bool = True) -> dict: + async def change_to_private( + self, job_id: str, job, user_email: str, regenerate_if_missing: bool = True, holds_claim: bool = False, + ) -> dict: """ Change a public job to private (fast path). @@ -67,6 +74,9 @@ class VisibilityChangeService: public may be destroyed until the private archive exists. If redistribution fails, the job is reverted to public and left fully intact (no partial deletion, no leaked NOMADNP code — see redistribute_video's failure path). + + ``holds_claim``: the caller already owns ``visibility_change_in_progress`` + (the regenerate-then-private continuation, whose regenerate set it). """ logger.info(f"[job:{job_id}] Starting visibility change: public -> private (by {user_email})") @@ -107,6 +117,14 @@ class VisibilityChangeService: # destination. Public outputs remain live until the private archive exists. # No "changed to private" timeline event yet — it is only recorded once the # move actually succeeds (this is reverted if redistribution fails). + # Claim atomically: two requests that both read the job before either wrote + # the flag must not both redistribute and delete the public outputs. + if not holds_claim: + claimed = await asyncio.to_thread( + self.job_manager.firestore.try_set_job_state_flag, job_id, "visibility_change_in_progress" + ) + if not claimed: + raise VisibilityChangeInProgress("A visibility change is already in progress for this job") job_ref.update({ "state_data.visibility_change_in_progress": True, "is_private": True, @@ -141,7 +159,8 @@ class VisibilityChangeService: # the public outputs + recycling the public NOMAD number is best-effort # (each failure is logged inside the helper, never raised), so a stray # orphaned public artifact cannot undo the successful private move. - await self._delete_public_outputs(job_id, public_outputs) + # Sync YouTube/Dropbox/GDrive/GCS deletes: keep them off the API event loop. + await asyncio.to_thread(self._delete_public_outputs, job_id, public_outputs) # Record the completed move and clear the guard flag. redistribute_video # already updated state_data with the new private brand_code etc. @@ -171,12 +190,61 @@ class VisibilityChangeService: """ logger.info(f"[job:{job_id}] Starting visibility change: private -> public (by {user_email})") + # Claim the job before any deletes: two concurrent requests that both passed + # validate_change must not both recycle the brand code and re-process. + claimed = await asyncio.to_thread( + self.job_manager.firestore.try_set_job_state_flag, job_id, "visibility_change_in_progress" + ) + if not claimed: + raise VisibilityChangeInProgress("A visibility change is already in progress for this job") + db = self.job_manager.firestore.db job_ref = db.collection("jobs").document(job_id) + try: + # Steps 1-5 are sync YouTube/Dropbox/GDrive/GCS/Firestore I/O: off the event loop. + await asyncio.to_thread(self._reset_job_for_public, job_id, job, user_email, job_ref) + except Exception: + try: + job_ref.update({"state_data.visibility_change_in_progress": DELETE_FIELD}) + except Exception: + # The job now refuses every visibility change until the flag is cleared. + logger.exception(f"[job:{job_id}] Failed to release visibility_change_in_progress after a failed reset") + raise + + # Step 6: Trigger screens worker + from backend.services.worker_service import get_worker_service + worker_service = get_worker_service() + try: + triggered = await worker_service.trigger_screens_worker(job_id) + except Exception as e: + logger.warning(f"[job:{job_id}] Screens worker trigger raised during visibility change: {e}") + triggered = False + + if not triggered: + logger.error(f"[job:{job_id}] Failed to trigger screens worker for visibility change") + # Restore to complete status on failure + job_ref.update({ + "status": "complete", + "is_private": True, + "state_data.visibility_change_in_progress": DELETE_FIELD, + "state_data.regen_restore_status": DELETE_FIELD, + }) + raise RuntimeError("Failed to trigger screens worker") + + logger.info(f"[job:{job_id}] Visibility change to public initiated, screens worker triggered") + return { + "status": "processing", + "message": "Visibility change to public started. The video will be re-rendered " + "with default branding and published publicly. This takes ~15-30 minutes.", + "reprocessing_required": True, + } + + def _reset_job_for_public(self, job_id: str, job, user_email: str, job_ref) -> None: + """Steps 1-5 of change_to_public (all sync I/O; run via asyncio.to_thread).""" storage = StorageService() # Step 1: Delete distributed outputs + GCS finals - await self._delete_distributed_outputs(job_id, job, keep_gcs_finals=False) + self._delete_distributed_outputs(job_id, job, keep_gcs_finals=False) # Step 2: Delete screens and rendered video from GCS for path in [ @@ -252,31 +320,7 @@ class VisibilityChangeService: } job_ref.update(update_payload) - # Step 6: Trigger screens worker - from backend.services.worker_service import get_worker_service - worker_service = get_worker_service() - triggered = await worker_service.trigger_screens_worker(job_id) - - if not triggered: - logger.error(f"[job:{job_id}] Failed to trigger screens worker for visibility change") - # Restore to complete status on failure - job_ref.update({ - "status": "complete", - "is_private": True, - "state_data.visibility_change_in_progress": DELETE_FIELD, - "state_data.regen_restore_status": DELETE_FIELD, - }) - raise RuntimeError("Failed to trigger screens worker") - - logger.info(f"[job:{job_id}] Visibility change to public initiated, screens worker triggered") - return { - "status": "processing", - "message": "Visibility change to public started. The video will be re-rendered " - "with default branding and published publicly. This takes ~15-30 minutes.", - "reprocessing_required": True, - } - - async def _delete_public_outputs(self, job_id: str, public_outputs: dict): + def _delete_public_outputs(self, job_id: str, public_outputs: dict): """Delete the captured PUBLIC outputs and recycle the public NOMAD number. Used by change_to_private AFTER a successful private redistribution. Operates @@ -373,7 +417,7 @@ class VisibilityChangeService: f"public cleanup incomplete (youtube={youtube_ok}, dropbox={dropbox_ok}, gdrive={gdrive_ok})" ) - async def _delete_distributed_outputs(self, job_id: str, job, keep_gcs_finals: bool = False): + def _delete_distributed_outputs(self, job_id: str, job, keep_gcs_finals: bool = False): """Delete distributed outputs (YouTube, Dropbox, GDrive) and recycle brand code.""" import re diff --git a/backend/tests/emulator/test_try_set_job_state_flag.py b/backend/tests/emulator/test_try_set_job_state_flag.py new file mode 100644 index 00000000..52f6e633 --- /dev/null +++ b/backend/tests/emulator/test_try_set_job_state_flag.py @@ -0,0 +1,78 @@ +""" +FirestoreService.try_set_job_state_flag is an atomic test-and-set: of several +concurrent callers exactly one wins. change_to_public relies on it to claim +visibility_change_in_progress before deleting anything. + +Run with: scripts/run-emulator-tests.sh +""" + +import threading +import uuid +from datetime import datetime, UTC + +import pytest + +from backend.tests.emulator.conftest import emulators_running + +pytestmark = pytest.mark.skipif( + not emulators_running(), + reason="GCP emulators not running. Start with: scripts/start-emulators.sh" +) + +if emulators_running(): + from backend.models.job import Job, JobStatus + from backend.services.firestore_service import FirestoreService + + +def _create_job(service, state_data): + job_id = f"tas-{uuid.uuid4().hex[:8]}" + service.create_job(Job( + job_id=job_id, + status=JobStatus.COMPLETE, + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + artist="Test", + title="Test", + state_data=state_data, + )) + return job_id + + +def test_sets_flag_once(): + service = FirestoreService() + job_id = _create_job(service, {}) + assert service.try_set_job_state_flag(job_id, "visibility_change_in_progress") is True + assert service.try_set_job_state_flag(job_id, "visibility_change_in_progress") is False + assert service.get_job(job_id).state_data["visibility_change_in_progress"] is True + + +def test_already_set_flag_is_not_claimed(): + service = FirestoreService() + job_id = _create_job(service, {"visibility_change_in_progress": True}) + assert service.try_set_job_state_flag(job_id, "visibility_change_in_progress") is False + + +def test_missing_job_is_not_claimed(): + assert FirestoreService().try_set_job_state_flag("no-such-job", "visibility_change_in_progress") is False + + +def test_exactly_one_concurrent_caller_wins(): + service = FirestoreService() + job_id = _create_job(service, {}) + barrier = threading.Barrier(4) + results = [] + + def attempt(): + barrier.wait() + try: + results.append(FirestoreService().try_set_job_state_flag(job_id, "visibility_change_in_progress")) + except Exception as e: # contention abort after retries: a loss, never a second winner + results.append(e) + + threads = [threading.Thread(target=attempt) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(results) == 4 + assert [r for r in results if r is True] == [True] diff --git a/backend/tests/test_change_visibility.py b/backend/tests/test_change_visibility.py index 5b70f292..a185975b 100644 --- a/backend/tests/test_change_visibility.py +++ b/backend/tests/test_change_visibility.py @@ -119,7 +119,7 @@ class TestChangeToPrivate: call_order = [] mock_redist = AsyncMock(return_value=True, side_effect=lambda *a, **k: call_order.append("redist") or True) - mock_delete = AsyncMock(side_effect=lambda *a, **k: call_order.append("delete")) + mock_delete = MagicMock(side_effect=lambda *a, **k: call_order.append("delete")) with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_public_outputs", mock_delete), \ patch("backend.workers.video_worker.redistribute_video", mock_redist): @@ -146,7 +146,7 @@ class TestChangeToPrivate: service = VisibilityChangeService(job_manager=mock_job_manager) job = _make_job(is_private=False, state_data={}) - with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_public_outputs", new_callable=AsyncMock), \ + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_public_outputs"), \ patch("backend.workers.video_worker.redistribute_video", new_callable=AsyncMock, return_value=True): await service.change_to_private("test-123", job, "user@example.com") @@ -175,7 +175,7 @@ class TestChangeToPublic: "dropbox_link": "https://dropbox.com/test", }) - with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \ + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs"), \ patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \ patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("jobs/test-123/style/style_params.json", {"bg": "bg.png"}, None)) as mock_theme, \ patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: @@ -205,7 +205,7 @@ class TestChangeToPublic: service = VisibilityChangeService(job_manager=mock_job_manager) job = _make_job(is_private=True, state_data={}) - with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \ + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs"), \ patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \ patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("jobs/test-123/style/style_params.json", {"bg": "bg.png"}, None)), \ patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: @@ -233,7 +233,7 @@ class TestChangeToPublic: service = VisibilityChangeService(job_manager=mock_job_manager) job = _make_job(is_private=True, state_data={}) - with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \ + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs"), \ patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \ patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("jobs/test-123/style/style_params.json", {"bg": "bg.png"}, None)), \ patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: @@ -277,7 +277,7 @@ class TestChangeToPublic: service = VisibilityChangeService(job_manager=mock_job_manager) job = _make_job(is_private=True, state_data={}) - with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \ + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs"), \ patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \ patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("p.json", {}, None)), \ patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: @@ -301,7 +301,7 @@ class TestChangeToPublic: service = VisibilityChangeService(job_manager=mock_job_manager) job = _make_job(is_private=True, state_data={}) - with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \ + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs"), \ patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \ patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("jobs/test-123/style/style_params.json", {}, None)), \ patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: @@ -321,6 +321,96 @@ class TestChangeToPublic: assert payload["is_private"] is True +class TestChangeToPublicClaim: + """change_to_public claims visibility_change_in_progress atomically before deleting anything.""" + + @pytest.mark.asyncio + async def test_lost_claim_deletes_nothing(self): + from backend.services.visibility_change_service import VisibilityChangeInProgress + + mock_job_manager = MagicMock() + mock_job_manager.firestore.try_set_job_state_flag.return_value = False + service = VisibilityChangeService(job_manager=mock_job_manager) + job = _make_job(is_private=True, state_data={}) + + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs") as mock_delete: + with pytest.raises(VisibilityChangeInProgress): + await service.change_to_public("test-123", job, "user@example.com") + mock_delete.assert_not_called() + mock_job_manager.firestore.try_set_job_state_flag.assert_called_once_with("test-123", "visibility_change_in_progress") + + @pytest.mark.asyncio + async def test_failed_reset_releases_the_claim(self): + mock_job_manager = MagicMock() + mock_job_manager.firestore.try_set_job_state_flag.return_value = True + mock_job_ref = MagicMock() + mock_job_manager.firestore.db.collection.return_value.document.return_value = mock_job_ref + service = VisibilityChangeService(job_manager=mock_job_manager) + job = _make_job(is_private=True, state_data={}) + + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", + side_effect=RuntimeError("dropbox down")): + with pytest.raises(RuntimeError, match="dropbox down"): + await service.change_to_public("test-123", job, "user@example.com") + released = mock_job_ref.update.call_args_list[-1].args[0] + assert list(released) == ["state_data.visibility_change_in_progress"] + + + @pytest.mark.asyncio + async def test_trigger_exception_rolls_back_like_a_failed_trigger(self): + mock_job_manager = MagicMock() + mock_job_ref = MagicMock() + mock_job_manager.firestore.db.collection.return_value.document.return_value = mock_job_ref + service = VisibilityChangeService(job_manager=mock_job_manager) + job = _make_job(is_private=True, state_data={}) + + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs"), \ + patch("backend.services.visibility_change_service.StorageService"), \ + patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("p", {}, None)), \ + patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: + mock_worker_svc.return_value.trigger_screens_worker = AsyncMock(side_effect=ConnectionError("tasks down")) + with pytest.raises(RuntimeError, match="Failed to trigger screens worker"): + await service.change_to_public("test-123", job, "user@example.com") + rollback = mock_job_ref.update.call_args_list[-1].args[0] + assert rollback["status"] == "complete" + assert "state_data.visibility_change_in_progress" in rollback + + +class TestChangeToPrivateClaim: + @pytest.mark.asyncio + async def test_lost_claim_touches_nothing(self): + from backend.services.visibility_change_service import VisibilityChangeInProgress + + mock_job_manager = MagicMock() + mock_job_manager.firestore.try_set_job_state_flag.return_value = False + mock_job_ref = MagicMock() + mock_job_manager.firestore.db.collection.return_value.document.return_value = mock_job_ref + service = VisibilityChangeService(job_manager=mock_job_manager) + job = _make_job(is_private=False, state_data={"brand_code": "NOMAD-0001"}) + + redist = AsyncMock(return_value=True) + with patch("backend.workers.video_worker.redistribute_video", redist): + with pytest.raises(VisibilityChangeInProgress): + await service.change_to_private("test-123", job, "user@example.com") + redist.assert_not_awaited() + mock_job_ref.update.assert_not_called() + + @pytest.mark.asyncio + async def test_chained_continuation_keeps_its_existing_claim(self): + mock_job_manager = MagicMock() + mock_job_ref = MagicMock() + mock_job_manager.firestore.db.collection.return_value.document.return_value = mock_job_ref + service = VisibilityChangeService(job_manager=mock_job_manager) + job = _make_job(is_private=False, state_data={"brand_code": "NOMAD-0001", "visibility_change_in_progress": True}) + + with patch("backend.workers.video_worker.redistribute_video", AsyncMock(return_value=True)), \ + patch("backend.services.visibility_change_service.VisibilityChangeService._delete_public_outputs"): + result = await service.change_to_private( + "test-123", job, "user@example.com", regenerate_if_missing=False, holds_claim=True) + assert result["status"] == "success" + mock_job_manager.firestore.try_set_job_state_flag.assert_not_called() + + class TestChangeToPrivateErrors: """Test error handling in public -> private flow.""" @@ -339,7 +429,7 @@ class TestChangeToPrivateErrors: "gdrive_files": {"mp4": "id1"}, }) - mock_delete = AsyncMock() + mock_delete = MagicMock() with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_public_outputs", mock_delete), \ patch("backend.workers.video_worker.redistribute_video", new_callable=AsyncMock, return_value=False): @@ -373,7 +463,7 @@ class TestDeleteDistributedOutputs: mock_brand_svc = MagicMock() mock_bcs.return_value = mock_brand_svc - await service._delete_distributed_outputs("test-123", job, keep_gcs_finals=True) + service._delete_distributed_outputs("test-123", job, keep_gcs_finals=True) mock_brand_svc.recycle_brand_code.assert_called_once_with("NOMAD", 42) @@ -398,7 +488,7 @@ class TestDeleteDistributedOutputs: mock_brand_svc = MagicMock() mock_bcs.return_value = mock_brand_svc - await service._delete_public_outputs("test-123", public_outputs) + service._delete_public_outputs("test-123", public_outputs) # No public YouTube/Dropbox/GDrive present → all deletions "succeed" → # the public number is recycled and the mirror is cleaned. @@ -432,7 +522,7 @@ class TestDeleteDistributedOutputs: mock_brand_svc = MagicMock() mock_bcs.return_value = mock_brand_svc - await service._delete_public_outputs("test-123", public_outputs) + service._delete_public_outputs("test-123", public_outputs) mock_brand_svc.recycle_brand_code.assert_not_called() @@ -459,7 +549,7 @@ class TestDeleteDistributedOutputs: mock_yt.return_value = MagicMock(is_configured=False) mock_dbx.return_value = MagicMock(is_configured=False) - await service._delete_distributed_outputs("test-123", job, keep_gcs_finals=True) + service._delete_distributed_outputs("test-123", job, keep_gcs_finals=True) update_call = mock_job_ref.update.call_args payload = update_call.args[0] @@ -482,7 +572,7 @@ class TestDeleteDistributedOutputs: mock_storage.delete_folder.return_value = 3 mock_storage_cls.return_value = mock_storage - await service._delete_distributed_outputs("test-123", job, keep_gcs_finals=False) + service._delete_distributed_outputs("test-123", job, keep_gcs_finals=False) mock_storage.delete_folder.assert_called_once_with("jobs/test-123/finals/") @@ -499,7 +589,7 @@ class TestDeleteDistributedOutputs: mock_storage = MagicMock() mock_storage_cls.return_value = mock_storage - await service._delete_distributed_outputs("test-123", job, keep_gcs_finals=True) + service._delete_distributed_outputs("test-123", job, keep_gcs_finals=True) mock_storage.delete_folder.assert_not_called() @@ -511,3 +601,31 @@ class TestStateTransitions: from backend.models.job import STATE_TRANSITIONS, JobStatus allowed = STATE_TRANSITIONS[JobStatus.COMPLETE] assert JobStatus.LYRICS_COMPLETE in allowed + + +def test_route_maps_lost_visibility_claim_to_409(): + from fastapi.testclient import TestClient + + from backend.api.dependencies import require_auth + from backend.main import app + from backend.services.auth_service import AuthResult, UserType + from backend.services.visibility_change_service import VisibilityChangeInProgress + + async def override(): + return AuthResult(is_valid=True, user_type=UserType.UNLIMITED, remaining_uses=-1, message="Valid", + user_email="user@example.com", is_admin=False) + + app.dependency_overrides[require_auth] = override + try: + job = _make_job(is_private=True, state_data={}) + job.user_email = "user@example.com" + job_manager = MagicMock() + job_manager.get_job.return_value = job + with patch("backend.api.routes.jobs.JobManager", return_value=job_manager), \ + patch.object(VisibilityChangeService, "change_to_public", + AsyncMock(side_effect=VisibilityChangeInProgress("A visibility change is already in progress for this job"))): + resp = TestClient(app).post(f"/api/jobs/{job.job_id}/change-visibility", json={"target_visibility": "public"}) + finally: + app.dependency_overrides.clear() + assert resp.status_code == 409, resp.text + assert "already in progress" in resp.json()["detail"] diff --git a/backend/tests/test_log_write_offload.py b/backend/tests/test_log_write_offload.py new file mode 100644 index 00000000..a60782b9 --- /dev/null +++ b/backend/tests/test_log_write_offload.py @@ -0,0 +1,109 @@ +"""Job-log Firestore writes must not run on the API event loop (log_write_offload).""" +import asyncio +import queue +import threading +from unittest.mock import MagicMock + +import pytest + +from backend.services import log_write_offload + + +@pytest.fixture(autouse=True) +def _not_in_cloud_run_job(monkeypatch): + monkeypatch.delenv("CLOUD_RUN_JOB", raising=False) + yield + assert log_write_offload.drain(timeout=5) + + +def test_runs_inline_off_the_event_loop(): + ran_on = [] + log_write_offload.submit_or_run(lambda: ran_on.append(threading.get_ident())) + assert ran_on == [threading.get_ident()] + + +def test_offloads_from_a_running_event_loop(): + release = threading.Event() + ran_on = [] + + def slow_write(): + release.wait(5) + ran_on.append(threading.get_ident()) + + async def on_loop(): + log_write_offload.submit_or_run(slow_write) + # submit returned without waiting for the (blocked) write + assert ran_on == [] + return threading.get_ident() + + loop_thread = asyncio.run(on_loop()) + release.set() + assert log_write_offload.drain(timeout=5) + assert len(ran_on) == 1 and ran_on[0] != loop_thread + + +def test_runs_inline_on_the_loop_inside_a_cloud_run_job(monkeypatch): + monkeypatch.setenv("CLOUD_RUN_JOB", "video-encoding-job") + ran_on = [] + + async def on_loop(): + log_write_offload.submit_or_run(lambda: ran_on.append(threading.get_ident())) + return threading.get_ident() + + assert ran_on == [asyncio.run(on_loop())] + + +def test_offloaded_writes_keep_submission_order(): + written = [] + + async def on_loop(): + for i in range(50): + log_write_offload.submit_or_run(lambda i=i: written.append(i)) + + asyncio.run(on_loop()) + assert log_write_offload.drain(timeout=5) + assert written == list(range(50)) + + +def test_full_queue_drops_with_a_warning(monkeypatch, caplog): + release = threading.Event() + monkeypatch.setattr(log_write_offload, "_queue", queue.Queue(maxsize=1)) + monkeypatch.setattr(log_write_offload, "_worker", None) + monkeypatch.setattr(log_write_offload, "_dropped", 0) + monkeypatch.setattr(log_write_offload, "_last_drop_warning", 0.0) + written = [] + + async def on_loop(): + log_write_offload.submit_or_run(lambda: release.wait(5)) # occupies the writer + await asyncio.sleep(0.05) + log_write_offload.submit_or_run(lambda: written.append("queued")) # fills the queue + log_write_offload.submit_or_run(lambda: written.append("dropped")) # no room + + with caplog.at_level("WARNING", logger=log_write_offload.__name__): + asyncio.run(on_loop()) + release.set() + assert log_write_offload.drain(timeout=5) + assert written == ["queued"] + assert log_write_offload._dropped == 1 + assert "dropped 1 job-log lines" in caplog.text + + +def test_firestore_append_log_does_not_write_on_the_loop_thread(): + from backend.models.worker_log import WorkerLogEntry + from backend.services.firestore_service import FirestoreService + + service = FirestoreService.__new__(FirestoreService) + service.collection = "jobs" + service.db = MagicMock() + doc_ref = service.db.collection.return_value.document.return_value.collection.return_value.document.return_value + set_threads = [] + doc_ref.set.side_effect = lambda data: set_threads.append(threading.get_ident()) + + async def on_loop(): + service.append_log_to_subcollection("job1", WorkerLogEntry.create(job_id="job1", worker="screens", level="INFO", message="hi")) + return threading.get_ident() + + loop_thread = asyncio.run(on_loop()) + assert log_write_offload.drain(timeout=5) + assert len(set_threads) == 1 and set_threads[0] != loop_thread + assert doc_ref.set.call_args.args[0]["message"] == "hi" diff --git a/backend/tests/test_no_blocking_async_routes.py b/backend/tests/test_no_blocking_async_routes.py index 2307304c..b4725165 100644 --- a/backend/tests/test_no_blocking_async_routes.py +++ b/backend/tests/test_no_blocking_async_routes.py @@ -209,6 +209,14 @@ BLOCKING_CALLS = { "upload_json", # GCS upload (complete_review: up to 21s) "_prepare_theme_for_job", # GCS reads/writes (job creation) "generate", # CustomLyricsService.generate → sync Gemini call (151s, 32 requests) + # 2026-10-19 watchdog culprits (7 days of EVENT_LOOP_STALL stacks): sync Firestore / + # GCS reads in the auth dependencies, tenant middleware and routes. + "validate_token_full", # require_auth: every authenticated request (up to 1.9s) + "validate_token", + "validate_session", + "get_tenant_config", # GCS read on a cache miss (tenant middleware, up to 1.2s) + "append_log_to_subcollection", # job-log write; on the loop it is offloaded, see log_write_offload + "append_worker_log", } _BLOCKING_MODULE_CALLS = {("requests", "get"), ("requests", "post"), ("requests", "put"), ("requests", "delete"), ("requests", "request"), ("time", "sleep")} @@ -226,13 +234,14 @@ def _blocking_call_name(call: ast.Call) -> str | None: return None -def find_blocking_calls_in_async_routes(source: str) -> list[str]: - """`async def` routes that directly call a known-blocking function (excluding - nested sync helpers, which are presumably run via to_thread).""" +def find_blocking_calls_in_async_routes(source: str, routes_only: bool = True) -> list[str]: + """`async def` routes (or, with routes_only=False, any `async def`) that directly + call a known-blocking function (excluding nested sync helpers, which are presumably + run via to_thread).""" tree = ast.parse(source) hits = [] for fn in ast.walk(tree): - if not (isinstance(fn, ast.AsyncFunctionDef) and _is_route(fn)): + if not (isinstance(fn, ast.AsyncFunctionDef) and (_is_route(fn) or not routes_only)): continue stack = list(fn.body) while stack: @@ -283,3 +292,33 @@ async def helper_ok(): hits = find_blocking_calls_in_async_routes(src) assert sorted(h.split(" line")[0] for h in hits) == [ "bad: check_dropbox_credentials()", "bad: requests.post()"] + + +# Auth dependencies and middleware run on the event loop for (nearly) every request, so +# every `async def` in them is held to the same rule as a route, not just decorated ones. +ALWAYS_ON_LOOP_FILES = [BACKEND / "api" / "dependencies.py"] + sorted((BACKEND / "middleware").glob("*.py")) + + +def test_no_known_blocking_calls_in_async_dependencies_or_middleware(): + offenders = [] + for path in ALWAYS_ON_LOOP_FILES: + for hit in find_blocking_calls_in_async_routes(path.read_text(), routes_only=False): + offenders.append(f"{path.relative_to(BACKEND.parent)}::{hit}") + assert not offenders, ( + "These `async def` dependencies/middleware call known-blocking functions on the " + "event loop, which stalls every request. Use `await asyncio.to_thread(fn, ...)`:\n " + + "\n ".join(sorted(offenders)) + ) + + +def test_blocking_call_detector_scans_non_route_async_functions(): + src = ''' +async def require_auth(request): + return auth_service.validate_token_full(token) + +async def fixed(request): + return await asyncio.to_thread(auth_service.validate_token_full, token) +''' + assert find_blocking_calls_in_async_routes(src) == [] + hits = find_blocking_calls_in_async_routes(src, routes_only=False) + assert [h.split(" line")[0] for h in hits] == ["require_auth: validate_token_full()"] diff --git a/backend/workers/auto_correct_worker.py b/backend/workers/auto_correct_worker.py index 61413b1e..698495e6 100644 --- a/backend/workers/auto_correct_worker.py +++ b/backend/workers/auto_correct_worker.py @@ -229,7 +229,7 @@ async def _run_suggest(job_id: str, data: dict, segments: list, reference_lyrics from backend.services.auto_correct.settings import AutoCorrectSettings from backend.services.job_manager import JobManager - job = JobManager().get_job(job_id) + job = await asyncio.to_thread(JobManager().get_job, job_id) service = get_auto_correct_service() result = await asyncio.wait_for( asyncio.to_thread( diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index 453b4f25..9ac7c084 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -610,7 +610,7 @@ async def _run_chained_change_to_private(job_id: str, job_manager: JobManager, r try: fresh = job_manager.get_job(job_id) await VisibilityChangeService(job_manager).change_to_private( - job_id, fresh, requested_by, regenerate_if_missing=False, + job_id, fresh, requested_by, regenerate_if_missing=False, holds_claim=True, ) job_log.info("Regenerated finals redistributed: visibility changed to private") except Exception as e: # noqa: BLE001 - the regenerate itself succeeded diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index d41c6062..f2af26a6 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -67,6 +67,13 @@ The "servers unavailable" banner (49 episodes in 2 weeks) was mostly the API's o - The 2 user-felt ones were cold-start `importlib` stalls (4-15s) from routes that lazily import modules pulling in `httpx`. Lifespan startup now imports `main._LAZY_ROUTE_IMPORTS` in a worker thread before readiness. A background warmup thread alone would still race the first request. - **An `await` somewhere in the handler doesn't make its other calls safe** (v0.267.1). `mark_uploads_complete`, `create_job_with_upload_urls` and others awaited `request.json()`, so they passed the guard, yet called `check_dropbox_credentials()` inline. When the Dropbox token had expired, that did an HTTP refresh plus a Secret Manager write and version cleanup on the loop, a 16s freeze. `trigger_gdrive_validation_endpoint` froze 5-10s after every job (OIDC fetch + POST). Both are now behind `asyncio.to_thread`, and the guard's `BLOCKING_CALLS` denylist fails CI if a known-blocking function is called directly in an `async def` route. Add to the denylist when the watchdog names a new culprit. - **Mostly-sync routes that await one trigger** (v0.268.1). `complete_review` (21s corrections `upload_json`), `create_job_from_search` (theme prep), and the `recover_stuck_jobs` / `retry_pending_render_jobs` crons (~260 stalls/week from Firestore scans) each awaited a worker trigger, so the guard passed them. Their bodies now run in a worker thread, and the trigger coroutines go back to the API loop with `backend/utils/loop_bridge.run_on_loop` (`run_coroutine_threadsafe`, 120s timeout), so loop APIs inside the triggers still work. Bodies that call `run_on_loop` run on their own pool (`run_in_body_thread`), not the default executor. Otherwise they could hold every default-pool thread while the coroutines they wait on need one, a deadlock. Watch out for an `async def` helper with no awaits inside, such as `_validate_audio_durations` (2 downloads + 2 decodes) or the YouTube queue's `_process_single_upload` (a whole upload): it's sync work on the loop with an async signature. To rank culprits, group `EVENT_LOOP_STALL in progress` stacks by their first `api/routes/` frame. +- **Auth, middleware and job logs were blocking the loop too** (v0.278.14). Over 7 days of stall stacks (2026-10-12→19), the top app-level sources were: + - the in-process screens worker and preview encodes, through their job-log handlers. Every log line was a sync Firestore write (`append_log_to_subcollection`); + - `require_auth`, through `validate_token_full`. That is a sync Firestore session read on **every** authenticated request; + - `require_review_auth` (`get_job`); + - the tenant middleware (`get_tenant_config`, a GCS read on a cache miss). + + Job-log writes made on the API loop now go to one background thread (`backend/services/log_write_offload.py`). They stay inline in worker threads and Cloud Run Jobs, where nothing else waits on the loop. The auth dependencies and tenant middleware use `asyncio.to_thread`. The guard now holds every `async def` in `api/dependencies.py` and `middleware/` to `BLOCKING_CALLS`, not just decorated routes. Still on the loop: about 70 direct `get_job`/`update_job` calls in `async def` routes. Each is a single 50-200 ms Firestore round-trip and rarely shows up in the watchdog. Move them when one does. - **Detection:** `backend/services/loop_watchdog.py` logs `EVENT_LOOP_STALL` with the loop thread's stack and a `culprit=file:line`. Grep for it before guessing. - **Alert on impact, not on gaps** (v0.268.5). After the CPU-throttle discount, the stall alert still flapped. Cloud Run autoscaling starts extra instances that serve one request, then sit idle and throttled while the startup warmup thread, which spends 10+ minutes loading SpaCy, competes for slices. Every 10-40s ERROR "stall" over 4 days (11 of them) was on an instance with **zero** requests, and all had an empty culprit. A freeze advances the 1/5/15s sample schedule without sampling, so whatever happens after the thaw has no stack. The watchdog now takes a `RequestTracker`, fed by the outermost pure-ASGI `RequestTrackingMiddleware`. A stall reaches ERROR only if a request was in flight at its start or arrived during it. A request that queued during the stall is counted if it starts within the 0.5s grace window after the loop resumes. Check whether anyone waited before chasing a culprit: join `labels.instanceId` between the stall lines and the request logs. - **Async services with sync bodies** (v0.268.7). `PushNotificationService.send_push` was an `async def` with no real awaits: a Firestore read, a `webpush` POST per device, and a Firestore update, all on the loop. The route guard can't see it because it's a service method. The watchdog caught it at 1.2s with a request waiting. The body is now `send_push_sync`, and `send_push` calls `await asyncio.to_thread(...)`. When `requests_affected>0` lines name a `backend/services/*` frame, that service method is the fix site. Also since v0.268.7: if a process freeze skips every sample point and the loop is still blocked once the process thaws, the watchdog takes one sample then, so the stall gets a culprit. diff --git a/pyproject.toml b/pyproject.toml index 4e1e0e10..5919beff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.278.13" +version = "0.278.14" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"