diff --git a/backend/api/routes/internal.py b/backend/api/routes/internal.py index 9f490f52..292c8da2 100644 --- a/backend/api/routes/internal.py +++ b/backend/api/routes/internal.py @@ -1319,6 +1319,35 @@ def _recover_stuck_jobs_sync(http_request: Request, loop: asyncio.AbstractEventL except Exception as e: logger.warning(f"[job:{job_id}] render re-trigger failed: {e}") + # --- GENERATING_SCREENS held for the auto-approval re-score, never released --- + # The screens worker holds lyrics-confident jobs out of review until stems + # land (review_gate.py); the audio worker releases them. If that release is + # lost (audio worker crash after mark_audio_complete, lost dispatch), release + # here after the timeout: re-score, then auto-complete or open review. + from backend.services.auto_approval.review_gate import audio_hold_expired, release_audio_hold + from backend.services.job_manager import JobManager as _JobManagerCls + hold_key = _JobManagerCls.AUTO_REVIEW_HOLD_KEY + hold_released = [] + gs_query = jobs_ref.where( + filter=FieldFilter("status", "==", JobStatus.GENERATING_SCREENS.value) + ).stream() + for doc in gs_query: + if len(hold_released) >= RENDER_RETRIGGERS_PER_TICK: + break + data = doc.to_dict() or {} + job_id = data.get("job_id", doc.id) + if not (data.get("state_data") or {}).get(hold_key): + continue + job = job_manager.get_job(job_id) + if not job or not audio_hold_expired(job): + continue + logger.warning(f"[job:{job_id}] auto-review hold expired with no stems release — releasing") + try: + if run_on_loop(loop, release_audio_hold(job_id, reason="hold_timeout")): + hold_released.append(job_id) + except Exception as e: # noqa: BLE001 + logger.warning(f"[job:{job_id}] auto-review hold release failed: {e}") + # --- Prep phase with lyrics done but screens never triggered: lost screens trigger --- # The lyrics worker triggers screen generation once transcription completes # (audio separation is decoupled). If that dispatch is lost — historically a @@ -1476,6 +1505,7 @@ def _recover_stuck_jobs_sync(http_request: Request, loop: asyncio.AbstractEventL "regen_screens_retriggered_jobs": regen_screens_retriggered, "video_retriggered_jobs": video_retriggered, "stems_wait_failed_jobs": stems_wait_failed, + "auto_review_hold_released_jobs": hold_released, } diff --git a/backend/config.py b/backend/config.py index 252c0e33..0ce90153 100644 --- a/backend/config.py +++ b/backend/config.py @@ -177,6 +177,14 @@ class Settings(BaseSettings): "AUTO_APPROVAL_TIMING_GATE_ENABLED", "true" ).lower() in ("true", "1", "yes") + # Review hold: when the lyrics would auto-ship but stems are still + # separating, keep the job out of review until the audio-worker re-score + # (backend/services/auto_approval/review_gate.py). Code-level default for + # the same two-environment reason as above. Kill switch: false. + auto_approval_audio_hold_enabled: bool = os.getenv( + "AUTO_APPROVAL_AUDIO_HOLD_ENABLED", "true" + ).lower() in ("true", "1", "yes") + # Match judge — artist/title formatting + match-acceptance for the job # submission flow. Deterministic + catalog layers always run; the light AI # layer (Vertex Gemini Flash) fires only when those aren't confident. diff --git a/backend/services/auto_approval/executor.py b/backend/services/auto_approval/executor.py index 8384c79c..ad7822f0 100644 --- a/backend/services/auto_approval/executor.py +++ b/backend/services/auto_approval/executor.py @@ -108,7 +108,10 @@ async def maybe_auto_complete_review(job_id: str, trigger: str) -> Dict[str, Any # Only act on the two known launch points; a human already in the # editor (IN_REVIEW) or any other state is out of scope. - if trigger == "screens_worker": + # "audio_hold" = re-score of a job the screens worker held out of + # review while stems separated (review_gate.release_audio_hold); the + # job is still GENERATING_SCREENS, exactly as on the first pass. + if trigger in ("screens_worker", "audio_hold"): expected_statuses = (JobStatus.GENERATING_SCREENS,) else: expected_statuses = (JobStatus.AWAITING_REVIEW,) @@ -245,10 +248,20 @@ async def maybe_auto_complete_review(job_id: str, trigger: str) -> Dict[str, Any logger.info(f"[job:{job_id}] {summary}") if not eligible or blockers: + # Worth holding review for the stems: the text verdict is AUTO and + # the ONLY thing missing is separated audio (backing analysis, + # timing + vocal-gap checks). The re-score then decides for real. + hold_for_audio = ( + lyrics_auto + and blockers == ["audio_incomplete"] + and (custom_instrumental or backing_preference != "review") + and getattr(settings, "auto_approval_audio_hold_enabled", True) + ) payload["outcome"] = "review" + payload["hold_for_audio"] = hold_for_audio job_manager.update_processing_metadata(job_id, "auto_approval", payload) return {"outcome": "review", "overall_auto": verdict.overall_auto, - "blockers": blockers} + "blockers": blockers, "hold_for_audio": hold_for_audio} # ---- ENFORCE: complete the review exactly like a human would ---- # The status gate above used a snapshot read. Re-read now so a human who diff --git a/backend/services/auto_approval/review_gate.py b/backend/services/auto_approval/review_gate.py new file mode 100644 index 00000000..6cd625cf --- /dev/null +++ b/backend/services/auto_approval/review_gate.py @@ -0,0 +1,168 @@ +"""Review gate: hold confident-lyrics jobs out of review until stems land. + +Lyrics finish ~2-3 min before stem separation. When the screens worker scores a +job whose lyrics would auto-ship but whose stems aren't ready, opening review +right away loses the auto-ship: users open the editor within seconds +(AWAITING_REVIEW -> IN_REVIEW), so the audio worker's re-score finds a human in +the editor and backs off. Prod 2026-08-31..10-29: 109/340 jobs were last scored +without stems, so they never got a backing verdict, and lyrics-auto jobs went to +a human anyway. + +So for those jobs the screens worker parks the job in GENERATING_SCREENS with +``state_data.auto_review_waiting_for_stems`` set (no review link, no +notification). Whoever atomically claims that flag re-scores and routes the job: +the audio worker once stems land (normal case), the screens worker itself if the +stems landed while it was parking, or recover_stuck_jobs after +``AUDIO_HOLD_TIMEOUT_SECONDS`` (backstop). The re-score either auto-completes +the review or opens human review exactly like the non-held path. +""" + +import logging +from datetime import datetime +from typing import Any, Dict, Optional + +from backend.models.job import JobStatus + +logger = logging.getLogger(__name__) + +# Separation normally lands within a few minutes of lyrics; past this the +# recover_stuck_jobs sweep releases the hold into human review. +AUDIO_HOLD_TIMEOUT_SECONDS = 20 * 60 + + +async def enter_human_review( + job_id: str, job_log: Optional[logging.Logger] = None, job_manager=None +) -> None: + """Pre-apply AI corrections server-side, then open the combined review.""" + from backend.services.auto_approval.pre_apply import ensure_and_pre_apply + from backend.services.job_manager import JobManager + + # Apply the AI corrections + heuristic fixes BEFORE the review-ready + # notification fires, so the review screen loads one final state (no + # in-browser auto-apply race). Best-effort + bounded: on any miss the UI + # falls back to its client-side on-load apply. + pre_apply_result = await ensure_and_pre_apply(job_id) + if pre_apply_result.get("outcome") == "pre_applied" and job_log: + job_log.info( + f"Pre-applied {pre_apply_result.get('applied', 0)} AI corrections " + "server-side before review notification" + ) + + logger.info(f"[job:{job_id}] Screens generated, awaiting combined review") + (job_manager or JobManager()).transition_to_state( + job_id=job_id, + new_status=JobStatus.AWAITING_REVIEW, + progress=55, + message="Ready for review. Please review lyrics and select your instrumental.", + ) + + +async def route_after_scoring( + job_id: str, + auto_result: Dict[str, Any], + job_log: Optional[logging.Logger] = None, + job_manager=None, +) -> str: + """Screens-worker path for a job the first scoring did not auto-complete. + + Returns "held", "review", or the outcome of an immediate re-score. + """ + from backend.services.job_manager import JobManager + + job_manager = job_manager or JobManager() + if not auto_result.get("hold_for_audio"): + await enter_human_review(job_id, job_log, job_manager) + return "review" + + job_manager.park_review_for_auto_check(job_id) + fresh = job_manager.get_job(job_id) + if fresh and not (fresh.state_data or {}).get("audio_complete", False): + msg = "Lyrics look auto-approvable: holding review until audio separation finishes" + logger.info(f"[job:{job_id}] {msg}") + if job_log: + job_log.info(msg) + return "held" + + # Stems landed while we were parking. The audio worker may already have + # claimed the hold; the claim decides who re-scores. + if job_manager.claim_auto_review_hold(job_id): + logger.info(f"[job:{job_id}] Stems landed while holding review — re-scoring now") + return await _route_claimed(job_id, job_log, job_manager) + return "released_elsewhere" + + +async def release_audio_hold(job_id: str, reason: str) -> Optional[str]: + """Release a held job (audio worker / stuck-job sweep). + + Returns None if the job wasn't held (or someone else already claimed it), + else the routing outcome ("auto_completed", "review", ...). + """ + from backend.services.job_manager import JobManager + + job_manager = JobManager() + if not job_manager.claim_auto_review_hold(job_id): + return None + logger.info(f"[job:{job_id}] Releasing auto-review hold ({reason})") + return await _route_claimed(job_id, None, job_manager) + + +async def _route_claimed(job_id: str, job_log, job_manager) -> str: + """Route a job whose hold we just claimed. The claim already cleared the + marker, so on failure fall back to human review, and if even that fails put + the marker back so the recover_stuck_jobs sweep retries (never strand it).""" + try: + return await _rescore_and_route(job_id, job_log, job_manager) + except Exception as e: # noqa: BLE001 + logger.error(f"[job:{job_id}] Re-score after hold release failed: {e}; opening review", + exc_info=True) + try: + job = job_manager.get_job(job_id) + if job and job.status == JobStatus.GENERATING_SCREENS: + await enter_human_review(job_id, job_log, job_manager) + return "review" + except Exception as e: # noqa: BLE001 + logger.error(f"[job:{job_id}] Opening review after hold release failed: {e}; re-holding", + exc_info=True) + job_manager.park_review_for_auto_check(job_id) + return "error" + + +async def _rescore_and_route( + job_id: str, job_log: Optional[logging.Logger] = None, job_manager=None +) -> str: + from backend.services.auto_approval.executor import maybe_auto_complete_review + from backend.services.job_manager import JobManager + + job_manager = job_manager or JobManager() + job = job_manager.get_job(job_id) + if not job or job.status != JobStatus.GENERATING_SCREENS: + return "wrong_status" + + result = await maybe_auto_complete_review(job_id, trigger="audio_hold") + if result.get("outcome") == "auto_completed": + if job_log: + job_log.info("Auto-approved after audio completion: review skipped") + return "auto_completed" + + # Anything short of auto-complete (review verdict, abort, error, or audio + # still missing on a timeout release) goes to a human, as before the hold. + job = job_manager.get_job(job_id) + if job and job.status == JobStatus.GENERATING_SCREENS: + await enter_human_review(job_id, job_log, job_manager) + return "review" + return str(result.get("outcome")) + + +def audio_hold_expired(job, now: Optional[datetime] = None) -> bool: + """True when a held job has waited longer than AUDIO_HOLD_TIMEOUT_SECONDS.""" + from backend.services.job_manager import JobManager + + since = (job.state_data or {}).get(JobManager.AUTO_REVIEW_HOLD_KEY) + if not since or not isinstance(since, str): + return False + try: + started = datetime.fromisoformat(since) + except ValueError: + return True # unparseable marker: release rather than strand the job + now = now or datetime.utcnow() + return (now - started).total_seconds() > AUDIO_HOLD_TIMEOUT_SECONDS diff --git a/backend/services/job_manager.py b/backend/services/job_manager.py index 3bff5d80..8811b613 100644 --- a/backend/services/job_manager.py +++ b/backend/services/job_manager.py @@ -1534,6 +1534,26 @@ class JobManager: """Atomically clear the waiting flag; True for exactly one claimant.""" return self.firestore.claim_job_state_flag(job_id, self.VIDEO_WAITING_FOR_STEMS_KEY) + # Set by the screens worker when the lyrics would auto-ship but stems aren't + # separated yet. The job stays in GENERATING_SCREENS (no review link, no + # notification) so nobody opens the editor before the audio-worker re-score + # can auto-complete it. Before this, users opened review seconds after + # AWAITING_REVIEW and the re-score hit IN_REVIEW (wrong_status): ~1/3 of jobs + # never got a backing verdict and auto-eligible jobs went to a human + # (2026-10-29 re-evaluation). The value is the ISO time the hold started. + AUTO_REVIEW_HOLD_KEY = 'auto_review_waiting_for_stems' + + def park_review_for_auto_check(self, job_id: str) -> None: + """Record that review is held until stems land for the auto-approval re-score.""" + self.update_job(job_id, { + f'state_data.{self.AUTO_REVIEW_HOLD_KEY}': datetime.utcnow().isoformat(), + 'message': 'Lyrics look good: finishing automatic checks once audio separation is done', + }) + + def claim_auto_review_hold(self, job_id: str) -> bool: + """Atomically clear the review hold; True for exactly one claimant.""" + return self.firestore.claim_job_state_flag(job_id, self.AUTO_REVIEW_HOLD_KEY) + async def resume_video_if_waiting_for_stems(self, job_id: str) -> bool: """Re-trigger the video worker for a job parked waiting on stems. diff --git a/backend/tests/services/auto_approval/test_executor.py b/backend/tests/services/auto_approval/test_executor.py index 53726b51..008a19a0 100644 --- a/backend/tests/services/auto_approval/test_executor.py +++ b/backend/tests/services/auto_approval/test_executor.py @@ -74,7 +74,7 @@ def _settings(enforce: bool = True) -> SimpleNamespace: return SimpleNamespace(auto_approval_enforce_enabled=enforce) -async def _run(job, *, corrections=None, suggestions=None, settings=None): +async def _run(job, *, corrections=None, suggestions=None, settings=None, trigger="screens_worker"): """Drive maybe_auto_complete_review with everything mocked; returns (result, job_manager_mock, storage_mock, worker_service_mock).""" job_manager = MagicMock() @@ -99,7 +99,7 @@ async def _run(job, *, corrections=None, suggestions=None, settings=None): patch("backend.services.storage_service.StorageService", return_value=storage), \ patch("backend.config.get_settings", return_value=settings or _settings()), \ patch("backend.services.worker_service.get_worker_service", return_value=worker_service): - result = await maybe_auto_complete_review("j1", trigger="screens_worker") + result = await maybe_auto_complete_review("j1", trigger=trigger) return result, job_manager, storage, worker_service @@ -191,6 +191,52 @@ async def test_incomplete_audio_blocks_enforcement() -> None: jm.transition_to_state.assert_not_called() +@pytest.mark.asyncio +async def test_confident_lyrics_waiting_on_audio_asks_for_hold() -> None: + # Lyrics would auto-ship; only the stems are missing -> hold review for the + # audio-worker re-score instead of opening the editor (review_gate.py). + result, jm, _, _ = await _run(_job(audio_complete=False, backing=None)) + assert result["outcome"] == "review" + assert result["blockers"] == ["audio_incomplete"] + assert result["hold_for_audio"] is True + assert jm.update_processing_metadata.call_args.args[2]["hold_for_audio"] is True + + +@pytest.mark.asyncio +async def test_unconfident_lyrics_never_hold() -> None: + corrections = _confident_corrections() + corrections["reference_lyrics"] = {} # no references -> lyrics review + result, _, _, _ = await _run(_job(audio_complete=False, backing=None), corrections=corrections) + assert result["hold_for_audio"] is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("job_kwargs, settings", [ + ({"review_mode": "always_review"}, None), + ({"made_for_you": True}, None), + ({}, SimpleNamespace(auto_approval_enforce_enabled=True, auto_approval_audio_hold_enabled=False)), +]) +async def test_other_blockers_or_kill_switch_never_hold(job_kwargs, settings) -> None: + result, _, _, _ = await _run(_job(audio_complete=False, backing=None, **job_kwargs), settings=settings) + assert result["outcome"] == "review" + assert result["hold_for_audio"] is False + + +@pytest.mark.asyncio +async def test_backing_preference_review_never_holds() -> None: + job = _job(audio_complete=False, backing=None) + job.backing_preference = "review" + result, _, _, _ = await _run(job) + assert result["hold_for_audio"] is False + + +@pytest.mark.asyncio +async def test_audio_hold_trigger_rescores_generating_screens_job() -> None: + result, jm, _, _ = await _run(_job(backing=_clean_backing()), trigger="audio_hold") + assert result["outcome"] == "auto_completed" + assert jm.transition_to_state.call_args.kwargs["new_status"] == JobStatus.REVIEW_COMPLETE + + @pytest.mark.asyncio async def test_existing_instrumental_completes_with_custom_selection() -> None: # Tenant/bulk jobs with a user-supplied instrumental have NO instrumental diff --git a/backend/tests/services/auto_approval/test_review_gate.py b/backend/tests/services/auto_approval/test_review_gate.py new file mode 100644 index 00000000..caf1c258 --- /dev/null +++ b/backend/tests/services/auto_approval/test_review_gate.py @@ -0,0 +1,158 @@ +"""Tests for the review gate: holding lyrics-confident jobs out of review until +stems land, and releasing them exactly once (audio worker / screens worker race / +stuck-job sweep).""" +from __future__ import annotations + +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from backend.models.job import JobStatus +from backend.services.auto_approval import review_gate + +HOLD_KEY = "auto_review_waiting_for_stems" + + +def _job(status=JobStatus.GENERATING_SCREENS, **state): + return SimpleNamespace(status=status, state_data=dict(state)) + + +def _jm(*jobs, claim=True): + jm = MagicMock() + jm.get_job.side_effect = list(jobs) + jm.claim_auto_review_hold.return_value = claim + return jm + + +def _patches(jm, auto_result=None, pre_apply=None): + return ( + patch("backend.services.job_manager.JobManager", return_value=jm), + patch("backend.services.auto_approval.executor.maybe_auto_complete_review", + AsyncMock(return_value=auto_result or {"outcome": "review"})), + patch("backend.services.auto_approval.pre_apply.ensure_and_pre_apply", + AsyncMock(return_value=pre_apply or {"outcome": "skipped"})), + ) + + +async def _call(fn, jm, *args, auto_result=None, **kwargs): + p1, p2, p3 = _patches(jm, auto_result) + with p1, p2 as auto, p3 as pre: + out = await fn(*args, **kwargs) + return out, auto, pre + + +@pytest.mark.asyncio +async def test_no_hold_opens_review_with_pre_apply() -> None: + jm = _jm() + out, auto, pre = await _call(review_gate.route_after_scoring, jm, "j1", {"outcome": "review"}) + assert out == "review" + pre.assert_awaited_once_with("j1") + assert jm.transition_to_state.call_args.kwargs["new_status"] == JobStatus.AWAITING_REVIEW + jm.park_review_for_auto_check.assert_not_called() + auto.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_hold_parks_job_without_opening_review() -> None: + jm = _jm(_job(audio_complete=False)) + out, auto, pre = await _call( + review_gate.route_after_scoring, jm, "j1", {"outcome": "review", "hold_for_audio": True}) + assert out == "held" + jm.park_review_for_auto_check.assert_called_once_with("j1") + jm.transition_to_state.assert_not_called() + pre.assert_not_awaited() + auto.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_stems_landing_while_parking_rescores_immediately() -> None: + # Audio finished between scoring and parking: the screens worker claims the + # hold itself (the audio worker already ran its release and found no flag). + jm = _jm(_job(audio_complete=True), _job(audio_complete=True)) + out, auto, _ = await _call( + review_gate.route_after_scoring, jm, "j1", {"outcome": "review", "hold_for_audio": True}, + auto_result={"outcome": "auto_completed"}) + assert out == "auto_completed" + auto.assert_awaited_once_with("j1", trigger="audio_hold") + + +@pytest.mark.asyncio +async def test_stems_landing_while_parking_but_audio_worker_claimed() -> None: + jm = _jm(_job(audio_complete=True), claim=False) + out, auto, _ = await _call( + review_gate.route_after_scoring, jm, "j1", {"outcome": "review", "hold_for_audio": True}) + assert out == "released_elsewhere" + auto.assert_not_awaited() + jm.transition_to_state.assert_not_called() + + +@pytest.mark.asyncio +async def test_release_without_hold_is_noop() -> None: + jm = _jm(claim=False) + out, auto, _ = await _call(review_gate.release_audio_hold, jm, "j1", reason="audio_worker") + assert out is None + auto.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_release_auto_completes() -> None: + jm = _jm(_job()) + out, auto, pre = await _call(review_gate.release_audio_hold, jm, "j1", reason="audio_worker", + auto_result={"outcome": "auto_completed"}) + assert out == "auto_completed" + pre.assert_not_awaited() + jm.transition_to_state.assert_not_called() + + +@pytest.mark.asyncio +async def test_release_falls_back_to_human_review() -> None: + # Re-score says review (e.g. timing gate fired / backing needs a human). + jm = _jm(_job(), _job()) + out, _, pre = await _call(review_gate.release_audio_hold, jm, "j1", reason="audio_worker", + auto_result={"outcome": "review"}) + assert out == "review" + pre.assert_awaited_once() + assert jm.transition_to_state.call_args.kwargs["new_status"] == JobStatus.AWAITING_REVIEW + + +@pytest.mark.asyncio +async def test_release_after_job_moved_on_does_nothing() -> None: + jm = _jm(_job(status=JobStatus.FAILED)) + out, auto, _ = await _call(review_gate.release_audio_hold, jm, "j1", reason="hold_timeout") + assert out == "wrong_status" + auto.assert_not_awaited() + jm.transition_to_state.assert_not_called() + + +def test_audio_hold_expiry() -> None: + now = datetime(2026, 10, 29, 12, 0, 0) + fresh = _job(**{HOLD_KEY: (now - timedelta(minutes=5)).isoformat()}) + old = _job(**{HOLD_KEY: (now - timedelta(minutes=25)).isoformat()}) + assert review_gate.audio_hold_expired(fresh, now) is False + assert review_gate.audio_hold_expired(old, now) is True + assert review_gate.audio_hold_expired(_job(), now) is False + assert review_gate.audio_hold_expired(_job(**{HOLD_KEY: False}), now) is False + assert review_gate.audio_hold_expired(_job(**{HOLD_KEY: "garbage"}), now) is True + + +@pytest.mark.asyncio +async def test_rescore_crash_after_claim_falls_back_to_review() -> None: + jm = _jm(_job(), _job()) + p1, p2, p3 = _patches(jm) + with p1, patch("backend.services.auto_approval.executor.maybe_auto_complete_review", + AsyncMock(side_effect=RuntimeError("boom"))), p3: + out = await review_gate.release_audio_hold("j1", reason="audio_worker") + assert out == "review" + assert jm.transition_to_state.call_args.kwargs["new_status"] == JobStatus.AWAITING_REVIEW + + +@pytest.mark.asyncio +async def test_review_fallback_crash_restores_hold_for_sweep() -> None: + jm = _jm(_job(), _job()) + jm.transition_to_state.side_effect = RuntimeError("firestore down") + out, _, _ = await _call(review_gate.release_audio_hold, jm, "j1", reason="audio_worker", + auto_result={"outcome": "review"}) + assert out == "error" + jm.park_review_for_auto_check.assert_called_once_with("j1") diff --git a/backend/tests/test_recover_stuck_jobs_repark.py b/backend/tests/test_recover_stuck_jobs_repark.py index aa2df018..cd98c9f3 100644 --- a/backend/tests/test_recover_stuck_jobs_repark.py +++ b/backend/tests/test_recover_stuck_jobs_repark.py @@ -56,7 +56,8 @@ def _job(job_id, status, *, minutes_stale=0, source_name=None, download_retry=No def _mock_jm(*, downloading=None, pending=None, rendering=None, review_complete=None, - downloading_prep=None, instrumental_selected=None, jobs=None): + downloading_prep=None, instrumental_selected=None, generating_screens=None, + jobs=None): """Wire a JobManager mock for the endpoint's 4 status queries. downloading/rendering/review_complete use .where().stream(); pending uses @@ -65,9 +66,11 @@ def _mock_jm(*, downloading=None, pending=None, rendering=None, review_complete= mock_jm = MagicMock() where = mock_jm.firestore.db.collection.return_value.where.return_value # Non-limited .stream() calls in order: downloading_audio, rendering_video, - # review_complete (lost-render-trigger sweep) + # review_complete (lost-render-trigger sweep), generating_screens (expired + # auto-review holds) where.stream.side_effect = [ iter(downloading or []), iter(rendering or []), iter(review_complete or []), + iter(generating_screens or []), ] # Four .limit().stream() calls in order: download_pending_retry, then downloading # (lost-screens-trigger sweep, bounded by SCREENS_RECOVERY_SCAN_LIMIT), then @@ -355,3 +358,61 @@ def test_instrumental_selected_older_than_24h_is_never_revived(client): mock_ws.trigger_video_worker.assert_not_awaited() mock_jm.mark_job_failed.assert_not_called() + + +# --- GENERATING_SCREENS auto-review hold never released (review_gate.py) --- + +HOLD_KEY = "auto_review_waiting_for_stems" + + +def _held_job(job_id, *, minutes_held): + since = (datetime.utcnow() - timedelta(minutes=minutes_held)).isoformat() + job = _job(job_id, JobStatus.GENERATING_SCREENS) + job.state_data[HOLD_KEY] = since + return job, since + + +def _gs_doc(job_id, since): + doc = MagicMock() + doc.id = job_id + doc.to_dict.return_value = {"job_id": job_id, "status": JobStatus.GENERATING_SCREENS.value, + "state_data": {HOLD_KEY: since}} + return doc + + +def _run_hold(client, job, since, release_result="review"): + mock_jm = _mock_jm(generating_screens=[_gs_doc(job.job_id, since)], jobs={job.job_id: job}) + release = AsyncMock(return_value=release_result) + with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \ + patch("backend.services.worker_service.get_worker_service", return_value=MagicMock()), \ + patch("backend.services.auto_approval.review_gate.release_audio_hold", release): + resp = client.post("/api/internal/recover-stuck-jobs") + assert resp.status_code == 200 + return resp.json(), release + + +def test_expired_auto_review_hold_is_released(client): + job, since = _held_job("gs1", minutes_held=25) + body, release = _run_hold(client, job, since) + assert body["auto_review_hold_released_jobs"] == ["gs1"] + release.assert_awaited_once_with("gs1", reason="hold_timeout") + + +def test_fresh_auto_review_hold_is_left_alone(client): + job, since = _held_job("gs2", minutes_held=3) + body, release = _run_hold(client, job, since) + assert body["auto_review_hold_released_jobs"] == [] + release.assert_not_awaited() + + +def test_generating_screens_without_hold_is_ignored(client): + job = _job("gs3", JobStatus.GENERATING_SCREENS) + doc = _doc("gs3", JobStatus.GENERATING_SCREENS) + mock_jm = _mock_jm(generating_screens=[doc], jobs={"gs3": job}) + release = AsyncMock() + with patch("backend.api.routes.internal.JobManager", return_value=mock_jm), \ + patch("backend.services.worker_service.get_worker_service", return_value=MagicMock()), \ + patch("backend.services.auto_approval.review_gate.release_audio_hold", release): + body = client.post("/api/internal/recover-stuck-jobs").json() + assert body["auto_review_hold_released_jobs"] == [] + release.assert_not_awaited() diff --git a/backend/tests/test_screens_worker_review_hold.py b/backend/tests/test_screens_worker_review_hold.py new file mode 100644 index 00000000..c513b91e --- /dev/null +++ b/backend/tests/test_screens_worker_review_hold.py @@ -0,0 +1,72 @@ +"""Screens worker + review gate: a lyrics-confident job whose stems are still +separating is held in GENERATING_SCREENS (no review link, no notification) +instead of opening review. See backend/services/auto_approval/review_gate.py.""" +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +from backend.models.job import Job, JobStatus + + +async def _run_screens(auto_result, *, audio_complete): + from backend.workers import screens_worker as sw + + job = Job( + job_id="j1", artist="A", title="T", status=JobStatus.DOWNLOADING, + created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc), + state_data={"lyrics_complete": True, "audio_complete": audio_complete}, + ) + jm = MagicMock() + jm.get_job.return_value = job + jm.transition_to_state.return_value = True + style_config = MagicMock(include_title_screen=False, include_end_screen=False) + + async def load_style(*args, **kwargs): + return style_config + + async def noop(*args, **kwargs): + return {} + + async def auto(*args, **kwargs): + return auto_result + + pre_apply = MagicMock(side_effect=noop) + with patch.object(sw, "JobManager", return_value=jm), \ + patch.object(sw, "StorageService"), \ + patch.object(sw, "get_settings"), \ + patch.object(sw, "create_job_logger"), \ + patch.object(sw, "setup_job_logging"), \ + patch.object(sw, "_validate_prerequisites", return_value=True), \ + patch("backend.services.stems_restore.maybe_start_stems_restore", return_value="not_needed"), \ + patch.object(sw, "load_style_config", side_effect=load_style), \ + patch.object(sw, "_create_video_generator"), \ + patch.object(sw, "_upload_screens", side_effect=noop), \ + patch.object(sw, "_apply_countdown_padding_if_needed", side_effect=noop), \ + patch.object(sw, "_transcode_review_audio", side_effect=noop), \ + patch("backend.services.auto_approval.executor.maybe_auto_complete_review", side_effect=auto), \ + patch("backend.services.auto_approval.pre_apply.ensure_and_pre_apply", pre_apply): + assert await sw.generate_screens("j1") is True, jm.mark_job_failed.call_args + return jm, pre_apply + + +def _review_transitions(jm): + return [c for c in jm.transition_to_state.call_args_list + if c.kwargs.get("new_status") == JobStatus.AWAITING_REVIEW] + + +@pytest.mark.asyncio +async def test_held_job_does_not_open_review(): + jm, pre_apply = await _run_screens( + {"outcome": "review", "hold_for_audio": True}, audio_complete=False) + jm.park_review_for_auto_check.assert_called_once_with("j1") + assert _review_transitions(jm) == [] + pre_apply.assert_not_called() + + +@pytest.mark.asyncio +async def test_unheld_job_opens_review_as_before(): + jm, pre_apply = await _run_screens({"outcome": "review"}, audio_complete=False) + jm.park_review_for_auto_check.assert_not_called() + assert len(_review_transitions(jm)) == 1 + pre_apply.assert_called_once() diff --git a/backend/workers/audio_worker.py b/backend/workers/audio_worker.py index 582dff49..a0b37ab3 100644 --- a/backend/workers/audio_worker.py +++ b/backend/workers/audio_worker.py @@ -426,11 +426,19 @@ async def process_audio_separation(job_id: str) -> bool: # screens-worker scoring ran without backing analysis. Now that # stems + analysis exist, re-score and auto-complete the review if # fully confident. No-op (fail-safe) in every other state. + # A job the screens worker held out of review (lyrics confident, + # stems pending) is released here: re-scored, then auto-completed + # or opened for review (review_gate.py). try: from backend.services.auto_approval.executor import maybe_auto_complete_review - auto_result = await maybe_auto_complete_review(job_id, trigger="audio_worker") - if auto_result.get("outcome") == "auto_completed": + from backend.services.auto_approval.review_gate import release_audio_hold + released = await release_audio_hold(job_id, reason="audio_worker") + if released == "auto_completed": job_log.info("Auto-approved after audio completion: review skipped") + elif released is None: + auto_result = await maybe_auto_complete_review(job_id, trigger="audio_worker") + if auto_result.get("outcome") == "auto_completed": + job_log.info("Auto-approved after audio completion: review skipped") except Exception as e: logger.warning(f"[job:{job_id}] auto-approval attempt failed (non-fatal): {e}") diff --git a/backend/workers/screens_worker.py b/backend/workers/screens_worker.py index ebba5461..c69cbae8 100644 --- a/backend/workers/screens_worker.py +++ b/backend/workers/screens_worker.py @@ -297,29 +297,14 @@ async def generate_screens(job_id: str) -> bool: job_log.info("Auto-approved: review screens skipped, render triggered") logger.info(f"[job:{job_id}] Auto-approved — review skipped") else: - # Server-side pre-apply: apply the AI corrections + heuristic - # fixes BEFORE the review-ready notification fires, so the - # review screen loads one final state (no in-browser auto-apply - # race). Best-effort + bounded: on any miss the UI falls back to - # its client-side on-load apply. - from backend.services.auto_approval.pre_apply import ensure_and_pre_apply - with job_span("pre-apply-corrections", job_id): - pre_apply_result = await ensure_and_pre_apply(job_id) - if pre_apply_result.get("outcome") == "pre_applied": - job_log.info( - f"Pre-applied {pre_apply_result.get('applied', 0)} AI corrections " - "server-side before review notification" - ) + # Lyrics-confident jobs whose stems are still separating + # are held out of review for the audio-worker re-score; + # everything else opens review now (with server-side + # pre-applied AI corrections). See review_gate.py. + from backend.services.auto_approval.review_gate import route_after_scoring + with job_span("review-gate", job_id): + await route_after_scoring(job_id, auto_result, job_log, job_manager) - # Normal flow: transition to combined review - logger.info(f"[job:{job_id}] Screens generated, awaiting combined review") - job_manager.transition_to_state( - job_id=job_id, - new_status=JobStatus.AWAITING_REVIEW, - progress=55, - message="Ready for review. Please review lyrics and select your instrumental." - ) - duration = time.time() - start_time # Store worker-level timing job_manager.update_processing_metadata(job_id, "timing.screens_worker_seconds", round(duration, 1)) diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 8793b76c..28ee33bb 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -2500,3 +2500,26 @@ count and the cap is 15,000. Lesson: a safety cap must be measured on the same r threat it guards against, and a cap that truncates silently should be generous or fail loudly. Never repair an affected in-progress job with `add-lyrics`/`search-lyrics`: the frontend replaces its whole working data with the server's pre-edit rerun, discarding the reviewer's edits. + +## The auto-approval "second chance" almost never ran: users opened review first (2026-10-29, v0.287.0) + +Lyrics finish ~2-3 min before stem separation, so the screens worker usually +scores a job before its stems exist. The design was: open review anyway, and +let the audio worker re-score and auto-complete once stems land. In prod +(2026-08-31..10-29) that re-score hit `wrong_status` on about a third of +jobs, because people opened the review link within 15-40 s of AWAITING_REVIEW +(IN_REVIEW). Those jobs never got a backing verdict, and 7 lyrics-auto jobs +went to a human anyway. + +Fix (`backend/services/auto_approval/review_gate.py`): when the text verdict +is AUTO and the only blocker is `audio_incomplete`, the screens worker keeps +the job in GENERATING_SCREENS with `state_data.auto_review_waiting_for_stems` +(no review link, no notification). The flag is claimed atomically, by the +audio worker after stems land, by the screens worker itself if stems landed +while it was parking, or by recover_stuck_jobs after 20 min. The claimant +re-scores (`trigger="audio_hold"`) and either auto-completes or opens review +exactly like the old path. Kill switch: `AUTO_APPROVAL_AUDIO_HOLD_ENABLED`. + +Lesson: an async "second chance" that only runs when no human has touched the +job will lose that race to a fast human. Hold the human-facing state until the +automatic decision is final, rather than racing it. diff --git a/docs/README.md b/docs/README.md index c0f1e50f..c818117c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,7 +26,7 @@ | Auto-correction (agentic + heuristic) | Disabled (raw transcription goes to human review) | | **AI auto-correct suggestions (opt-in)** | Working ("AI Suggest" button in lyrics review — whole-song LLM call, per-suggestion accept/reject) | | Combined review (lyrics + instrumental) | Working | -| **Auto-approval (fully-auto review)** | Live (default `review_mode="auto"`): when the scorer is fully confident (synced-reference lyrics with all gaps AI-covered + no audible backing vocals), the review screens are skipped and the job renders straight through; every other job falls back to human review. Verdict on every job in `processing_metadata.auto_approval`. Kill switch `AUTO_APPROVAL_ENFORCE_ENABLED=false`; per-job `review_mode="always_review"` (admin-editable). See `docs/archive/2026-08-25-full-auto-review-design.md` | +| **Auto-approval (fully-auto review)** | Live (default `review_mode="auto"`): when the scorer is fully confident (synced-reference lyrics with all gaps AI-covered + no audible backing vocals), the review screens are skipped and the job renders straight through; every other job falls back to human review. Verdict on every job in `processing_metadata.auto_approval`. Kill switch `AUTO_APPROVAL_ENFORCE_ENABLED=false`; per-job `review_mode="always_review"` (admin-editable). If the lyrics are confident but stems are still separating, the job waits in `generating_screens` (no review link) until the audio worker re-scores it (`auto_approval/review_gate.py`, v0.287.0, kill switch `AUTO_APPROVAL_AUDIO_HOLD_ENABLED`). See `docs/archive/2026-08-25-full-auto-review-design.md` | | **YouTube comment auto-reply** | Live (2026-10-22): hourly bot replies to song requests in channel comments with a requests-board link, pre-adding the song (`YOUTUBE_COMMENT_AUTOREPLY_ENABLED`). See [REQUESTS-BOARD.md](REQUESTS-BOARD.md#youtube-comment-auto-reply-song-requests--board) | | **Review replay tool (admin)** | Working (`?replay=1` read-only replay of completed jobs — see `docs/REPLAY.md`) | | Preview video generation | Working | diff --git a/pyproject.toml b/pyproject.toml index 8ddb2f9d..3886ddf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.286.2" +version = "0.287.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"