diff --git a/backend/api/routes/admin.py b/backend/api/routes/admin.py index d686503f..db40867a 100644 --- a/backend/api/routes/admin.py +++ b/backend/api/routes/admin.py @@ -799,6 +799,9 @@ class SendCompletionEmailRequest(BaseModel): """Request to send a completion email.""" to_email: str cc_admin: bool = True + # Send even though a job meant for Dropbox delivery has no Dropbox link + # (the Dropbox section is then left out of the email, never a placeholder). + allow_missing_dropbox_link: bool = False class SendCompletionEmailResponse(BaseModel): @@ -2097,6 +2100,19 @@ def send_job_completion_email( dropbox_url = state_data.get('dropbox_link') brand_code = state_data.get('brand_code') + # Refuse to send a delivery email with no files link unless explicitly overridden. + from backend.services.delivery_guard import completion_email_hold_reason + hold = completion_email_hold_reason(job) + if hold and not request.allow_missing_dropbox_link: + raise HTTPException( + status_code=409, + detail=( + f"Job {job_id} was meant to deliver to Dropbox but has no Dropbox link " + f"({hold['reason']}). Fix the Dropbox upload and set state_data.dropbox_link " + "first, or resend with allow_missing_dropbox_link=true to send without it." + ), + ) + # Render the completion message notification_service = get_job_notification_service() message = notification_service.get_completion_message( @@ -2122,6 +2138,12 @@ def send_job_completion_email( if success: logger.info(f"Admin sent completion email for job {job_id} to {request.to_email}") + if state_data.get('completion_email_held'): + try: + from google.cloud.firestore_v1 import DELETE_FIELD + job_manager.update_job(job_id, {'state_data.completion_email_held': DELETE_FIELD}) + except Exception as e: + logger.warning(f"Failed to clear completion_email_held for job {job_id}: {e}") return SendCompletionEmailResponse( success=True, job_id=job_id, diff --git a/backend/api/routes/internal.py b/backend/api/routes/internal.py index 568d7ad1..49378f5e 100644 --- a/backend/api/routes/internal.py +++ b/backend/api/routes/internal.py @@ -644,6 +644,7 @@ def process_stale_reviews_endpoint( - Signed-URL upload jobs whose browser upload never finished (>= 2h) are auto-cancelled with credit refund - Free-to-paid conversion emails (when CONVERSION_EMAILS_ENABLED) + - Dropbox space-usage check (one #alerts message when >= 95% full) """ from backend.workers.stale_review_processor import process_stale_reviews from backend.workers.stale_upload_processor import process_stale_uploads @@ -672,6 +673,14 @@ def process_stale_reviews_endpoint( logger.info(f"CONVERSION_EMAILS complete: {conversion_result}") except Exception as e: logger.exception(f"CONVERSION_EMAILS failed: {e}") + # Proactive Dropbox quota check (alerts #alerts once at >=95% used) โ€” + # a full Dropbox silently breaks customer delivery (NOMAD-1800). + try: + from backend.services.delivery_guard import check_dropbox_space_usage + quota_result = await asyncio.to_thread(check_dropbox_space_usage) + logger.info(f"DROPBOX_QUOTA_CHECK complete: {quota_result}") + except Exception as e: + logger.warning(f"DROPBOX_QUOTA_CHECK failed: {e}") background_tasks.add_task(_process) diff --git a/backend/services/delivery_guard.py b/backend/services/delivery_guard.py new file mode 100644 index 00000000..d8c0e0e8 --- /dev/null +++ b/backend/services/delivery_guard.py @@ -0,0 +1,278 @@ +"""Customer-delivery guard for Dropbox distribution. + +Why this exists +--------------- +Job ``dd892383`` (NOMAD-1800, 2026-10-22) finished while the business Dropbox was +at its 6 TiB quota. Both Dropbox upload passes failed with +``insufficient_space``, the failure was only recorded as a +``distribution_warnings`` entry + a Pushbullet push, and the job still went +COMPLETE and emailed the customer a delivery email whose Dropbox link was the +literal placeholder ``[Dropbox URL not available]``. The customer replied that +there was no link. The only #alerts signal was a raw traceback. + +This module makes that impossible to repeat silently: + +* :func:`completion_email_hold_reason` decides whether a completion email must + be HELD (the job was meant to deliver to Dropbox but has no share link). +* :func:`hold_completion_email` records ``state_data.completion_email_held`` and + posts ONE actionable #alerts message per job (what broke, impact, fix). +* :func:`check_dropbox_space_usage` is the proactive quota check: it alerts once + when the Dropbox crosses :data:`DROPBOX_QUOTA_ALERT_THRESHOLD` used, and stays + quiet while it remains high. + +Everything here is best-effort and never raises into the caller. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +# Fraction of the Dropbox allocation at which we page #alerts. +DROPBOX_QUOTA_ALERT_THRESHOLD = 0.95 + +# One held-delivery alert per job: throttle key is per job, window is a week so +# a re-completion of the same job within that time doesn't page twice. +_HELD_ALERT_THROTTLE_MINUTES = 7 * 24 * 60 + +# Firestore doc holding the quota-alert state (shares the ops-alert dedup collection). +_QUOTA_STATE_COLLECTION = "ops_failure_alerts" +_QUOTA_STATE_DOC = "dropbox-space-usage" + +_TROUBLESHOOTING_REF = "karaoke-gen docs/TROUBLESHOOTING.md ยง Held completion email (Dropbox delivery failed)" + +# Reason codes stored in state_data.completion_email_held.reason_code +REASON_DROPBOX_FULL = "dropbox_full" +REASON_DROPBOX_UPLOAD_FAILED = "dropbox_upload_failed" +REASON_DROPBOX_LINK_MISSING = "dropbox_link_missing" + + +def classify_dropbox_failure(warnings: Optional[list]) -> tuple[str, str]: + """Classify a job's Dropbox distribution failure from its warnings. + + Returns ``(reason_code, human_reason)``. + """ + dropbox_warnings = [ + str(w) for w in (warnings or []) if "dropbox" in str(w).lower() + ] + for w in dropbox_warnings: + if "insufficient_space" in w: + return REASON_DROPBOX_FULL, "Dropbox is FULL (insufficient_space)" + if dropbox_warnings: + detail = dropbox_warnings[0] + if len(detail) > 300: + detail = detail[:300] + " โ€ฆ" + return REASON_DROPBOX_UPLOAD_FAILED, detail + return REASON_DROPBOX_LINK_MISSING, "Dropbox upload produced no share link" + + +def job_expects_dropbox_delivery(job: Any) -> bool: + """True if this job's distribution settings upload it to Dropbox. + + Mirrors the worker condition (``dropbox_path and brand_prefix``) on the + effective settings, so private (NonPublished) jobs count too and tenant / + no-Dropbox jobs don't. + """ + try: + from backend.services.job_defaults_service import get_effective_distribution_for_job + + dist = get_effective_distribution_for_job(job) + return bool(dist.dropbox_path and dist.brand_prefix) + except Exception as exc: # noqa: BLE001 - never block an email on a settings bug + logger.warning("delivery_guard: could not resolve distribution for job %s: %s", + getattr(job, "job_id", "?"), exc) + return False + + +def completion_email_hold_reason(job: Any) -> Optional[Dict[str, str]]: + """Return a hold reason if the completion email must NOT be sent, else None. + + The email is held when the job was meant to deliver to Dropbox but + ``state_data.dropbox_link`` is missing โ€” sending it would hand the customer a + delivery email with no files link. + """ + state_data = getattr(job, "state_data", None) or {} + if state_data.get("dropbox_link"): + return None + if not job_expects_dropbox_delivery(job): + return None + reason_code, reason = classify_dropbox_failure(state_data.get("distribution_warnings")) + return {"reason_code": reason_code, "reason": reason} + + +def build_held_delivery_alert( + job: Any, hold: Dict[str, str], recipient: Optional[str] +) -> str: + """Build the actionable #alerts message for a held delivery email.""" + state_data = getattr(job, "state_data", None) or {} + brand_code = state_data.get("brand_code") + track = " - ".join(p for p in (getattr(job, "artist", None), getattr(job, "title", None)) if p) + lines = [ + "๐Ÿ”ด **Customer delivery HELD โ€” Dropbox upload failed**", + f"**Job:** `{job.job_id}`" + (f" ({brand_code})" if brand_code else ""), + f"**Track:** {track or '(unknown track)'}", + f"**Reason:** {hold['reason']}", + f"**Impact:** completion email to {_mask_email(recipient)} was NOT sent (no Dropbox link). " + "The job is COMPLETE; files are in GCS.", + ] + if hold.get("reason_code") == REASON_DROPBOX_FULL: + lines.append( + "**Fix:** 1) free space in the business Dropbox; 2) upload the job's outputs to its " + "Dropbox folder + create the share link, set `state_data.dropbox_link`; " + "3) `POST /api/admin/jobs/{id}/send-completion-email`." + ) + else: + lines.append( + "**Fix:** 1) check Dropbox credentials/API status; 2) upload the job's outputs to its " + "Dropbox folder + create the share link, set `state_data.dropbox_link`; " + "3) `POST /api/admin/jobs/{id}/send-completion-email`." + ) + lines.append(f"**Runbook:** {_TROUBLESHOOTING_REF}") + return "\n".join(lines) + + +def hold_completion_email( + job_manager: Any, job: Any, hold: Dict[str, str], recipient: Optional[str] +) -> None: + """Record the held email on the job and post one actionable #alerts message.""" + job_id = job.job_id + logger.warning( + f"[job:{job_id}] Completion email HELD: {hold['reason']} " + f"(job expects Dropbox delivery but has no dropbox_link)" + ) + try: + job_manager.update_job(job_id, { + "state_data.completion_email_held": { + "reason_code": hold["reason_code"], + "reason": hold["reason"], + "recipient": recipient, + "at": datetime.now(timezone.utc).isoformat(), + } + }) + except Exception as exc: # noqa: BLE001 + logger.warning(f"[job:{job_id}] Failed to record completion_email_held: {exc}") + + try: + from backend.services.ops_alerts import send_throttled_ops_alert + + send_throttled_ops_alert( + f"delivery-held-{job_id}", + build_held_delivery_alert(job, hold, recipient), + throttle_minutes=_HELD_ALERT_THROTTLE_MINUTES, + ) + except Exception as exc: # noqa: BLE001 - alerting must never raise + logger.warning(f"[job:{job_id}] Failed to send held-delivery alert: {exc}") + + +# ============================================================================= +# Proactive Dropbox quota check +# ============================================================================= + +def space_usage_bytes(usage: Any) -> tuple[int, int]: + """Extract ``(used, allocated)`` bytes from a Dropbox ``SpaceUsage``. + + For a team (business) account the quota is the TEAM allocation, so use the + team-wide ``used`` against it. ``allocated == 0`` means unknown/unlimited. + """ + allocation = usage.allocation + if allocation.is_team(): + team = allocation.get_team() + return int(team.used), int(team.allocated) + if allocation.is_individual(): + return int(usage.used), int(allocation.get_individual().allocated) + return int(usage.used), 0 + + +def evaluate_quota( + used: int, allocated: int, was_alerting: bool, + threshold: float = DROPBOX_QUOTA_ALERT_THRESHOLD, +) -> str: + """Decide what to do for one quota reading. + + Returns ``"alert"`` (crossed the threshold โ€” page once), ``"recovered"`` + (dropped back below it after an alert), or ``"none"``. + """ + if allocated <= 0: + return "none" + high = (used / allocated) >= threshold + if high and not was_alerting: + return "alert" + if not high and was_alerting: + return "recovered" + return "none" + + +def _fmt_tib(n: int) -> str: + return f"{n / (1024 ** 4):.3f} TiB" + + +def build_quota_alert(used: int, allocated: int) -> str: + pct = used / allocated * 100 + return "\n".join([ + f"๐Ÿ”ด **Dropbox almost FULL โ€” {pct:.1f}% used** ({_fmt_tib(used)} of {_fmt_tib(allocated)})", + "**Impact:** once full, every job's Dropbox upload fails with `insufficient_space` and " + "customer completion emails are HELD (no files link).", + "**Fix:** free space in the business Dropbox (or raise the quota). " + "This alert fires once; it re-arms after usage drops below " + f"{DROPBOX_QUOTA_ALERT_THRESHOLD * 100:.0f}%.", + ]) + + +def check_dropbox_space_usage(db: Any = None, dropbox_service: Any = None) -> Dict[str, Any]: + """Read Dropbox space usage and alert #alerts once when it crosses the threshold. + + Rides on an existing hourly cron. Best-effort; never raises. Returns a small + result dict for logging. + """ + try: + if dropbox_service is None: + from backend.services.dropbox_service import get_dropbox_service + dropbox_service = get_dropbox_service() + if not dropbox_service.is_configured: + return {"status": "skipped", "reason": "dropbox not configured"} + + used, allocated = space_usage_bytes(dropbox_service.client.users_get_space_usage()) + if db is None: + from backend.services.firestore_service import get_firestore_client + db = get_firestore_client() + + doc_ref = db.collection(_QUOTA_STATE_COLLECTION).document(_QUOTA_STATE_DOC) + snap = doc_ref.get() + state = (snap.to_dict() or {}) if snap.exists else {} + was_alerting = bool(state.get("alerting")) + + action = evaluate_quota(used, allocated, was_alerting) + pct = round(used / allocated * 100, 2) if allocated else None + alerting = was_alerting + if action == "alert": + from backend.services.ops_alerts import send_ops_alert + # Only latch "alerting" once delivery succeeded, so a failed send retries next run. + alerting = bool(send_ops_alert(build_quota_alert(used, allocated))) + logger.warning(f"DROPBOX_QUOTA high: {pct}% used (alert_sent={alerting})") + elif action == "recovered": + alerting = False + logger.info(f"DROPBOX_QUOTA recovered: {pct}% used") + else: + logger.info(f"DROPBOX_QUOTA ok: {pct}% used") + + doc_ref.set({ + "alerting": alerting, + "used_bytes": used, + "allocated_bytes": allocated, + "used_pct": pct, + "checked_at": datetime.now(timezone.utc).isoformat(), + }, merge=True) + return {"status": "ok", "used_pct": pct, "action": action, "alerting": alerting} + except Exception as exc: # noqa: BLE001 - a health check must never raise + logger.warning(f"DROPBOX_QUOTA check failed: {exc}") + return {"status": "error", "error": str(exc)} + + +def _mask_email(email: Optional[str]) -> str: + if not email or "@" not in email: + return "(no recipient)" + local, domain = email.split("@", 1) + return f"{local[:2]}***@{domain}" diff --git a/backend/services/job_manager.py b/backend/services/job_manager.py index fd0c9f25..815ab784 100644 --- a/backend/services/job_manager.py +++ b/backend/services/job_manager.py @@ -732,6 +732,18 @@ class JobManager: except Exception as e: logger.error(f"Failed to transfer ownership for job {job.job_id}: {e}") + # Never send a delivery email without its files link (NOMAD-1800: Dropbox + # full -> customer got "[Dropbox URL not available]"). Hold it, flag the + # job, and post one actionable #alerts message instead. + from backend.services.delivery_guard import ( + completion_email_hold_reason, + hold_completion_email, + ) + hold = completion_email_hold_reason(job) + if hold: + hold_completion_email(self, job, hold, recipient_email) + return + # Create async task (fire-and-forget) async def send_email(): await notification_service.send_job_completion_email( diff --git a/backend/services/template_service.py b/backend/services/template_service.py index 20599027..f79c0b0d 100644 --- a/backend/services/template_service.py +++ b/backend/services/template_service.py @@ -357,10 +357,35 @@ class TemplateService: queued_msg ) + elif not youtube_url: + # No video link and none pending: drop the YouTube section rather than + # rendering a "[... not available]" placeholder to the customer. + template = self._remove_link_section( + template, + intros=[ + "Here's the link for the karaoke video published to YouTube:", + t(locale, "emails.templates.jobCompletion.youtubeLink"), + ], + placeholder="{youtube_url}", + ) + + if not dropbox_url: + # Never show a placeholder Dropbox link (NOMAD-1800). Jobs meant for + # Dropbox delivery have their email HELD upstream (delivery_guard); this + # covers jobs with no Dropbox delivery at all (and admin overrides). + template = self._remove_link_section( + template, + intros=[ + "Here's the dropbox folder", + t(locale, "emails.templates.jobCompletion.dropboxIntro"), + ], + placeholder="{dropbox_url}", + ) + variables = { "name": name or "there", - "youtube_url": youtube_url or "[YouTube URL not available]", - "dropbox_url": dropbox_url or "[Dropbox URL not available]", + "youtube_url": youtube_url or "", + "dropbox_url": dropbox_url or "", "artist": artist or "Unknown Artist", "title": title or "Unknown Title", "job_id": job_id or "", @@ -368,6 +393,26 @@ class TemplateService: } return self.render_template(template, variables) + @staticmethod + def _remove_link_section(template: str, intros: list, placeholder: str) -> str: + """Remove the paragraph from an intro line through its link placeholder. + + Tries each intro text (English GCS/default wording, then the localized + one). If none matches, only the placeholder line is dropped, so a missing + link renders as nothing rather than a placeholder. + """ + for intro in intros: + if not intro or intro.startswith("emails."): # untranslated key + continue + # Match from the intro line's start through the placeholder line. + pattern = r"\n*[^\n]*" + re.escape(intro.rstrip(":").rstrip()[:40]) + r".*?" + re.escape(placeholder) + r"[^\n]*\n*" + match = re.search(pattern, template, flags=re.DOTALL) + # Don't swallow a neighbouring section: the span may only contain + # its own placeholder. + if match and not (set(re.findall(r"\{[a-z_]+\}", match.group(0))) - {placeholder}): + return template[:match.start()] + "\n\n" + template[match.end():] + return re.sub(r"\n*[^\n]*" + re.escape(placeholder) + r"[^\n]*\n*", "\n\n", template, count=1) + def render_action_needed_lyrics( self, name: Optional[str] = None, diff --git a/backend/tests/test_admin_email_endpoints.py b/backend/tests/test_admin_email_endpoints.py index 10888275..2188cf16 100644 --- a/backend/tests/test_admin_email_endpoints.py +++ b/backend/tests/test_admin_email_endpoints.py @@ -46,6 +46,11 @@ def mock_job(): job.audio_hash = "hash123" job.review_token = "review123" job.instrumental_token = "inst123" + # Distribution settings (Mock(spec=Job) attrs would otherwise be truthy Mocks) + job.is_private = False + job.tenant_id = None + job.dropbox_path = None + job.brand_prefix = None job.state_data = { "youtube_url": "https://youtube.com/watch?v=test123", "dropbox_link": "https://dropbox.com/folder/test", @@ -330,6 +335,75 @@ class TestSendCompletionEmail: assert call_kwargs["to_email"] == "invalid-email" +class TestSendCompletionEmailDeliveryGuard: + """NOMAD-1800: refuse to send a delivery email without its Dropbox link.""" + + def _patches(self, mock_job): + mock_jm = Mock() + mock_jm.get_job.return_value = mock_job + mock_ns = Mock() + mock_ns.get_completion_message.return_value = "Message" + mock_es = Mock() + mock_es.send_job_completion.return_value = True + return mock_jm, mock_ns, mock_es + + def _post(self, client, mock_job, body): + mock_jm, mock_ns, mock_es = self._patches(mock_job) + with patch('backend.api.routes.admin.JobManager', return_value=mock_jm), \ + patch('backend.services.job_notification_service.get_job_notification_service', return_value=mock_ns), \ + patch('backend.services.email_service.get_email_service', return_value=mock_es): + response = client.post( + "/api/admin/jobs/test-job-123/send-completion-email", + json=body, + headers={"Authorization": "Bearer admin-token"}, + ) + return response, mock_jm, mock_es + + def _dropbox_job(self, mock_job, **state): + mock_job.dropbox_path = "/Karaoke/Tracks-Organized" + mock_job.brand_prefix = "NOMAD" + mock_job.state_data = {"brand_code": "NOMAD-1800", **state} + return mock_job + + def test_refuses_when_dropbox_link_missing(self, client, mock_job): + self._dropbox_job(mock_job, distribution_warnings=[ + "Dropbox upload failed: WriteError('insufficient_space')" + ]) + response, _, mock_es = self._post(client, mock_job, {"to_email": "c@example.com"}) + + assert response.status_code == 409 + assert "Dropbox is FULL" in response.json()["detail"] + mock_es.send_job_completion.assert_not_called() + + def test_override_sends_and_clears_held_marker(self, client, mock_job): + self._dropbox_job(mock_job, completion_email_held={"reason_code": "dropbox_full"}) + response, mock_jm, mock_es = self._post( + client, mock_job, {"to_email": "c@example.com", "allow_missing_dropbox_link": True} + ) + + assert response.status_code == 200 + mock_es.send_job_completion.assert_called_once() + mock_jm.update_job.assert_called_once() + assert "state_data.completion_email_held" in mock_jm.update_job.call_args[0][1] + + def test_sends_and_clears_marker_once_link_is_fixed(self, client, mock_job): + self._dropbox_job(mock_job, dropbox_link="https://dropbox.com/fixed", + completion_email_held={"reason_code": "dropbox_full"}) + response, mock_jm, mock_es = self._post(client, mock_job, {"to_email": "c@example.com"}) + + assert response.status_code == 200 + mock_es.send_job_completion.assert_called_once() + assert "state_data.completion_email_held" in mock_jm.update_job.call_args[0][1] + + def test_job_without_dropbox_delivery_sends_normally(self, client, mock_job): + mock_job.state_data = {"brand_code": "TEN-0001"} + response, mock_jm, mock_es = self._post(client, mock_job, {"to_email": "c@example.com"}) + + assert response.status_code == 200 + mock_es.send_job_completion.assert_called_once() + mock_jm.update_job.assert_not_called() + + class TestIdleReminderEndpoint: """Tests for the internal idle reminder endpoint.""" diff --git a/backend/tests/test_delivery_guard.py b/backend/tests/test_delivery_guard.py new file mode 100644 index 00000000..2a31fe82 --- /dev/null +++ b/backend/tests/test_delivery_guard.py @@ -0,0 +1,290 @@ +"""Unit tests for backend.services.delivery_guard (NOMAD-1800 Dropbox-full delivery guard).""" +from datetime import datetime, UTC +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from backend.models.job import Job, JobStatus +from backend.services import delivery_guard as dg + + +INSUFFICIENT_SPACE_WARNING = ( + "Dropbox upload failed: ApiError('abc', UploadError('path', " + "UploadWriteFailed(reason=WriteError('insufficient_space', None))))" +) + + +def _job(**overrides) -> Job: + base = dict( + job_id="dd892383", + status=JobStatus.COMPLETE, + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + user_email="madeforyou@nomadkaraoke.com", + artist="Test Artist", + title="Test Song", + dropbox_path="/Karaoke/Tracks-Organized", + brand_prefix="NOMAD", + state_data={"brand_code": "NOMAD-1800"}, + ) + base.update(overrides) + return Job(**base) + + +# --------------------------------------------------------------------------- +# classify_dropbox_failure +# --------------------------------------------------------------------------- + +class TestClassifyDropboxFailure: + def test_insufficient_space_is_dropbox_full(self): + code, reason = dg.classify_dropbox_failure([INSUFFICIENT_SPACE_WARNING]) + assert code == dg.REASON_DROPBOX_FULL + assert "FULL" in reason + + def test_insufficient_space_found_among_other_warnings(self): + code, _ = dg.classify_dropbox_failure([ + "Google Drive upload failed: boom", + INSUFFICIENT_SPACE_WARNING, + ]) + assert code == dg.REASON_DROPBOX_FULL + + def test_other_dropbox_error_is_upload_failed_with_detail(self): + code, reason = dg.classify_dropbox_failure(["Dropbox upload failed: InternalServerError 500"]) + assert code == dg.REASON_DROPBOX_UPLOAD_FAILED + assert "500" in reason + + def test_long_detail_is_truncated(self): + _, reason = dg.classify_dropbox_failure(["Dropbox upload failed: " + "x" * 1000]) + assert len(reason) < 320 + + def test_no_dropbox_warning_is_link_missing(self): + code, _ = dg.classify_dropbox_failure(["Google Drive upload failed: boom"]) + assert code == dg.REASON_DROPBOX_LINK_MISSING + assert dg.classify_dropbox_failure(None)[0] == dg.REASON_DROPBOX_LINK_MISSING + + +# --------------------------------------------------------------------------- +# completion_email_hold_reason +# --------------------------------------------------------------------------- + +class TestCompletionEmailHoldReason: + def test_holds_when_dropbox_expected_and_link_missing(self): + job = _job(state_data={"distribution_warnings": [INSUFFICIENT_SPACE_WARNING]}) + hold = dg.completion_email_hold_reason(job) + assert hold == {"reason_code": dg.REASON_DROPBOX_FULL, "reason": "Dropbox is FULL (insufficient_space)"} + + def test_no_hold_when_link_present(self): + job = _job(state_data={"dropbox_link": "https://dropbox.com/x"}) + assert dg.completion_email_hold_reason(job) is None + + def test_no_hold_when_no_dropbox_configured(self): + """Jobs that never deliver to Dropbox (e.g. tenant/no-dropbox) still email.""" + job = _job(dropbox_path=None, brand_prefix=None, state_data={}) + assert dg.completion_email_hold_reason(job) is None + + def test_no_hold_for_tenant_job_without_dropbox(self): + job = _job(tenant_id="acme", is_private=True, dropbox_path=None, brand_prefix=None, state_data={}) + assert dg.completion_email_hold_reason(job) is None + + def test_no_hold_when_dropbox_path_but_no_brand_prefix(self): + """Workers only upload when BOTH are set โ€” mirror that.""" + job = _job(brand_prefix=None, state_data={}) + assert dg.completion_email_hold_reason(job) is None + + def test_private_job_uses_private_dropbox_defaults(self): + settings = MagicMock(default_private_dropbox_path="/Karaoke/Tracks-NonPublished", + default_private_brand_prefix="NOMADNP") + job = _job(is_private=True, dropbox_path=None, brand_prefix=None, state_data={}) + with patch("backend.services.job_defaults_service.get_settings", return_value=settings): + hold = dg.completion_email_hold_reason(job) + assert hold["reason_code"] == dg.REASON_DROPBOX_LINK_MISSING + + def test_private_job_without_private_dropbox_configured_does_not_hold(self): + settings = MagicMock(default_private_dropbox_path=None, default_private_brand_prefix="NOMADNP") + job = _job(is_private=True, dropbox_path=None, brand_prefix=None, state_data={}) + with patch("backend.services.job_defaults_service.get_settings", return_value=settings): + assert dg.completion_email_hold_reason(job) is None + + def test_settings_error_fails_open(self): + job = _job(state_data={}) + with patch("backend.services.job_defaults_service.get_effective_distribution_for_job", + side_effect=RuntimeError("boom")): + assert dg.completion_email_hold_reason(job) is None + + +# --------------------------------------------------------------------------- +# hold_completion_email + alert text +# --------------------------------------------------------------------------- + +class TestHoldCompletionEmail: + def test_records_marker_and_sends_one_per_job_alert(self): + job = _job(state_data={"brand_code": "NOMAD-1800", + "distribution_warnings": [INSUFFICIENT_SPACE_WARNING]}) + hold = dg.completion_email_hold_reason(job) + jm = Mock() + jm.update_job.return_value = None + + with patch("backend.services.ops_alerts.send_throttled_ops_alert", return_value=True) as send: + dg.hold_completion_email(jm, job, hold, "christian@example.de") + + jm.update_job.assert_called_once() + job_id, updates = jm.update_job.call_args[0] + assert job_id == "dd892383" + marker = updates["state_data.completion_email_held"] + assert marker["reason_code"] == dg.REASON_DROPBOX_FULL + assert marker["recipient"] == "christian@example.de" + assert marker["at"] + + send.assert_called_once() + key, message = send.call_args[0] + assert key == "delivery-held-dd892383" + assert send.call_args.kwargs["throttle_minutes"] >= 24 * 60 + assert "dd892383" in message and "NOMAD-1800" in message + assert "Dropbox is FULL" in message + assert "NOT sent" in message + assert "free space" in message + assert "send-completion-email" in message + # Customer email is masked in #alerts + assert "christian@example.de" not in message + + def test_firestore_and_alert_failures_never_raise(self): + job = _job(state_data={}) + jm = Mock() + jm.update_job.side_effect = RuntimeError("firestore down") + with patch("backend.services.ops_alerts.send_throttled_ops_alert", side_effect=RuntimeError("x")): + dg.hold_completion_email(jm, job, {"reason_code": "x", "reason": "y"}, None) + + def test_non_full_alert_points_at_credentials(self): + job = _job(state_data={}) + msg = dg.build_held_delivery_alert( + job, {"reason_code": dg.REASON_DROPBOX_UPLOAD_FAILED, "reason": "500"}, None + ) + assert "credentials" in msg + assert "(no recipient)" in msg + + +# --------------------------------------------------------------------------- +# Quota check +# --------------------------------------------------------------------------- + +TIB = 1024 ** 4 + + +def _team_usage(used, allocated): + from dropbox.users import SpaceAllocation, SpaceUsage, TeamSpaceAllocation + + team = TeamSpaceAllocation( + used=used, allocated=allocated, user_within_team_space_allocated=0, + user_within_team_space_used_cached=0, + ) + return SpaceUsage(used=1, allocation=SpaceAllocation.team(team)) + + +def _individual_usage(used, allocated): + from dropbox.users import IndividualSpaceAllocation, SpaceAllocation, SpaceUsage + + return SpaceUsage(used=used, allocation=SpaceAllocation.individual( + IndividualSpaceAllocation(allocated=allocated))) + + +class TestSpaceUsageBytes: + def test_team_allocation_uses_team_totals(self): + assert dg.space_usage_bytes(_team_usage(6 * TIB, 6 * TIB)) == (6 * TIB, 6 * TIB) + + def test_individual_allocation(self): + assert dg.space_usage_bytes(_individual_usage(10, 100)) == (10, 100) + + +class TestEvaluateQuota: + @pytest.mark.parametrize("used,allocated,was_alerting,expected", [ + (94, 100, False, "none"), + (95, 100, False, "alert"), # threshold is inclusive + (100, 100, False, "alert"), + (99, 100, True, "none"), # still high: don't re-page every run + (90, 100, True, "recovered"), + (50, 100, False, "none"), + (10, 0, False, "none"), # unknown allocation + ]) + def test_threshold_logic(self, used, allocated, was_alerting, expected): + assert dg.evaluate_quota(used, allocated, was_alerting) == expected + + +class _FakeDoc: + def __init__(self, data=None): + self.data = data + self.writes = [] + + def get(self): + snap = Mock() + snap.exists = self.data is not None + snap.to_dict.return_value = dict(self.data or {}) + return snap + + def set(self, data, merge=False): + self.writes.append(data) + self.data = {**(self.data or {}), **data} + + +def _db_with(doc): + db = MagicMock() + db.collection.return_value.document.return_value = doc + return db + + +def _dropbox(usage): + svc = MagicMock() + svc.is_configured = True + svc.client.users_get_space_usage.return_value = usage + return svc + + +class TestCheckDropboxSpaceUsage: + def test_alerts_once_when_crossing_threshold(self): + doc = _FakeDoc() + svc = _dropbox(_team_usage(int(6.001 * TIB), 6 * TIB)) + with patch("backend.services.ops_alerts.send_ops_alert", return_value=True) as send: + first = dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=svc) + second = dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=svc) + + assert first["action"] == "alert" and first["alerting"] is True + assert second["action"] == "none" + send.assert_called_once() + assert "Dropbox almost FULL" in send.call_args[0][0] + assert doc.data["alerting"] is True + + def test_failed_send_does_not_latch(self): + doc = _FakeDoc() + svc = _dropbox(_team_usage(99, 100)) + with patch("backend.services.ops_alerts.send_ops_alert", return_value=False) as send: + dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=svc) + dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=svc) + assert send.call_count == 2 + assert doc.data["alerting"] is False + + def test_rearms_after_recovery(self): + doc = _FakeDoc({"alerting": True}) + with patch("backend.services.ops_alerts.send_ops_alert", return_value=True) as send: + r = dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=_dropbox(_team_usage(50, 100))) + assert r["action"] == "recovered" + assert doc.data["alerting"] is False + r = dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=_dropbox(_team_usage(97, 100))) + assert r["action"] == "alert" + send.assert_called_once() + + def test_below_threshold_no_alert(self): + doc = _FakeDoc() + with patch("backend.services.ops_alerts.send_ops_alert") as send: + r = dg.check_dropbox_space_usage(db=_db_with(doc), dropbox_service=_dropbox(_team_usage(10, 100))) + assert r == {"status": "ok", "used_pct": 10.0, "action": "none", "alerting": False} + send.assert_not_called() + + def test_skips_when_dropbox_not_configured(self): + svc = MagicMock() + svc.is_configured = False + assert dg.check_dropbox_space_usage(db=MagicMock(), dropbox_service=svc)["status"] == "skipped" + + def test_api_error_never_raises(self): + svc = MagicMock() + svc.is_configured = True + svc.client.users_get_space_usage.side_effect = RuntimeError("api down") + assert dg.check_dropbox_space_usage(db=MagicMock(), dropbox_service=svc)["status"] == "error" diff --git a/backend/tests/test_made_for_you.py b/backend/tests/test_made_for_you.py index 1899d84c..715b1aa4 100644 --- a/backend/tests/test_made_for_you.py +++ b/backend/tests/test_made_for_you.py @@ -2672,3 +2672,92 @@ class TestJobManagerCompletionEmail: # No ownership transfer without customer_email job_manager.firestore.update_job.assert_not_called() + + +class TestCompletionEmailDeliveryGuard: + """NOMAD-1800: a job meant for Dropbox delivery with no Dropbox link must not + send the customer a completion email with a placeholder link.""" + + @pytest.fixture + def job_manager(self): + with patch('backend.services.job_manager.FirestoreService'), \ + patch('backend.services.job_manager.StorageService'): + from backend.services.job_manager import JobManager + manager = JobManager() + manager.firestore = MagicMock() + return manager + + def _job(self, **overrides): + base = dict( + job_id="dd892383", + status=JobStatus.COMPLETE, + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + user_email="madeforyou@nomadkaraoke.com", + artist="Test Artist", + title="Test Song", + made_for_you=True, + customer_email="customer@example.com", + dropbox_path="/Karaoke/Tracks-Organized", + brand_prefix="NOMAD", + state_data={ + 'youtube_url': 'https://youtube.com/watch?v=1', + 'brand_code': 'NOMAD-1800', + 'distribution_warnings': [ + "Dropbox upload failed: UploadWriteFailed(reason=WriteError('insufficient_space'))" + ], + }, + ) + base.update(overrides) + return Job(**base) + + def test_holds_email_when_dropbox_link_missing(self, job_manager): + job = self._job() + mock_ns = MagicMock() + with patch('backend.services.job_notification_service.get_job_notification_service', return_value=mock_ns), \ + patch('backend.services.ops_alerts.send_throttled_ops_alert', return_value=True) as alert: + job_manager._schedule_completion_email(job) + import time + time.sleep(0.1) + + mock_ns.send_job_completion_email.assert_not_called() + updates = [c[0][1] for c in job_manager.firestore.update_job.call_args_list] + # Ownership still transfers (customer can see the job in their account) + assert {'user_email': 'customer@example.com'} in updates + held = [u for u in updates if 'state_data.completion_email_held' in u] + assert len(held) == 1 + marker = held[0]['state_data.completion_email_held'] + assert marker['reason_code'] == 'dropbox_full' + assert marker['recipient'] == 'customer@example.com' + alert.assert_called_once() + assert "Dropbox is FULL" in alert.call_args[0][1] + + def test_sends_email_when_dropbox_link_present(self, job_manager): + job = self._job(state_data={'dropbox_link': 'https://dropbox.com/x', 'brand_code': 'NOMAD-1800'}) + mock_ns = MagicMock() + mock_ns.send_job_completion_email = AsyncMock(return_value=True) + with patch('backend.services.job_notification_service.get_job_notification_service', return_value=mock_ns), \ + patch('backend.services.ops_alerts.send_throttled_ops_alert') as alert: + job_manager._schedule_completion_email(job) + import time + time.sleep(0.2) + + mock_ns.send_job_completion_email.assert_called_once() + assert mock_ns.send_job_completion_email.call_args.kwargs['dropbox_url'] == 'https://dropbox.com/x' + alert.assert_not_called() + + def test_sends_email_for_job_without_dropbox_delivery(self, job_manager): + """Tenant / no-Dropbox jobs have no link by design and still get their email.""" + job = self._job(dropbox_path=None, brand_prefix=None, made_for_you=False, + customer_email=None, tenant_id="acme", state_data={}) + mock_ns = MagicMock() + mock_ns.send_job_completion_email = AsyncMock(return_value=True) + with patch('backend.services.job_notification_service.get_job_notification_service', return_value=mock_ns), \ + patch('backend.services.ops_alerts.send_throttled_ops_alert') as alert: + job_manager._schedule_completion_email(job) + import time + time.sleep(0.2) + + mock_ns.send_job_completion_email.assert_called_once() + alert.assert_not_called() + job_manager.firestore.update_job.assert_not_called() diff --git a/backend/tests/test_template_service.py b/backend/tests/test_template_service.py index e57846ab..fc26523e 100644 --- a/backend/tests/test_template_service.py +++ b/backend/tests/test_template_service.py @@ -112,8 +112,65 @@ class TestJobCompletionTemplate: result = service.render_job_completion() assert "there" in result # Default name - assert "[YouTube URL not available]" in result - assert "[Dropbox URL not available]" in result + # Missing links never render as placeholders (NOMAD-1800) โ€” the whole + # section is dropped instead. + assert "not available" not in result + assert "dropbox folder" not in result + assert "published to YouTube" not in result + assert "{" not in result + + def test_render_job_completion_missing_dropbox_drops_section_only(self): + """No Dropbox link: the Dropbox paragraph (incl. file list) goes, YouTube stays.""" + service = TemplateService() + + with patch.object(service, "_fetch_template_from_gcs", return_value=None): + result = service.render_job_completion( + youtube_url="https://youtube.com/watch?v=1", + feedback_url="https://example.com/feedback", + ) + + assert "https://youtube.com/watch?v=1" in result + assert "dropbox folder" not in result + assert "(Final Karaoke Lossless).mkv" not in result + assert "[Dropbox URL not available]" not in result + assert "https://example.com/feedback" in result + assert "Let me know if anything isn't perfect" in result + + def test_render_job_completion_missing_youtube_keeps_dropbox(self): + """Public job with no YouTube link and nothing queued: YouTube paragraph goes.""" + service = TemplateService() + + with patch.object(service, "_fetch_template_from_gcs", return_value=None): + result = service.render_job_completion(dropbox_url="https://dropbox.com/folder/abc") + + assert "https://dropbox.com/folder/abc" in result + assert "dropbox folder" in result + assert "published to YouTube" not in result + assert "[YouTube URL not available]" not in result + + def test_render_job_completion_missing_dropbox_localized(self): + """Localized default template also drops the Dropbox section.""" + service = TemplateService() + + with patch.object(service, "_fetch_template_from_gcs", return_value=None): + result = service.render_job_completion( + youtube_url="https://youtube.com/watch?v=1", locale="de" + ) + + assert "https://youtube.com/watch?v=1" in result + assert "Dropbox" not in result + assert "{" not in result + + def test_render_job_completion_missing_dropbox_unknown_template(self): + """A custom (GCS) template with unknown wording drops just the placeholder line.""" + service = TemplateService() + custom = "Hi {name}\n\nFiles:\n{dropbox_url}\n\nVideo: {youtube_url}\nBye" + + with patch.object(service, "_fetch_template_from_gcs", return_value=custom): + result = service.render_job_completion(youtube_url="https://yt/1") + + assert "not available" not in result + assert "Video: https://yt/1" in result def test_render_job_completion_no_feedback_url(self): """Test rendering job completion without feedback URL removes section.""" diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index 9ac7c084..3b1dc8cd 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -1382,7 +1382,9 @@ async def _handle_native_distribution( except ImportError as e: job_log.warning(f"Dropbox SDK not installed: {e}") except Exception as e: - job_log.error(f"Native Dropbox upload failed: {e}", exc_info=True) + # WARNING, not ERROR: delivery_guard posts the actionable alert + # (held completion email) โ€” one per job, not one per pass. + job_log.warning(f"Native Dropbox upload failed: {e}", exc_info=True) result.setdefault('distribution_warnings', []).append(f"Dropbox upload failed: {e}") # Don't fail the job - distribution is optional diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index 614f04cc..dcd28ae6 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -1016,7 +1016,9 @@ class VideoWorkerOrchestrator: self.job_log.info(f"Dropbox upload complete: {remote_folder}") except Exception as e: - self.job_log.error(f"Dropbox upload failed: {e}") + # WARNING, not ERROR: the customer-impacting outcome (held completion + # email) gets its own actionable #alerts message from delivery_guard. + self.job_log.warning(f"Dropbox upload failed: {e}") self.result.distribution_warnings.append(f"Dropbox upload failed: {e}") # Don't fail the pipeline - Dropbox is optional diff --git a/docs/API.md b/docs/API.md index b454662a..49b6117e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2479,6 +2479,12 @@ Content-Type: application/json Sends the job completion email to the specified address. When `cc_admin` is true (default), CCs gen@nomadkaraoke.com. +Returns **409** if the job was meant to deliver to Dropbox but has no `state_data.dropbox_link` +(e.g. the Dropbox was full). Fix the Dropbox upload first, or pass +`"allow_missing_dropbox_link": true` to send anyway (the Dropbox section is then left out of the +email; a placeholder link is never rendered). A successful send clears +`state_data.completion_email_held`. + Response: ```json { diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 46240237..c9a6fbe8 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -436,6 +436,32 @@ Do **not** reset to `instrumental_selected` โ€” that triggers the final video wo --- +## Held completion email (Dropbox delivery failed) + +**Signal:** #alerts "๐Ÿ”ด Customer delivery HELD โ€” Dropbox upload failed" (from +`backend/services/delivery_guard.py`), or "๐Ÿ”ด Dropbox almost FULL" from the hourly quota check +(rides on `/api/internal/process-stale-reviews`; fires once at โ‰ฅ95% used and re-arms after usage +drops back below). + +**What happened:** the job was meant to deliver to Dropbox (`dropbox_path` + `brand_prefix` on its +effective distribution settings) but finished with no `state_data.dropbox_link`. Instead of emailing +the customer a delivery email with no files link (NOMAD-1800), the job went COMPLETE, the email +was held, and `state_data.completion_email_held = {reason_code, reason, recipient, at}` was set. +For made-for-you orders ownership still transfers to the customer. `reason_code` is +`dropbox_full` (`insufficient_space`), `dropbox_upload_failed`, or `dropbox_link_missing`. + +**Recovery (manual):** +1. If `dropbox_full`: free space in the business Dropbox (`rclone about` shows usage) or raise + the quota. +2. Upload the job's outputs (GCS `jobs/{job_id}/finals/`, `stems/`, `lyrics/`) to its Dropbox + folder `{dropbox_path}/{brand_code} - {Artist} - {Title}`, create a share link, and set + `state_data.dropbox_link` on the job. (An admin re-render also re-uploads to Dropbox, but it + deletes and re-uploads the YouTube video too, so the YouTube URL changes: only use it for + private jobs.) +3. `POST /api/admin/jobs/{job_id}/send-completion-email` with the recipient from + `completion_email_held.recipient`. It returns 409 while the link is still missing + (`allow_missing_dropbox_link: true` overrides). A successful send clears the held marker. + ## Google Drive uploads missing (gdrive_files empty) **Symptoms:** Jobs complete successfully but `state_data.gdrive_files` is empty `{}`. Cloud Run logs show `BrokenPipeError` or `SSL: UNEXPECTED_EOF_WHILE_READING`. diff --git a/pyproject.toml b/pyproject.toml index 8629ca94..bb5c1a53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.281.0" +version = "0.282.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"