diff --git a/.claude/plans/ai-credit-gating.plan.md b/.claude/plans/ai-credit-gating.plan.md new file mode 100644 index 00000000..f187e29b --- /dev/null +++ b/.claude/plans/ai-credit-gating.plan.md @@ -0,0 +1,151 @@ +# Plan: AI-Powered Credit Gating + +**Created:** 2026-03-23 +**Branch:** feat/sess-20260323-1716-ai-credit-gating +**Status:** Implemented + +## Overview + +Replace the automatic welcome/feedback credit grants with an AI-powered evaluation step. When a user triggers a credit grant (email verification or feedback submission), instead of immediately granting credits, we collect abuse signals and ask Gemini to make a grant/deny decision. Suspected abusers get a friendly rejection email (CC'd to Andrew) instead of free credits. + +## Requirements + +- [ ] Welcome credit grants go through AI evaluation before granting +- [ ] Feedback credit grants go through AI evaluation before granting +- [ ] AI evaluates: fingerprint correlations, IP correlations, IP geo info, user agent, signup recency patterns +- [ ] Approved users get credits as before (seamless experience) +- [ ] Rejected users get a friendly-but-firm email explaining the situation, CC'd to andrew@beveridge.uk +- [ ] Rejection email invites them to reply if it's a mistake +- [ ] Decision is logged to Firestore for audit trail +- [ ] Fail-open: if AI evaluation fails, grant credits (don't punish users for our errors) +- [ ] Admin can override decisions via existing admin credit grant endpoint + +## Technical Approach + +### Architecture + +``` +User verifies email / submits feedback + ↓ +Collect abuse signals (sync, fast): + - User's fingerprint + IP + - Other accounts with same fingerprint (from correlations) + - Other accounts with same IP + - IP geolocation info + - Recent signup count from same IP/fingerprint (last 24h) + - User agent string + - For feedback: the feedback content itself + ↓ +Call Gemini (vertexai/gemini-2.5-pro-preview) with structured prompt: + - Present all signals + - Ask for JSON response: {decision: "grant" | "deny", reasoning: string, confidence: number} + ↓ +If "grant": grant credits normally, log decision +If "deny": skip credit grant, send rejection email, log decision +If AI call fails: grant credits (fail-open), log error +``` + +### Key Design Decisions + +1. **Two-phase verification**: The magic link verify endpoint returns immediately with session + a `credit_status: "evaluating"` field. The frontend shows a "preparing your account" loading screen, then polls/waits for the evaluation result via a new lightweight endpoint. This keeps the auth flow snappy while the AI evaluation runs in the background of the same request or via a short follow-up call. The frontend then shows either a celebratory "2 free credits!" interstitial or a "sorry, couldn't grant credits" interstitial before navigating to the dashboard. + +2. **Fail-open**: If Gemini is down or returns an error, we grant credits. Better to let an occasional abuser through than block legitimate users. + +3. **Use google-genai directly**: The project already has `langchain-google-genai` installed. We can use the simpler `google.generativeai` library directly for a focused prompt/response, or use the existing LangChain setup. We'll use `google.generativeai` for simplicity since this is a single-turn evaluation, not an agentic workflow. + +4. **Structured JSON output**: Ask Gemini for structured output with decision, reasoning, and confidence to make decisions parseable and auditable. + +5. **Log all decisions**: Store every evaluation in a `credit_evaluations` Firestore collection for audit trail and to tune the prompt over time. + +## Implementation Steps + +### Step 1: Add google-generativeai dependency +- Add `google-generativeai` to pyproject.toml (lightweight, direct Gemini access) +- Verify it works with Vertex AI auth in the nomadkaraoke project + +### Step 2: Create credit evaluation service +- New file: `backend/services/credit_evaluation_service.py` +- `CreditEvaluationService` class with: + - `evaluate_credit_grant(email, grant_type, feedback_content=None) -> CreditEvaluation` + - Collects all abuse signals by calling existing services (user_service, ip_geolocation_service) + - Builds prompt with signals + - Calls Gemini for decision + - Logs decision to Firestore + - Returns structured result + +### Step 3: Build the Gemini prompt +- System prompt explaining the role: "You are an anti-abuse evaluator for a karaoke video service..." +- Present all signals in a structured format +- Request JSON response: `{decision: "grant"|"deny", reasoning: "...", confidence: 0.0-1.0}` +- Include clear guidelines: same fingerprint across accounts = very suspicious, same IP = somewhat suspicious, etc. + +### Step 4: Create rejection email template +- Add `send_credit_denied_email(email, reasoning)` to EmailService +- Friendly-but-firm tone +- CC andrew@beveridge.uk +- Invite reply if it's a mistake +- Use existing email template infrastructure (branding, styling) + +### Step 5: Integrate into welcome credit flow +- Modify `grant_welcome_credits_if_eligible()` in user_service.py +- Before granting: call `CreditEvaluationService.evaluate_credit_grant(email, "welcome")` +- If denied: send rejection email, return False +- If error: log warning, grant anyway (fail-open) + +### Step 6: Integrate into feedback credit flow +- Modify feedback endpoint in users.py +- Before granting credits: evaluate with feedback content included +- If denied: return different response message, send rejection email +- If error: grant anyway + +### Step 7: Add Firestore collection for audit trail +- Collection: `credit_evaluations` +- Document fields: email, grant_type, decision, reasoning, confidence, signals_snapshot, model_used, evaluated_at +- TTL: keep indefinitely (small volume, high audit value) + +### Step 8: Tests +- Unit tests for signal collection +- Unit tests for prompt building +- Unit tests for decision parsing (grant/deny/error cases) +- Integration test for the evaluation service with mocked Gemini +- Integration test for welcome credit flow with evaluation +- Integration test for feedback credit flow with evaluation +- Test fail-open behavior + +### Step 9: Configuration +- Add env vars: `CREDIT_EVAL_MODEL` (default: "gemini-2.5-pro-preview"), `CREDIT_EVAL_ENABLED` (default: true) +- Add to config.py + +## Files to Create/Modify + +| File | Action | Description | +|------|--------|-------------| +| `pyproject.toml` | Modify | Add google-generativeai dependency | +| `backend/config.py` | Modify | Add credit evaluation config vars | +| `backend/services/credit_evaluation_service.py` | Create | New service: collect signals, call Gemini, log decision | +| `backend/services/user_service.py` | Modify | Integrate evaluation into welcome credit grant | +| `backend/api/routes/users.py` | Modify | Integrate evaluation into feedback credit grant | +| `backend/services/email_service.py` | Modify | Add credit denied email template | +| `backend/tests/test_credit_evaluation.py` | Create | Tests for evaluation service | +| `backend/tests/test_anti_abuse.py` | Modify | Update welcome credit tests for evaluation | +| `frontend/app/auth/verify/page.tsx` | Modify | Add "preparing account" loading + credit result interstitial | +| `frontend/lib/api.ts` | Modify | Add credit evaluation status endpoint | + +## Testing Strategy + +- **Unit tests**: Signal collection, prompt building, response parsing, fail-open behavior +- **Integration tests**: Full evaluation flow with mocked Gemini responses +- **E2E tests**: Manual verification in production with test accounts +- **Edge cases**: Gemini timeout, malformed response, missing signals, first-ever user (no correlations) + +## Open Questions + +1. **Gemini model**: Using `gemini-3.1-pro-preview` as specified — already available in the nomadkaraoke GCP project. + +2. **Latency UX**: Frontend shows a friendly "preparing your account" loading screen while evaluation runs. Then transitions to either a celebratory "2 free credits!" message (with gentle reminder not to abuse) or a "sorry, couldn't grant free credits" message with a buy credits CTA. User clicks through to dashboard from either. + +3. **Should we also gate the existing rate limiter?** Currently `is_signup_rate_limited()` silently blocks at 2 accounts per IP/fingerprint per 24h. The AI evaluation is complementary — rate limiter blocks obvious rapid abuse, AI evaluation catches more subtle patterns. + +## Rollback Plan + +Set `CREDIT_EVAL_ENABLED=false` in Cloud Run env vars → immediate rollback to auto-grant behavior. No data model changes needed. The evaluation service is purely additive. diff --git a/backend/api/routes/users.py b/backend/api/routes/users.py index e2135487..762c6bd2 100644 --- a/backend/api/routes/users.py +++ b/backend/api/routes/users.py @@ -297,13 +297,17 @@ async def verify_magic_link( if not success or not user: raise HTTPException(status_code=401, detail=message) - # Grant welcome credits on first verification (not at account creation) + # Grant welcome credits on first verification (with AI abuse evaluation) credits_granted = 0 - if user_service.grant_welcome_credits_if_eligible(user.email): + credit_status = "not_applicable" # returning user, not first login + granted, credit_status = user_service.grant_welcome_credits_if_eligible(user.email) + if granted: credits_granted = user_service.NEW_USER_FREE_CREDITS logger.info(f"Granted {credits_granted} welcome credits to {_mask_email(user.email)}") # Refresh user to get updated credit balance user = user_service.get_user(user.email) + elif credit_status == "denied": + logger.info(f"Welcome credits denied for {_mask_email(user.email)}") # Create session with tenant context and device fingerprint from the magic link magic_link_fingerprint = magic_link_data.get('device_fingerprint') if magic_link_doc.exists else None @@ -346,6 +350,7 @@ async def verify_magic_link( message="Successfully signed in", tenant_subdomain=tenant_subdomain, credits_granted=credits_granted, + credit_status=credit_status, ) @@ -1210,7 +1215,58 @@ async def submit_user_feedback( has_submitted_feedback=True, ) - # Grant 2 credits + # AI evaluation before granting feedback credits + credits_granted = 0 + feedback_content = { + "overall_rating": request.overall_rating, + "what_went_well": request.what_went_well, + "what_could_improve": request.what_could_improve, + "additional_comments": request.additional_comments, + } + try: + from backend.services.credit_evaluation_service import get_credit_evaluation_service + eval_service = get_credit_evaluation_service() + evaluation = eval_service.evaluate(user.email, "feedback", feedback_content) + + if evaluation.decision == "deny": + logger.info(f"Feedback credits denied for {_mask_email(user.email)}: {evaluation.reasoning}") + try: + from backend.services.email_service import get_email_service + get_email_service().send_credit_denied_email(user.email, "feedback") + except Exception: + logger.exception(f"Failed to send credit denied email to {_mask_email(user.email)}") + return UserFeedbackResponse( + status="success", + message="Thank you for your feedback! We appreciate your input.", + credits_granted=0, + ) + + if evaluation.decision == "pending_review": + logger.info(f"Feedback credits pending review for {_mask_email(user.email)}: {evaluation.reasoning}") + try: + from backend.services.email_service import get_email_service + get_email_service().send_credit_review_needed_email(user.email, "feedback", evaluation.reasoning) + except Exception: + logger.exception(f"Failed to send review needed email for {_mask_email(user.email)}") + return UserFeedbackResponse( + status="success", + message="Thank you for your feedback! Our team will review your account shortly.", + credits_granted=0, + ) + except Exception: + logger.exception(f"Credit evaluation failed for {_mask_email(user.email)} — pending review (fail-closed)") + try: + from backend.services.email_service import get_email_service + get_email_service().send_credit_review_needed_email(user.email, "feedback", "Evaluation error") + except Exception: + pass + return UserFeedbackResponse( + status="success", + message="Thank you for your feedback! Our team will review your account shortly.", + credits_granted=0, + ) + + # Grant 2 credits (passed evaluation) credits_granted = 2 user_service.add_credits( email=user.email, diff --git a/backend/config.py b/backend/config.py index b57489a9..015a34e9 100644 --- a/backend/config.py +++ b/backend/config.py @@ -57,6 +57,12 @@ class Settings(BaseSettings): # use uncorrected transcription - human review will fix any issues. agentic_correction_timeout_seconds: int = int(os.getenv("AGENTIC_CORRECTION_TIMEOUT_SECONDS", "180")) + # AI Credit Evaluation (anti-abuse gating for free credits) + # When enabled, welcome and feedback credit grants are evaluated by Gemini + # before granting. Fail-open: if evaluation fails, credits are granted anyway. + credit_eval_enabled: bool = os.getenv("CREDIT_EVAL_ENABLED", "true").lower() in ("true", "1", "yes") + credit_eval_model: str = os.getenv("CREDIT_EVAL_MODEL", "gemini-3.1-pro-preview") + # Cloud Tasks (for scalable worker coordination) # When enabled, workers are triggered via Cloud Tasks for guaranteed delivery # When disabled (default), workers are triggered via direct HTTP (for development) diff --git a/backend/models/user.py b/backend/models/user.py index 696f889b..3570474a 100644 --- a/backend/models/user.py +++ b/backend/models/user.py @@ -180,6 +180,7 @@ class VerifyMagicLinkResponse(BaseModel): message: str tenant_subdomain: Optional[str] = None credits_granted: int = 0 + credit_status: str = "not_applicable" # "granted", "denied", "pending_review", "already_granted", "not_applicable" class UserPublic(BaseModel): diff --git a/backend/services/credit_evaluation_service.py b/backend/services/credit_evaluation_service.py new file mode 100644 index 00000000..bd6bd0e8 --- /dev/null +++ b/backend/services/credit_evaluation_service.py @@ -0,0 +1,428 @@ +""" +AI-powered credit evaluation service. + +Evaluates whether to grant free credits (welcome or feedback) by collecting +abuse signals and asking Gemini for a grant/deny decision. Fail-open: if +the AI call fails for any reason, credits are granted. +""" +import json +import logging +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional + +from backend.config import get_settings + +logger = logging.getLogger(__name__) + +CREDIT_EVALUATIONS_COLLECTION = "credit_evaluations" + + +@dataclass +class CreditEvaluation: + """Result of an AI credit evaluation.""" + decision: str # "grant" or "deny" + reasoning: str + confidence: float + error: Optional[str] = None # Set if evaluation failed (decision defaults to "grant") + + +SYSTEM_PROMPT = """You are an anti-abuse evaluator for Nomad Karaoke, a web service that creates professional karaoke videos. Each job costs the service real money in API credits and cloud compute. + +New users get 2 free welcome credits to try the service. Users who complete 2 jobs and submit feedback get 2 more credits. Some users abuse this by creating multiple accounts to get unlimited free karaoke generation. + +Your job: Given the signals below, decide whether to GRANT or DENY free credits to this user. + +## Decision Guidelines + +**DENY if any of these are true:** +- The user's device fingerprint matches other accounts (strong signal of multi-accounting) +- The user's signup IP matches other accounts AND those accounts also have suspicious patterns (e.g., no spend, same user agent) +- There's a clear pattern of account creation from the same device/network to farm free credits + +**GRANT if:** +- This appears to be a genuine new user with no suspicious correlations +- The IP match is explainable (e.g., different fingerprints suggest different actual users on a shared network like a school or office) +- There's insufficient evidence to confidently flag abuse + +**When in doubt, GRANT.** False positives (blocking real users) are worse than false negatives (letting an occasional abuser through). We'd rather lose a few dollars than alienate a real customer. + +## Response Format + +Respond with ONLY a JSON object, no other text: +{"decision": "grant" or "deny", "reasoning": "Brief explanation", "confidence": 0.0 to 1.0} +""" + + +def _build_user_prompt( + email: str, + grant_type: str, + user_data: dict, + fingerprint_matches: list, + ip_matches: list, + ip_geo: Optional[dict], + recent_signups_ip: int, + recent_signups_fp: int, + user_agent: Optional[str], + feedback_content: Optional[dict] = None, +) -> str: + """Build the user prompt with all abuse signals.""" + lines = [ + f"## Credit Grant Request", + f"- **Email:** {email}", + f"- **Grant type:** {grant_type} credits", + f"- **User's device fingerprint:** {user_data.get('device_fingerprint', 'unknown')}", + f"- **User's signup IP:** {user_data.get('signup_ip', 'unknown')}", + f"- **User agent:** {user_agent or 'unknown'}", + f"- **Account created:** {user_data.get('created_at', 'unknown')}", + f"- **Jobs created:** {user_data.get('total_jobs_created', 0)}", + f"- **Jobs completed:** {user_data.get('total_jobs_completed', 0)}", + f"- **Total spent (real money):** ${user_data.get('total_spent', 0) / 100:.2f}", + f"- **Current credits:** {user_data.get('credits', 0)}", + "", + ] + + # IP geolocation + if ip_geo and ip_geo.get("status") == "success": + lines.extend([ + f"## IP Geolocation", + f"- **Country:** {ip_geo.get('country', 'unknown')} ({ip_geo.get('country_code', '')})", + f"- **City/Region:** {ip_geo.get('city', '')}, {ip_geo.get('region', '')}", + f"- **ISP:** {ip_geo.get('isp', 'unknown')}", + f"- **Organization:** {ip_geo.get('org', 'unknown')}", + f"- **ASN:** {ip_geo.get('as_number', '')} {ip_geo.get('as_name', '')}", + "", + ]) + + # Recent signup patterns + lines.extend([ + f"## Recent Signup Patterns (last 24 hours)", + f"- **New accounts from same IP:** {recent_signups_ip}", + f"- **New accounts from same fingerprint:** {recent_signups_fp}", + "", + ]) + + # Fingerprint correlations + if fingerprint_matches: + lines.append("## Other Accounts with Same Device Fingerprint (RED FLAG)") + for match in fingerprint_matches: + lines.append( + f"- {match.get('email', '?')} | " + f"Jobs: {match.get('total_jobs_created', 0)} | " + f"Spent: ${match.get('total_spent', 0) / 100:.2f} | " + f"Created: {match.get('created_at', '?')}" + ) + lines.append("") + else: + lines.extend(["## Other Accounts with Same Device Fingerprint", "None found.", ""]) + + # IP correlations + if ip_matches: + lines.append("## Other Accounts from Same Signup IP") + for match in ip_matches: + lines.append( + f"- {match.get('email', '?')} | " + f"Fingerprint: {match.get('device_fingerprint', 'unknown')} | " + f"Jobs: {match.get('total_jobs_created', 0)} | " + f"Spent: ${match.get('total_spent', 0) / 100:.2f} | " + f"Created: {match.get('created_at', '?')}" + ) + lines.append("") + else: + lines.extend(["## Other Accounts from Same Signup IP", "None found.", ""]) + + # Feedback content (if applicable) + if feedback_content: + lines.extend([ + "## Feedback Submitted", + f"- **Overall rating:** {feedback_content.get('overall_rating', '?')}/5", + f"- **What went well:** {feedback_content.get('what_went_well', 'N/A')[:200]}", + f"- **What could improve:** {feedback_content.get('what_could_improve', 'N/A')[:200]}", + f"- **Additional comments:** {feedback_content.get('additional_comments', 'N/A')[:200]}", + "", + ]) + + lines.append("Based on these signals, should we GRANT or DENY free credits to this user?") + return "\n".join(lines) + + +def _parse_gemini_response(response_text: str) -> CreditEvaluation: + """Parse Gemini's JSON response into a CreditEvaluation.""" + try: + # Strip markdown code fences if present + text = response_text.strip() + if text.startswith("```"): + text = text.split("\n", 1)[1] if "\n" in text else text[3:] + if text.endswith("```"): + text = text.rsplit("```", 1)[0] + text = text.strip() + if text.startswith("json"): + text = text[4:].strip() + + data = json.loads(text) + decision = data.get("decision", "").lower() + if decision not in ("grant", "deny"): + decision = "pending_review" # fail-closed: unknown decision → manual review + + return CreditEvaluation( + decision=decision, + reasoning=str(data.get("reasoning", "No reasoning provided")), + confidence=float(data.get("confidence", 0.5)), + ) + except (json.JSONDecodeError, ValueError, TypeError) as e: + logger.warning(f"Failed to parse Gemini response: {e}. Response: {response_text[:200]}") + return CreditEvaluation( + decision="pending_review", + reasoning="Failed to parse AI response — pending manual review", + confidence=0.0, + error=f"Parse error: {e}", + ) + + +class CreditEvaluationService: + """Evaluates credit grant requests using Gemini AI.""" + + def __init__(self): + self.settings = get_settings() + + def _collect_signals(self, email: str) -> dict: + """Collect all abuse signals for evaluation.""" + from backend.services.user_service import get_user_service + from backend.services.ip_geolocation_service import get_ip_geolocation_service + + user_service = get_user_service() + user = user_service.get_user(email) + + if not user: + return { + "user_data": {}, + "fingerprint_matches": [], + "ip_matches": [], + "ip_geo": None, + "recent_signups_ip": 0, + "recent_signups_fp": 0, + "user_agent": None, + } + + user_data = { + "email": user.email, + "device_fingerprint": user.device_fingerprint, + "signup_ip": user.signup_ip, + "credits": user.credits, + "total_jobs_created": user.total_jobs_created, + "total_jobs_completed": user.total_jobs_completed, + "total_spent": user.total_spent, + "created_at": str(user.created_at), + } + + # Find other accounts with same fingerprint + fingerprint_matches = [] + if user.device_fingerprint: + fp_users = user_service.find_users_by_fingerprint(user.device_fingerprint) + fingerprint_matches = [ + { + "email": u.email, + "total_jobs_created": u.total_jobs_created, + "total_spent": u.total_spent, + "created_at": str(u.created_at), + } + for u in fp_users if u.email != email + ] + + # Find other accounts from same IP + ip_matches = [] + if user.signup_ip: + ip_users = user_service.find_users_by_signup_ip(user.signup_ip) + ip_matches = [ + { + "email": u.email, + "device_fingerprint": u.device_fingerprint, + "total_jobs_created": u.total_jobs_created, + "total_spent": u.total_spent, + "created_at": str(u.created_at), + } + for u in ip_users if u.email != email + ] + + # IP geolocation + ip_geo = None + if user.signup_ip: + try: + geo_service = get_ip_geolocation_service() + ip_geo = geo_service.lookup_ip(user.signup_ip) + except Exception: + pass + + # Recent signup counts + recent_signups_ip = 0 + recent_signups_fp = 0 + if user.signup_ip: + recent_signups_ip = user_service.count_recent_signups_from_ip(user.signup_ip) + if user.device_fingerprint: + recent_signups_fp = user_service.count_recent_signups_from_fingerprint(user.device_fingerprint) + + # User agent from latest session + user_agent = None + try: + from google.cloud.firestore_v1 import FieldFilter + from google.cloud import firestore as fs + sessions = ( + user_service.db.collection("sessions") + .where(filter=FieldFilter("user_email", "==", email)) + .order_by("created_at", direction=fs.Query.DESCENDING) + .limit(1) + .stream() + ) + for session_doc in sessions: + user_agent = session_doc.to_dict().get("user_agent") + except Exception: + pass + + return { + "user_data": user_data, + "fingerprint_matches": fingerprint_matches, + "ip_matches": ip_matches, + "ip_geo": ip_geo, + "recent_signups_ip": recent_signups_ip, + "recent_signups_fp": recent_signups_fp, + "user_agent": user_agent, + } + + def _call_gemini(self, prompt: str) -> str: + """Call Gemini and return the response text.""" + import google.generativeai as genai + + model = genai.GenerativeModel( + model_name=self.settings.credit_eval_model, + system_instruction=SYSTEM_PROMPT, + ) + response = model.generate_content(prompt) + return response.text + + def _log_evaluation( + self, + email: str, + grant_type: str, + evaluation: CreditEvaluation, + signals: dict, + ): + """Log the evaluation decision to Firestore for audit trail.""" + try: + from google.cloud import firestore as fs + db = fs.Client(project=self.settings.google_cloud_project) + doc_id = str(uuid.uuid4()) + db.collection(CREDIT_EVALUATIONS_COLLECTION).document(doc_id).set({ + "id": doc_id, + "email": email, + "grant_type": grant_type, + "decision": evaluation.decision, + "reasoning": evaluation.reasoning, + "confidence": evaluation.confidence, + "error": evaluation.error, + "model": self.settings.credit_eval_model, + "signals_snapshot": { + "fingerprint_match_count": len(signals.get("fingerprint_matches", [])), + "ip_match_count": len(signals.get("ip_matches", [])), + "recent_signups_ip": signals.get("recent_signups_ip", 0), + "recent_signups_fp": signals.get("recent_signups_fp", 0), + "user_ip": signals.get("user_data", {}).get("signup_ip"), + "user_fingerprint": signals.get("user_data", {}).get("device_fingerprint"), + }, + "evaluated_at": datetime.now(timezone.utc).isoformat(), + }) + except Exception: + logger.exception(f"Failed to log credit evaluation for {email}") + + def evaluate( + self, + email: str, + grant_type: str, + feedback_content: Optional[dict] = None, + ) -> CreditEvaluation: + """ + Evaluate whether to grant free credits. + + Args: + email: User's email + grant_type: "welcome" or "feedback" + feedback_content: Feedback form data (for feedback grants only) + + Returns: + CreditEvaluation with decision, reasoning, confidence. + Fail-closed: returns "pending_review" on any error (admin notified). + """ + if not self.settings.credit_eval_enabled: + return CreditEvaluation( + decision="grant", + reasoning="Credit evaluation disabled — auto-granting", + confidence=1.0, + ) + + try: + # Collect signals + signals = self._collect_signals(email) + + # Quick-grant: if no correlations found at all, skip AI call + if ( + not signals["fingerprint_matches"] + and not signals["ip_matches"] + and signals["recent_signups_ip"] <= 1 + and signals["recent_signups_fp"] <= 1 + ): + evaluation = CreditEvaluation( + decision="grant", + reasoning="No suspicious correlations found — clean user", + confidence=1.0, + ) + self._log_evaluation(email, grant_type, evaluation, signals) + return evaluation + + # Build prompt and call Gemini + prompt = _build_user_prompt( + email=email, + grant_type=grant_type, + user_data=signals["user_data"], + fingerprint_matches=signals["fingerprint_matches"], + ip_matches=signals["ip_matches"], + ip_geo=signals["ip_geo"], + recent_signups_ip=signals["recent_signups_ip"], + recent_signups_fp=signals["recent_signups_fp"], + user_agent=signals["user_agent"], + feedback_content=feedback_content, + ) + + response_text = self._call_gemini(prompt) + evaluation = _parse_gemini_response(response_text) + + self._log_evaluation(email, grant_type, evaluation, signals) + logger.info( + f"Credit evaluation for {email} ({grant_type}): " + f"{evaluation.decision} (confidence={evaluation.confidence:.2f})" + ) + return evaluation + + except Exception as e: + logger.exception(f"Credit evaluation failed for {email} — pending manual review (fail-closed)") + evaluation = CreditEvaluation( + decision="pending_review", + reasoning="Evaluation failed — pending manual review", + confidence=0.0, + error=str(e), + ) + try: + self._log_evaluation(email, grant_type, evaluation, {}) + except Exception: + pass + return evaluation + + +_service_instance: Optional[CreditEvaluationService] = None + + +def get_credit_evaluation_service() -> CreditEvaluationService: + """Get or create the singleton credit evaluation service.""" + global _service_instance + if _service_instance is None: + _service_instance = CreditEvaluationService() + return _service_instance diff --git a/backend/services/email_service.py b/backend/services/email_service.py index c99caa30..e6a883fe 100644 --- a/backend/services/email_service.py +++ b/backend/services/email_service.py @@ -1548,6 +1548,115 @@ View all feedback: {admin_url} ) + def send_credit_review_needed_email(self, user_email: str, grant_type: str, reason: str) -> bool: + """ + Notify Andrew that a credit grant needs manual review. + + Sent when AI evaluation fails, is uncertain, or encounters an error. + Includes a direct link to the user's admin page for quick action. + """ + admin_user_url = f"{self.frontend_url}/admin/users/detail?email={user_email}" + subject = f"Credit review needed: {user_email} ({grant_type})" + + content = f""" +

A {grant_type} credit grant for {user_email} needs your review.

+ +

Reason: {reason}

+ +

The user has been told that their signup is being reviewed and credits will be + assigned shortly if everything checks out.

+ +

+ Review User +

+ +

+ If the user looks legitimate, grant them credits from their admin page. + If not, no action needed — they already have 0 credits. +

+""" + + text_content = f"""Credit review needed: {user_email} ({grant_type}) + +Reason: {reason} + +Review user: {admin_user_url} + +If the user looks legitimate, grant them credits from their admin page. +""" + + html_content = self._build_email_html(content) + + return self.provider.send_email( + to_email="andrew@beveridge.uk", + subject=subject, + html_content=html_content, + text_content=text_content, + ) + + def send_credit_denied_email(self, email: str, grant_type: str = "welcome") -> bool: + """ + Send a friendly-but-firm email when free credits are denied due to abuse signals. + + CC'd to andrew@beveridge.uk so he can intervene if it's a false positive. + The tone invites the user to reply if it's a mistake. + """ + subject = "About your Nomad Karaoke free credits" + + credit_type = "welcome credits" if grant_type == "welcome" else "feedback bonus credits" + + content = f""" +

Hi there,

+ +

Thanks for signing up for Nomad Karaoke!

+ +

Unfortunately, we weren't able to grant your free {credit_type} this time. Our system detected + patterns that suggest this may not be your first account with us, and each karaoke generation job + costs us real money in cloud computing and API fees.

+ +

We totally understand if this is a mistake! If you're a genuine new user, please just + reply to this email and let Andrew (the founder) know. He built this service and is + happy to grant you some free credits to try it out — he just asks that you provide honest feedback + on the experience in return.

+ +

If you'd like to get started right away, you can also purchase credits:

+ +

+ Buy Credits +

+ +

We keep costs low so we can offer the service affordably, + which means we need to be careful about free credit abuse. Thanks for understanding!

+ + {self._get_email_signature()} +""" + + text_content = f"""Hi there, + +Thanks for signing up for Nomad Karaoke! + +Unfortunately, we weren't able to grant your free {credit_type} this time. Our system detected patterns that suggest this may not be your first account with us, and each karaoke generation job costs us real money in cloud computing and API fees. + +We totally understand if this is a mistake! If you're a genuine new user, please just reply to this email and let Andrew (the founder) know. He built this service and is happy to grant you some free credits to try it out - he just asks that you provide honest feedback on the experience in return. + +If you'd like to get started right away, you can also purchase credits: +{self.buy_url} + +--- +© {self._get_year()} Nomad Karaoke +""" + + html_content = self._build_email_html(content) + + return self.provider.send_email( + to_email=email, + subject=subject, + html_content=html_content, + text_content=text_content, + cc_emails=["andrew@beveridge.uk"], + ) + + # Global instance _email_service = None diff --git a/backend/services/user_service.py b/backend/services/user_service.py index c4d4d6f2..a09fad5d 100644 --- a/backend/services/user_service.py +++ b/backend/services/user_service.py @@ -370,31 +370,76 @@ class UserService: logger.exception("Error verifying magic link") return False, None, "An error occurred during verification" - def grant_welcome_credits_if_eligible(self, email: str) -> bool: + def grant_welcome_credits_if_eligible(self, email: str) -> Tuple[bool, str]: """ Grant welcome credits on first magic link verification. + Uses AI evaluation to check for abuse before granting. Fail-open: + if evaluation fails, credits are granted anyway. + Uses a dedicated welcome_credits_granted flag as primary idempotency check (survives credit_transactions list trimming). Falls back to checking for welcome_credit transaction as secondary defense. Returns: - True if credits were granted, False if already received or user not found. + Tuple of (credits_granted: bool, status: str). + status is one of: "granted", "denied", "already_granted", "not_found", "error" """ try: email = email.lower() user = self.get_user(email) if not user: - return False + return False, "not_found" # Primary check: dedicated flag (survives transaction list trimming) if user.welcome_credits_granted: - return False + return False, "already_granted" # Secondary check: look for existing welcome_credit transaction for txn in user.credit_transactions: if txn.reason == "welcome_credit": - return False # Already received + return False, "already_granted" + + # AI evaluation: check for abuse signals before granting + try: + from backend.services.credit_evaluation_service import get_credit_evaluation_service + eval_service = get_credit_evaluation_service() + evaluation = eval_service.evaluate(email, "welcome") + + if evaluation.decision == "deny": + # Mark as evaluated but not granted so we don't re-evaluate + self.update_user(email, welcome_credits_granted=True) + logger.info( + f"Denied welcome credits to {email}: {evaluation.reasoning}" + ) + try: + from backend.services.email_service import get_email_service + get_email_service().send_credit_denied_email(email, "welcome") + except Exception: + logger.exception(f"Failed to send credit denied email to {email}") + return False, "denied" + + if evaluation.decision == "pending_review": + # Don't mark welcome_credits_granted — allow retry after manual review + logger.info( + f"Welcome credits pending review for {email}: {evaluation.reasoning}" + ) + try: + from backend.services.email_service import get_email_service + email_service = get_email_service() + email_service.send_credit_review_needed_email(email, "welcome", evaluation.reasoning) + except Exception: + logger.exception(f"Failed to send review needed email for {email}") + return False, "pending_review" + except Exception: + logger.exception(f"Credit evaluation failed for {email} — pending review (fail-closed)") + try: + from backend.services.email_service import get_email_service + email_service = get_email_service() + email_service.send_credit_review_needed_email(email, "welcome", f"Evaluation error: {email}") + except Exception: + pass + return False, "pending_review" # Grant welcome credits welcome_credit = self.NEW_USER_FREE_CREDITS @@ -416,11 +461,11 @@ class UserService: ) logger.info(f"Granted {welcome_credit} welcome credits to {email}") - return True + return True, "granted" except Exception: logger.exception(f"Error granting welcome credits to {email}") - return False + return False, "error" # ========================================================================= # Anti-Abuse Rate Limiting diff --git a/backend/tests/test_anti_abuse.py b/backend/tests/test_anti_abuse.py index e4f067ea..11ba330c 100644 --- a/backend/tests/test_anti_abuse.py +++ b/backend/tests/test_anti_abuse.py @@ -118,9 +118,12 @@ class TestGrantWelcomeCredits: from backend.services.user_service import UserService service = UserService() - result = service.grant_welcome_credits_if_eligible("new@example.com") + with patch('backend.services.credit_evaluation_service.get_credit_evaluation_service') as mock_eval: + from backend.services.credit_evaluation_service import CreditEvaluation + mock_eval.return_value.evaluate.return_value = CreditEvaluation(decision="grant", reasoning="OK", confidence=1.0) + granted, status = service.grant_welcome_credits_if_eligible("new@example.com") - assert result is True + assert granted is True # Verify update was called with credits and flag mock_db.collection.return_value.document.return_value.update.assert_called_once() update_args = mock_db.collection.return_value.document.return_value.update.call_args[0][0] @@ -148,9 +151,9 @@ class TestGrantWelcomeCredits: from backend.services.user_service import UserService service = UserService() - result = service.grant_welcome_credits_if_eligible("existing@example.com") + granted, status = service.grant_welcome_credits_if_eligible("existing@example.com") - assert result is False + assert granted is False @patch('backend.services.user_service.get_settings') @patch('backend.services.user_service.firestore') @@ -180,9 +183,9 @@ class TestGrantWelcomeCredits: from backend.services.user_service import UserService service = UserService() - result = service.grant_welcome_credits_if_eligible("existing@example.com") + granted, status = service.grant_welcome_credits_if_eligible("existing@example.com") - assert result is False + assert granted is False @patch('backend.services.user_service.get_settings') @patch('backend.services.user_service.firestore') @@ -199,9 +202,9 @@ class TestGrantWelcomeCredits: from backend.services.user_service import UserService service = UserService() - result = service.grant_welcome_credits_if_eligible("ghost@example.com") + granted, status = service.grant_welcome_credits_if_eligible("ghost@example.com") - assert result is False + assert granted is False # ============================================================================= @@ -524,7 +527,7 @@ class TestVerifyGrantsCredits: from backend.services.email_service import get_email_service mock_user_svc.verify_magic_link.return_value = (True, user, "Success") - mock_user_svc.grant_welcome_credits_if_eligible.return_value = credits_eligible + mock_user_svc.grant_welcome_credits_if_eligible.return_value = (credits_eligible, "granted" if credits_eligible else "already_granted") mock_user_svc.get_user.return_value = user mock_user_svc.create_session.return_value = MagicMock(token="session-token") mock_user_svc.NEW_USER_FREE_CREDITS = 2 @@ -652,8 +655,8 @@ class TestErrorPaths: from backend.services.user_service import UserService service = UserService() - result = service.grant_welcome_credits_if_eligible("test@example.com") - assert result is False + granted, status = service.grant_welcome_credits_if_eligible("test@example.com") + assert granted is False @patch('backend.services.user_service.get_settings') @patch('backend.services.user_service.firestore') diff --git a/backend/tests/test_credit_evaluation.py b/backend/tests/test_credit_evaluation.py new file mode 100644 index 00000000..510b77b3 --- /dev/null +++ b/backend/tests/test_credit_evaluation.py @@ -0,0 +1,346 @@ +""" +Tests for AI-powered credit evaluation service. +""" +import pytest +from unittest.mock import MagicMock, patch, PropertyMock + +import sys +sys.modules.setdefault('google.cloud.firestore', MagicMock()) +sys.modules.setdefault('google.cloud.firestore_v1', MagicMock()) + +from backend.models.user import User + + +# ============================================================================= +# Prompt Building Tests +# ============================================================================= + + +class TestBuildUserPrompt: + + def test_builds_prompt_with_all_signals(self): + from backend.services.credit_evaluation_service import _build_user_prompt + + prompt = _build_user_prompt( + email="test@example.com", + grant_type="welcome", + user_data={ + "device_fingerprint": "fp123", + "signup_ip": "1.2.3.4", + "credits": 0, + "total_jobs_created": 0, + "total_jobs_completed": 0, + "total_spent": 0, + "created_at": "2026-03-23", + }, + fingerprint_matches=[ + {"email": "other@test.com", "total_jobs_created": 5, "total_spent": 0, "created_at": "2026-03-20"}, + ], + ip_matches=[], + ip_geo={"status": "success", "country": "US", "country_code": "US", "city": "Portland", "region": "OR", "isp": "Comcast", "org": "Comcast", "as_number": "AS7922", "as_name": "Comcast"}, + recent_signups_ip=1, + recent_signups_fp=2, + user_agent="Mozilla/5.0", + ) + + assert "test@example.com" in prompt + assert "fp123" in prompt + assert "1.2.3.4" in prompt + assert "other@test.com" in prompt + assert "RED FLAG" in prompt + assert "Portland" in prompt + assert "Comcast" in prompt + assert "Mozilla/5.0" in prompt + + def test_builds_prompt_without_correlations(self): + from backend.services.credit_evaluation_service import _build_user_prompt + + prompt = _build_user_prompt( + email="clean@user.com", + grant_type="welcome", + user_data={"device_fingerprint": "fp999", "signup_ip": "5.6.7.8"}, + fingerprint_matches=[], + ip_matches=[], + ip_geo=None, + recent_signups_ip=1, + recent_signups_fp=1, + user_agent=None, + ) + + assert "None found" in prompt + assert "clean@user.com" in prompt + + def test_includes_feedback_content(self): + from backend.services.credit_evaluation_service import _build_user_prompt + + prompt = _build_user_prompt( + email="test@example.com", + grant_type="feedback", + user_data={}, + fingerprint_matches=[], + ip_matches=[], + ip_geo=None, + recent_signups_ip=0, + recent_signups_fp=0, + user_agent=None, + feedback_content={ + "overall_rating": 5, + "what_went_well": "Great audio quality!", + "what_could_improve": "Faster processing", + "additional_comments": "Love it", + }, + ) + + assert "Feedback Submitted" in prompt + assert "Great audio quality!" in prompt + assert "5" in prompt + + +# ============================================================================= +# Response Parsing Tests +# ============================================================================= + + +class TestParseGeminiResponse: + + def test_parses_grant_decision(self): + from backend.services.credit_evaluation_service import _parse_gemini_response + + result = _parse_gemini_response('{"decision": "grant", "reasoning": "Clean user", "confidence": 0.95}') + assert result.decision == "grant" + assert result.reasoning == "Clean user" + assert result.confidence == 0.95 + assert result.error is None + + def test_parses_deny_decision(self): + from backend.services.credit_evaluation_service import _parse_gemini_response + + result = _parse_gemini_response('{"decision": "deny", "reasoning": "Same fingerprint as 3 other accounts", "confidence": 0.9}') + assert result.decision == "deny" + assert "fingerprint" in result.reasoning + + def test_handles_markdown_code_fences(self): + from backend.services.credit_evaluation_service import _parse_gemini_response + + result = _parse_gemini_response('```json\n{"decision": "grant", "reasoning": "OK", "confidence": 0.8}\n```') + assert result.decision == "grant" + + def test_fails_closed_on_invalid_json(self): + from backend.services.credit_evaluation_service import _parse_gemini_response + + result = _parse_gemini_response("I think this user is fine") + assert result.decision == "pending_review" + assert result.error is not None + + def test_fails_closed_on_unexpected_decision(self): + from backend.services.credit_evaluation_service import _parse_gemini_response + + result = _parse_gemini_response('{"decision": "maybe", "reasoning": "Unsure", "confidence": 0.5}') + assert result.decision == "pending_review" # fail-closed + + def test_fails_closed_on_empty_response(self): + from backend.services.credit_evaluation_service import _parse_gemini_response + + result = _parse_gemini_response("") + assert result.decision == "pending_review" + assert result.error is not None + + +# ============================================================================= +# Evaluation Service Tests +# ============================================================================= + + +class TestCreditEvaluationService: + + @patch('backend.services.credit_evaluation_service.get_settings') + def test_returns_grant_when_disabled(self, mock_settings): + mock_settings.return_value = MagicMock(credit_eval_enabled=False) + + from backend.services.credit_evaluation_service import CreditEvaluationService + service = CreditEvaluationService() + result = service.evaluate("test@example.com", "welcome") + + assert result.decision == "grant" + assert "disabled" in result.reasoning + + @patch('backend.services.credit_evaluation_service.get_settings') + def test_quick_grants_clean_users(self, mock_settings): + mock_settings.return_value = MagicMock( + credit_eval_enabled=True, + credit_eval_model="gemini-3.1-pro-preview", + google_cloud_project="test", + ) + + from backend.services.credit_evaluation_service import CreditEvaluationService + service = CreditEvaluationService() + + with patch.object(service, '_collect_signals', return_value={ + "user_data": {"email": "clean@user.com"}, + "fingerprint_matches": [], + "ip_matches": [], + "ip_geo": None, + "recent_signups_ip": 1, + "recent_signups_fp": 1, + "user_agent": "Mozilla/5.0", + }), patch.object(service, '_log_evaluation'): + result = service.evaluate("clean@user.com", "welcome") + + assert result.decision == "grant" + assert "clean" in result.reasoning.lower() + + @patch('backend.services.credit_evaluation_service.get_settings') + def test_calls_gemini_when_correlations_found(self, mock_settings): + mock_settings.return_value = MagicMock( + credit_eval_enabled=True, + credit_eval_model="gemini-3.1-pro-preview", + google_cloud_project="test", + ) + + from backend.services.credit_evaluation_service import CreditEvaluationService + service = CreditEvaluationService() + + with patch.object(service, '_collect_signals', return_value={ + "user_data": {"email": "sus@user.com", "device_fingerprint": "fp123", "signup_ip": "1.2.3.4"}, + "fingerprint_matches": [{"email": "other@test.com", "total_jobs_created": 5, "total_spent": 0, "created_at": "2026-03-20"}], + "ip_matches": [], + "ip_geo": None, + "recent_signups_ip": 1, + "recent_signups_fp": 2, + "user_agent": None, + }), patch.object(service, '_call_gemini', return_value='{"decision": "deny", "reasoning": "Same fingerprint", "confidence": 0.9}'), \ + patch.object(service, '_log_evaluation'): + result = service.evaluate("sus@user.com", "welcome") + + assert result.decision == "deny" + + @patch('backend.services.credit_evaluation_service.get_settings') + def test_fails_closed_on_gemini_error(self, mock_settings): + mock_settings.return_value = MagicMock( + credit_eval_enabled=True, + credit_eval_model="gemini-3.1-pro-preview", + google_cloud_project="test", + ) + + from backend.services.credit_evaluation_service import CreditEvaluationService + service = CreditEvaluationService() + + with patch.object(service, '_collect_signals', side_effect=Exception("Firestore down")), \ + patch.object(service, '_log_evaluation'): + result = service.evaluate("test@example.com", "welcome") + + assert result.decision == "pending_review" + assert result.error is not None + + +# ============================================================================= +# Welcome Credit Integration Tests +# ============================================================================= + + +class TestWelcomeCreditsWithEvaluation: + + @patch('backend.services.user_service.get_settings') + @patch('backend.services.user_service.firestore') + def test_grants_credits_when_evaluation_approves(self, mock_fs, mock_settings): + mock_settings.return_value = MagicMock(google_cloud_project='test') + mock_db = MagicMock() + mock_fs.Client.return_value = mock_db + + user = User(email="new@user.com", welcome_credits_granted=False) + user_doc = MagicMock() + user_doc.exists = True + user_doc.to_dict.return_value = user.model_dump(mode='json') + mock_db.collection.return_value.document.return_value.get.return_value = user_doc + + from backend.services.user_service import UserService + from backend.services.credit_evaluation_service import CreditEvaluation + + service = UserService() + + with patch('backend.services.credit_evaluation_service.get_credit_evaluation_service') as mock_eval: + mock_eval.return_value.evaluate.return_value = CreditEvaluation( + decision="grant", reasoning="Clean user", confidence=0.95 + ) + granted, status = service.grant_welcome_credits_if_eligible("new@user.com") + + assert granted is True + assert status == "granted" + + @patch('backend.services.user_service.get_settings') + @patch('backend.services.user_service.firestore') + def test_denies_credits_when_evaluation_rejects(self, mock_fs, mock_settings): + mock_settings.return_value = MagicMock(google_cloud_project='test') + mock_db = MagicMock() + mock_fs.Client.return_value = mock_db + + user = User(email="abuser@test.com", welcome_credits_granted=False) + user_doc = MagicMock() + user_doc.exists = True + user_doc.to_dict.return_value = user.model_dump(mode='json') + mock_db.collection.return_value.document.return_value.get.return_value = user_doc + + from backend.services.user_service import UserService + from backend.services.credit_evaluation_service import CreditEvaluation + + service = UserService() + + with patch('backend.services.credit_evaluation_service.get_credit_evaluation_service') as mock_eval, \ + patch('backend.services.email_service.get_email_service') as mock_email: + mock_eval.return_value.evaluate.return_value = CreditEvaluation( + decision="deny", reasoning="Same fingerprint as 3 accounts", confidence=0.9 + ) + granted, status = service.grant_welcome_credits_if_eligible("abuser@test.com") + + assert granted is False + assert status == "denied" + # Verify rejection email was sent + mock_email.return_value.send_credit_denied_email.assert_called_once_with("abuser@test.com", "welcome") + + @patch('backend.services.user_service.get_settings') + @patch('backend.services.user_service.firestore') + def test_pending_review_when_evaluation_fails(self, mock_fs, mock_settings): + """Fail-closed: if evaluation crashes, credits are NOT granted, admin notified.""" + mock_settings.return_value = MagicMock(google_cloud_project='test') + mock_db = MagicMock() + mock_fs.Client.return_value = mock_db + + user = User(email="new@user.com", welcome_credits_granted=False) + user_doc = MagicMock() + user_doc.exists = True + user_doc.to_dict.return_value = user.model_dump(mode='json') + mock_db.collection.return_value.document.return_value.get.return_value = user_doc + + from backend.services.user_service import UserService + + service = UserService() + + with patch('backend.services.credit_evaluation_service.get_credit_evaluation_service') as mock_eval, \ + patch('backend.services.email_service.get_email_service') as mock_email: + mock_eval.return_value.evaluate.side_effect = Exception("Gemini is down") + granted, status = service.grant_welcome_credits_if_eligible("new@user.com") + + assert granted is False + assert status == "pending_review" + # Verify admin review email was sent + mock_email.return_value.send_credit_review_needed_email.assert_called_once() + + @patch('backend.services.user_service.get_settings') + @patch('backend.services.user_service.firestore') + def test_skips_already_granted(self, mock_fs, mock_settings): + mock_settings.return_value = MagicMock(google_cloud_project='test') + mock_db = MagicMock() + mock_fs.Client.return_value = mock_db + + user = User(email="existing@user.com", welcome_credits_granted=True) + user_doc = MagicMock() + user_doc.exists = True + user_doc.to_dict.return_value = user.model_dump(mode='json') + mock_db.collection.return_value.document.return_value.get.return_value = user_doc + + from backend.services.user_service import UserService + service = UserService() + granted, status = service.grant_welcome_credits_if_eligible("existing@user.com") + + assert granted is False + assert status == "already_granted" diff --git a/docs/AI-CREDIT-GATING.md b/docs/AI-CREDIT-GATING.md new file mode 100644 index 00000000..3bca2c8d --- /dev/null +++ b/docs/AI-CREDIT-GATING.md @@ -0,0 +1,155 @@ +# AI-Powered Credit Gating + +## Overview + +Free credit grants (welcome credits and feedback bonus credits) are evaluated by an AI model before being approved. This prevents multi-account abuse where users create multiple accounts to farm free karaoke generation credits. + +The system is **fail-closed**: if anything goes wrong (AI error, unclear response, network issue), credits are NOT granted automatically. Instead, the admin is notified to review manually. + +## How It Works + +``` +User verifies email (or submits feedback) + ↓ +Collect abuse signals: + • Device fingerprint correlations (other accounts with same fingerprint) + • IP address correlations (other accounts from same IP) + • IP geolocation (country, ISP, VPN detection) + • Recent signup patterns (new accounts from same IP/fingerprint in last 24h) + • User agent string + • For feedback: the submitted feedback content + ↓ +Clean user? (no correlations, no recent signup spike) + → YES: Grant credits immediately (no AI call needed) + → NO: Call Gemini for evaluation + ↓ +Gemini decision: + • "grant" → Grant credits, user sees celebratory welcome screen + • "deny" → No credits, user gets rejection email (CC'd to Andrew) + • Error/uncertain → No credits, user told "team will review shortly", + Andrew gets review email with admin link +``` + +## Decision Outcomes + +| Scenario | Credits | User Experience | Admin Notification | +|----------|---------|-----------------|-------------------| +| Clean user (no correlations) | Granted | "2 free credits!" interstitial | None | +| AI approves | Granted | Same | None | +| AI denies (abuse detected) | Not granted | "Couldn't grant free credits" + buy CTA | CC on rejection email to user | +| AI error / uncertain | Not granted | "Team will review shortly" + buy CTA | Review email with admin page link | +| Evaluation disabled | Granted | Same as clean | None | + +## Frontend Flow + +The verify page (`/auth/verify`) shows these states in sequence: + +1. **"Verifying your sign-in link..."** — Token validation +2. **"Preparing your account..."** — AI evaluation running (2-5 seconds) +3. One of: + - **Credits granted** — Celebratory screen with credit count, gentle reminder not to abuse, "Start Creating Karaoke" button + - **Credits denied** — "Couldn't grant free credits" with buy credits CTA and link to dashboard + - **Credits pending review** — "Team will review shortly" with buy credits CTA and "Explore the App" button + - **Success** (returning user) — "Welcome back!" with auto-redirect + +## Emails + +### To User: Credit Denied +- Subject: "About your Nomad Karaoke free credits" +- CC: andrew@beveridge.uk +- Tone: Friendly but firm — explains costs, invites reply if mistake +- Includes buy credits link + +### To Admin: Review Needed +- Subject: "Credit review needed: user@example.com (welcome)" +- To: andrew@beveridge.uk +- Includes direct link to user's admin page (`/admin/users/detail?email=...`) +- Explains what to do: grant credits if legitimate, do nothing if not + +## Configuration + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `CREDIT_EVAL_ENABLED` | `true` | Set to `false` to disable AI evaluation and auto-grant all credits | +| `CREDIT_EVAL_MODEL` | `gemini-3.1-pro-preview` | Gemini model to use for evaluation | + +### Emergency Rollback + +Set `CREDIT_EVAL_ENABLED=false` in Cloud Run environment variables. This immediately reverts to the old behavior of auto-granting all credits without AI evaluation. + +## Audit Trail + +Every evaluation is logged to the `credit_evaluations` Firestore collection: + +```json +{ + "id": "uuid", + "email": "user@example.com", + "grant_type": "welcome", + "decision": "grant", + "reasoning": "No suspicious correlations found — clean user", + "confidence": 1.0, + "error": null, + "model": "gemini-3.1-pro-preview", + "signals_snapshot": { + "fingerprint_match_count": 0, + "ip_match_count": 0, + "recent_signups_ip": 1, + "recent_signups_fp": 1, + "user_ip": "1.2.3.4", + "user_fingerprint": "abc123..." + }, + "evaluated_at": "2026-03-23T21:00:00Z" +} +``` + +## Admin Workflow + +When you receive a "Credit review needed" email: + +1. Click the "Review User" link to go to their admin page +2. Check their abuse signals: fingerprint matches, IP matches, job history +3. If legitimate: Use the admin credits endpoint to grant 2 credits manually +4. If abusive: No action needed — they already have 0 credits + +Note: For `pending_review` cases, the `welcome_credits_granted` flag is NOT set, so the system won't interfere if you grant credits manually later. + +## Architecture + +### Files + +| File | Purpose | +|------|---------| +| `backend/services/credit_evaluation_service.py` | Core service: signal collection, Gemini prompt, response parsing, audit logging | +| `backend/config.py` | `CREDIT_EVAL_ENABLED`, `CREDIT_EVAL_MODEL` config | +| `backend/services/user_service.py` | `grant_welcome_credits_if_eligible()` integration | +| `backend/api/routes/users.py` | Feedback credit flow integration, verify response includes `credit_status` | +| `backend/services/email_service.py` | `send_credit_denied_email()`, `send_credit_review_needed_email()` | +| `backend/models/user.py` | `VerifyMagicLinkResponse.credit_status` field | +| `frontend/app/auth/verify/page.tsx` | 5-state verify flow with interstitials | +| `frontend/lib/types.ts` | `credit_status` type on verify response | + +### Signal Collection + +The evaluation service calls these existing methods to collect signals: + +- `user_service.find_users_by_fingerprint()` — other accounts with same device fingerprint +- `user_service.find_users_by_signup_ip()` — other accounts from same IP +- `user_service.count_recent_signups_from_ip()` — new accounts from same IP in last 24h +- `user_service.count_recent_signups_from_fingerprint()` — new accounts from same fingerprint in last 24h +- `ip_geolocation_service.lookup_ip()` — country, ISP, org info +- Session collection query — latest user agent + +### Gemini Prompt + +The prompt instructs Gemini to: +- **DENY** if fingerprint matches other accounts (strong multi-accounting signal) +- **DENY** if IP matches + other suspicious patterns (same UA, no spend) +- **GRANT** if appears genuine or insufficient evidence +- **When in doubt, GRANT** — false positives worse than false negatives + +The response is structured JSON: `{"decision": "grant"|"deny", "reasoning": "...", "confidence": 0.0-1.0}` + +### Quick-Grant Optimization + +If no correlations are found at all (no fingerprint matches, no IP matches, recent signup counts ≤ 1), the system skips the AI call entirely and grants immediately. This means most legitimate users experience zero additional latency. diff --git a/frontend/app/auth/verify/page.tsx b/frontend/app/auth/verify/page.tsx index 30c79ebf..edffe4d2 100644 --- a/frontend/app/auth/verify/page.tsx +++ b/frontend/app/auth/verify/page.tsx @@ -2,24 +2,24 @@ import { Suspense, useEffect, useState, useRef } from "react" import { useRouter, useSearchParams } from "next/navigation" -import { Loader2, CheckCircle, XCircle } from "lucide-react" +import { Loader2, CheckCircle, XCircle, Gift, ShoppingCart } from "lucide-react" import { Button } from "@/components/ui/button" import { useAuth } from "@/lib/auth" -type VerifyState = "loading" | "success" | "error" +type VerifyState = "verifying" | "preparing" | "credits_granted" | "credits_denied" | "credits_pending" | "success" | "error" function VerifyMagicLinkContent() { const router = useRouter() const searchParams = useSearchParams() - const [state, setState] = useState("loading") + const [state, setState] = useState("verifying") const [errorMessage, setErrorMessage] = useState("") const [creditsGranted, setCreditsGranted] = useState(0) + const [creditStatus, setCreditStatus] = useState("not_applicable") const hasVerified = useRef(false) const { verifyMagicLink, user, error } = useAuth() useEffect(() => { - // Prevent duplicate verification attempts if (hasVerified.current) return hasVerified.current = true @@ -31,36 +31,47 @@ function VerifyMagicLinkContent() { return } - // Verify the token const verify = async () => { + // Show "preparing your account" while the backend runs AI evaluation + setState("preparing") + const success = await verifyMagicLink(token) if (success) { - setState("success") - // Check verify response for welcome credits and tenant redirect const lastVerifyResponse = (window as any).__LAST_VERIFY_RESPONSE__ - if (lastVerifyResponse?.credits_granted > 0) { - setCreditsGranted(lastVerifyResponse.credits_granted) - } - // Check if the verify response included a tenant subdomain - // If we're on the wrong domain, redirect to the correct tenant portal + const credits = lastVerifyResponse?.credits_granted || 0 + const status = lastVerifyResponse?.credit_status || "not_applicable" + setCreditsGranted(credits) + setCreditStatus(status) + + // Check for tenant subdomain redirect if (lastVerifyResponse?.tenant_subdomain) { const currentHost = window.location.hostname.toLowerCase() const tenantSubdomain = lastVerifyResponse.tenant_subdomain.toLowerCase() if (currentHost !== tenantSubdomain) { - // We're on the wrong domain — redirect to the correct tenant portal + setState("success") setTimeout(() => { window.location.href = `https://${tenantSubdomain}/app` }, 1500) return } } - // Same domain or no tenant — redirect normally - setTimeout(() => { - router.push("/app") - }, 2000) + + // Show appropriate interstitial based on credit status + if (status === "granted" && credits > 0) { + setState("credits_granted") + } else if (status === "denied") { + setState("credits_denied") + } else if (status === "pending_review") { + setState("credits_pending") + } else { + // Returning user or not applicable — go straight to app + setState("success") + setTimeout(() => { + router.push("/app") + }, 1500) + } } else { setState("error") - // Get error from auth store at this point const authStore = useAuth.getState() setErrorMessage(authStore.error || "Invalid or expired link") } @@ -69,9 +80,13 @@ function VerifyMagicLinkContent() { verify() }, [searchParams, verifyMagicLink, router]) + const goToApp = () => router.push("/app") + const goToBuy = () => router.push("/app?buy=true") + return (
- {state === "loading" && ( + {/* Verifying token */} + {state === "verifying" && ( <>

@@ -83,6 +98,118 @@ function VerifyMagicLinkContent() { )} + {/* Preparing account (AI evaluation running) */} + {state === "preparing" && ( + <> + +

+ Preparing your account... +

+

+ Just a moment while we set things up for you. +

+ + )} + + {/* Credits granted - celebratory interstitial */} + {state === "credits_granted" && ( + <> +
+ +
+

+ Welcome to Nomad Karaoke! +

+
+

{creditsGranted}

+

free credits to get started

+
+

+ Each credit creates one professional karaoke video. +

+

+ These credits cost us real money to fulfil, so please use them wisely + and let us know what you think! +

+ + + )} + + {/* Credits denied - friendly rejection interstitial */} + {state === "credits_denied" && ( + <> + +

+ Welcome to Nomad Karaoke +

+

+ We weren't able to grant free credits this time. + Karaoke generation costs us real money for every job, + so we need to be careful about free credit distribution. +

+

+ If you think this is a mistake, check your email — we've sent you + details on how to reach out to get this resolved. +

+
+ + +
+ + )} + + {/* Credits pending review - friendly holding message */} + {state === "credits_pending" && ( + <> + +

+ Welcome to Nomad Karaoke! +

+

+ We weren't able to assign your free credits automatically this time. + Our team will review your signup shortly and assign credits if we're able to. +

+

+ You can still explore the app while you wait, or purchase credits + to get started right away. +

+
+ + +
+ + )} + + {/* Simple success (returning user) */} {state === "success" && ( <> @@ -90,19 +217,15 @@ function VerifyMagicLinkContent() { Successfully signed in!

- Welcome{user?.email ? `, ${user.email}` : ""}! + Welcome back{user?.email ? `, ${user.email}` : ""}!

- {creditsGranted > 0 && ( -

- You received {creditsGranted} free credits to get started! -

- )}

Redirecting you to the app...

)} + {/* Error */} {state === "error" && ( <> diff --git a/frontend/lib/auth.ts b/frontend/lib/auth.ts index 6ac113d9..e9975645 100644 --- a/frontend/lib/auth.ts +++ b/frontend/lib/auth.ts @@ -81,6 +81,7 @@ export const useAuth = create()( (window as any).__LAST_VERIFY_RESPONSE__ = { tenant_subdomain: response.tenant_subdomain, credits_granted: response.credits_granted || 0, + credit_status: response.credit_status || "not_applicable", } } diff --git a/frontend/lib/types.ts b/frontend/lib/types.ts index 746afb16..6e9ee60a 100644 --- a/frontend/lib/types.ts +++ b/frontend/lib/types.ts @@ -33,6 +33,7 @@ export interface VerifyMagicLinkResponse { message: string tenant_subdomain?: string | null credits_granted?: number + credit_status?: string // "granted", "denied", "already_granted", "not_applicable" } export interface UserProfileResponse { diff --git a/poetry.lock b/poetry.lock index 00e14512..f9b730ea 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2439,6 +2439,27 @@ files = [ {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"}, ] +[[package]] +name = "google-ai-generativelanguage" +version = "0.6.15" +description = "Google Ai Generativelanguage API client library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_ai_generativelanguage-0.6.15-py3-none-any.whl", hash = "sha256:5a03ef86377aa184ffef3662ca28f19eeee158733e45d7947982eb953c6ebb6c"}, + {file = "google_ai_generativelanguage-0.6.15.tar.gz", hash = "sha256:8f6d9dc4c12b065fe2d0289026171acea5183ebf2d0b11cefe12f3821e159ec3"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" +proto-plus = [ + {version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0dev"}, +] +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" + [[package]] name = "google-api-core" version = "2.28.1" @@ -2812,6 +2833,30 @@ websockets = ">=13.0.0,<15.1.0" aiohttp = ["aiohttp (<3.13.3)"] local-tokenizer = ["protobuf", "sentencepiece (>=0.2.0)"] +[[package]] +name = "google-generativeai" +version = "0.8.6" +description = "Google Generative AI High level API client library and tools." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_generativeai-0.8.6-py3-none-any.whl", hash = "sha256:37a0eaaa95e5bbf888828e20a4a1b2c196cc9527d194706e58a68ff388aeb0fa"}, +] + +[package.dependencies] +google-ai-generativelanguage = "0.6.15" +google-api-core = "*" +google-api-python-client = "*" +google-auth = ">=2.15.0" +protobuf = "*" +pydantic = "*" +tqdm = "*" +typing-extensions = "*" + +[package.extras] +dev = ["Pillow", "absl-py", "black", "ipython", "nose2", "pandas", "pytype", "pyyaml"] + [[package]] name = "google-resumable-media" version = "2.8.0" @@ -2946,20 +2991,20 @@ protobuf = ["grpcio-tools (>=1.76.0)"] [[package]] name = "grpcio-status" -version = "1.76.0" +version = "1.71.2" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "grpcio_status-1.76.0-py3-none-any.whl", hash = "sha256:380568794055a8efbbd8871162df92012e0228a5f6dffaf57f2a00c534103b18"}, - {file = "grpcio_status-1.76.0.tar.gz", hash = "sha256:25fcbfec74c15d1a1cb5da3fab8ee9672852dc16a5a9eeb5baf7d7a9952943cd"}, + {file = "grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3"}, + {file = "grpcio_status-1.71.2.tar.gz", hash = "sha256:c7a97e176df71cdc2c179cd1847d7fc86cca5832ad12e9798d7fed6b7a1aab50"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.76.0" -protobuf = ">=6.31.1,<7.0.0" +grpcio = ">=1.71.2" +protobuf = ">=5.26.1,<6.0dev" [[package]] name = "h11" @@ -6748,22 +6793,23 @@ testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "6.33.2" +version = "5.29.6" description = "" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "protobuf-6.33.2-cp310-abi3-win32.whl", hash = "sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d"}, - {file = "protobuf-6.33.2-cp310-abi3-win_amd64.whl", hash = "sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4"}, - {file = "protobuf-6.33.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43"}, - {file = "protobuf-6.33.2-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e"}, - {file = "protobuf-6.33.2-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872"}, - {file = "protobuf-6.33.2-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f"}, - {file = "protobuf-6.33.2-cp39-cp39-win32.whl", hash = "sha256:7109dcc38a680d033ffb8bf896727423528db9163be1b6a02d6a49606dcadbfe"}, - {file = "protobuf-6.33.2-cp39-cp39-win_amd64.whl", hash = "sha256:2981c58f582f44b6b13173e12bb8656711189c2a70250845f264b877f00b1913"}, - {file = "protobuf-6.33.2-py3-none-any.whl", hash = "sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c"}, - {file = "protobuf-6.33.2.tar.gz", hash = "sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4"}, + {file = "protobuf-5.29.6-cp310-abi3-win32.whl", hash = "sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1"}, + {file = "protobuf-5.29.6-cp310-abi3-win_amd64.whl", hash = "sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda"}, + {file = "protobuf-5.29.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269"}, + {file = "protobuf-5.29.6-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6"}, + {file = "protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9"}, + {file = "protobuf-5.29.6-cp38-cp38-win32.whl", hash = "sha256:36ade6ff88212e91aef4e687a971a11d7d24d6948a66751abc1b3238648f5d05"}, + {file = "protobuf-5.29.6-cp38-cp38-win_amd64.whl", hash = "sha256:831e2da16b6cc9d8f1654c041dd594eda43391affd3c03a91bea7f7f6da106d6"}, + {file = "protobuf-5.29.6-cp39-cp39-win32.whl", hash = "sha256:cb4c86de9cd8a7f3a256b9744220d87b847371c6b2f10bde87768918ef33ba49"}, + {file = "protobuf-5.29.6-cp39-cp39-win_amd64.whl", hash = "sha256:76e07e6567f8baf827137e8d5b8204b6c7b6488bbbff1bf0a72b383f77999c18"}, + {file = "protobuf-5.29.6-py3-none-any.whl", hash = "sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86"}, + {file = "protobuf-5.29.6.tar.gz", hash = "sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723"}, ] [[package]] @@ -10705,4 +10751,4 @@ local-whisper = ["whisper-timestamped"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "6f4c1aa73c0b4adecdc901fdec83dea48a9c431056c4b8fd04da5461469d3095" +content-hash = "927fa221a036dbc7b7d4c8a6d2fb3499ebac5240a5ced3424c9ebed2245cfd1d" diff --git a/pyproject.toml b/pyproject.toml index 0b408828..dcd5a124 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.149.0" +version = "0.150.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" @@ -66,6 +66,7 @@ langchain-openai = ">=0.2.0" langchain-anthropic = ">=0.2.0" langchain-ollama = ">=0.2.0" langchain-google-genai = ">=2.0.0" +google-generativeai = ">=0.8.0" langfuse = ">=3.0.0" jiwer = ">=3.0.0" kbputils = "^0.0.16"