Compare commits

...
Author SHA1 Message Date
Andrew BeveridgeandClaude Opus 5.5 bdca8f5542 fix(gemini): pin vertexai=False on the agentic ChatGoogleGenerativeAI model
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 21:47:28 -04:00
Andrew BeveridgeandClaude Opus 5.5 b9f9c90c6d fix(gemini): pin vertexai=False on the Developer API client
google-genai selects Vertex AI when GOOGLE_GENAI_USE_VERTEXAI is set and
vertexai is omitted, even with an api_key. Pin it explicitly.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 21:26:57 -04:00
Andrew BeveridgeandClaude Opus 5.5 b8c6c6d796 fix(gemini): address review — classifier and degradation hardening
- Classifier follows only explicit __cause__ chains (no implicit
  __context__), and per-day quotas count as exhausted even with a retry
  delay.
- credit eval: classify only the Gemini call, not Firestore errors.
- Quota alert I/O runs on a daemon thread (never blocks the event loop) and
  re-arms if the Discord send fails.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 21:26:57 -04:00
Andrew BeveridgeandClaude Opus 5.5 50b3b7db8e chore(gemini): move every Gemini call from Vertex AI to the Gemini Developer API
Eliminates AI spend in the nomadkaraoke GCP project: all Gemini calls now use
the Gemini Developer API with the gemini-api-key secret instead of Vertex AI.

- backend/services/gemini_client.py: get_genai_client(timeout_ms) (key from
  GEMINI_API_KEY env, else Secret Manager via settings.get_secret),
  is_quota_or_billing_error(), note_gemini_failure() which logs and sends a
  Discord ops alert at most once per 6h (Firestore-throttled across
  instances via new ops_alerts.send_throttled_ops_alert).
- Migrated callers: auto-correct (quota -> non-retryable, compare mode keeps
  Opus), credit eval (pending_review), lyrics translation (no retry, render
  without translation), parse-titles / match-judge / free-text / tenant bulk
  (existing non-LLM fallbacks), custom lyrics (clean 503 + 5-min timeout),
  error-monitor LLM (no retry; alerts sent without AI summary), agentic
  LangChain provider (gemini/ via ChatGoogleGenerativeAI with api key, no
  project).
- frontend/scripts/translate.py + gemini_client.py: env key else gcloud
  secrets access (in-process only); exit 2 with a clear message on
  quota/credit/key errors.
- Per-minute 429 rate limits stay transient/retryable.
- CI: GEMINI_API_KEY=gemini-api-key:latest in backend --set-secrets.
- Pulumi: GEMINI_API_KEY secret env on lyrics-transcription, video-encoding
  and error-monitor jobs (already applied); drop roles/aiplatform.user from
  karaoke-backend (to be applied after deploy).
- Version 0.261.1 -> 0.262.0; docs updated.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 21:26:57 -04:00
44 changed files with 1132 additions and 191 deletions

No files matched your search

+1 -1
View File
@@ -2022,7 +2022,7 @@ jobs:
--min-instances 0 \
--clear-vpc-connector \
--set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,USE_CLOUD_RUN_JOBS_FOR_RENDER=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-5-5,AUTO_CORRECT_COMPARE_MODELS=claude-opus-5-5;gemini-3.8-flash,AUTO_CORRECT_PROACTIVE_ENABLED=true,AUTO_APPROVAL_BACKING_KEEP_ENABLED=true,COMMUNITY_DAILY_PICK_ENABLED=true,EDGE_AUTH_MODE=enforce,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \
--set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest"
--set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest"
# R3/R4 — smoke the tagged candidate revision BEFORE it serves any traffic.
- name: Smoke candidate revision
+1 -1
View File
@@ -112,7 +112,7 @@ See `docs/TESTING.md` § "Ad-Hoc Production Debugging" for full details.
- Only `/admin/*` is English-only — it has no `[locale]/admin` counterpart and inherits `DefaultIntlProvider` from the root layout
- Internal links use `Link` from `@/i18n/routing` (locale-aware)
- After adding/changing English strings: `python frontend/scripts/translate.py --messages-dir frontend/messages --target all`
- Translation uses Gemini 3.8 Flash via Vertex AI with GCS cache for efficiency
- Translation uses Gemini 3.8 Flash via the **Gemini Developer API** (key: `GEMINI_API_KEY`, else Secret Manager `gemini-api-key` via your gcloud login — `frontend/scripts/gemini_client.py`) with GCS cache for efficiency. **Vertex AI is disabled in the GCP project — never use `vertexai=True`; backend code uses `backend/services/gemini_client.get_genai_client()`.**
- CI validates all locale files have matching keys — **PR will fail if translations are missing**
- Pre-commit hook auto-translates when en.json changes (enable: `git config core.hooksPath .githooks`)
- Don't hardcode user-facing strings — add to `messages/en.json` and use `t('key')`
+2 -2
View File
@@ -82,9 +82,9 @@ class Settings(BaseSettings):
# Agentic AI Correction (for lyrics correction via LLM)
# Only used when skip_correction=false
# When enabled, uses Gemini via Vertex AI for intelligent lyrics correction
# When enabled, uses Gemini (Developer API, key GEMINI_API_KEY) for intelligent lyrics correction
use_agentic_ai: bool = os.getenv("USE_AGENTIC_AI", "true").lower() in ("true", "1", "yes")
agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "vertexai/gemini-3.8-flash")
agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "gemini/gemini-3.8-flash")
# Timeout for agentic correction in seconds. If correction takes longer, abort and
# use uncorrected transcription - human review will fix any issues.
agentic_correction_timeout_seconds: int = int(os.getenv("AGENTIC_CORRECTION_TIMEOUT_SECONDS", "180"))
+4 -2
View File
@@ -54,8 +54,10 @@ MODEL_PRICING: dict[str, _Rate] = {
"claude-opus-4-8": {"input": 5.0, "output": 25.0, "cache_read": 0.5},
"claude-sonnet-4-6": {"input": 3.0, "output": 15.0, "cache_read": 0.3},
"claude-haiku-4-5": {"input": 1.0, "output": 5.0, "cache_read": 0.1},
# --- Gemini (ESTIMATE — Vertex 3.x Pro preview, <=200K-token context
# tier; confirm against current Google pricing before trusting USD) ---
# --- Gemini (ESTIMATE — 3.x Pro preview, <=200K-token context tier;
# confirm against current Google pricing before trusting USD). Calls go
# via the Gemini Developer API (paid tier), assumed at Vertex list-price
# parity. ---
"gemini-3.1-pro-preview": {"input": 2.0, "output": 12.0, "cache_read": 0.2},
# Promo rate through 2026-12-31; see _RATE_CHANGES for the standard rate.
"gemini-3.8-flash": {"input": 0.75, "output": 3.75, "cache_read": 0.075},
+14 -8
View File
@@ -710,22 +710,19 @@ class AutoCorrectService:
def _call_gemini(
self, model: str, system_prompt: str, user_prompt: str, *, job_id: str
) -> tuple[Any, Optional[TokenUsage]]:
from google import genai
from google.genai import types
client = genai.Client(
vertexai=True,
project=self.settings.google_cloud_project,
location="global",
# Milliseconds; bounds the call so a hung model never strands the UI.
http_options=types.HttpOptions(timeout=120_000),
)
from backend.services.gemini_client import get_genai_client, note_gemini_failure
# Deep-copy: the google-genai SDK mutates the schema dict in place,
# which would poison the shared module-level constant for other
# providers running in the same process.
import copy
try:
# Gemini Developer API (API key). Timeout in milliseconds bounds the
# call so a hung model never strands the UI.
client = get_genai_client(timeout_ms=120_000)
response = client.models.generate_content(
model=model,
contents=[user_prompt],
@@ -736,6 +733,15 @@ class AutoCorrectService:
),
)
except Exception as exc: # surface as 502, never a stuck job
if note_gemini_failure("auto_correct", exc):
# Quota / prepaid credit / key problem: retrying won't help.
# Non-retryable so compare mode drops the Gemini leg at once
# and keeps the other model's (Opus) suggestions.
raise AutoCorrectServiceError(
"Gemini is unavailable (quota/billing/key) — skipped",
status_code=503,
retryable=False,
) from exc
if _is_transient_model_error(exc):
logger.warning(
"auto-correct model rate-limited/overloaded job=%s model=%s (%s)",
+10 -6
View File
@@ -295,11 +295,9 @@ class CreditEvaluationService:
"""Call Gemini and return the response text."""
from google import genai
client = genai.Client(
vertexai=True,
project=self.settings.google_cloud_project,
location="global",
)
from backend.services.gemini_client import get_genai_client
client = get_genai_client(timeout_ms=60_000)
response = client.models.generate_content(
model=self.settings.credit_eval_model,
contents=prompt,
@@ -405,7 +403,13 @@ class CreditEvaluationService:
feedback_content=feedback_content,
)
response_text = self._call_gemini(prompt)
try:
response_text = self._call_gemini(prompt)
except Exception as gemini_exc:
from backend.services.gemini_client import note_gemini_failure
note_gemini_failure("credit_evaluation", gemini_exc)
raise
evaluation = _parse_gemini_response(response_text)
evaluation.signals = signals
+28 -21
View File
@@ -562,36 +562,43 @@ class CustomLyricsService:
pdf_bytes: Optional[bytes],
settings: GenerationSettings,
) -> list[str]:
from google import genai
from google.genai import types
client = genai.Client(
vertexai=True,
project=self.settings.google_cloud_project,
location="global",
)
from backend.services.gemini_client import get_genai_client, note_gemini_failure
contents: list = [user_prompt]
if pdf_bytes is not None:
contents.append(
types.Part.from_bytes(mime_type="application/pdf", data=pdf_bytes)
)
response = client.models.generate_content(
model=self.settings.custom_lyrics_model,
contents=contents,
config=types.GenerateContentConfig(
system_instruction=system_prompt,
temperature=0.4,
response_mime_type="application/json",
response_schema={
"type": "object",
"properties": {
"lines": {"type": "array", "items": {"type": "string"}}
try:
# Bounded so a stalled call can't hang the request (Gemini Developer API).
client = get_genai_client(timeout_ms=300_000)
response = client.models.generate_content(
model=self.settings.custom_lyrics_model,
contents=contents,
config=types.GenerateContentConfig(
system_instruction=system_prompt,
temperature=0.4,
response_mime_type="application/json",
response_schema={
"type": "object",
"properties": {
"lines": {"type": "array", "items": {"type": "string"}}
},
"required": ["lines"],
},
"required": ["lines"],
},
),
)
),
)
except Exception as exc:
if note_gemini_failure("custom_lyrics", exc):
raise CustomLyricsServiceError(
"The AI lyrics service is temporarily unavailable. Please try "
"again later, or edit the lyrics manually.",
status_code=503,
) from exc
raise
# The model can return no text (safety block, MAX_TOKENS with no content,
# empty candidate). `response.text` is then None and json.loads(None) would
# raise a raw TypeError -> uncaught 500 without CORS headers -> the browser
-1
View File
@@ -110,7 +110,6 @@ Shared infrastructure:
# ---------------------------------------------------------------------------
LLM_ANALYSIS_MODEL: str = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-3.8-flash")
LLM_VERTEX_LOCATION: str = os.environ.get("LLM_VERTEX_LOCATION", "global")
#: Minimum number of active error patterns required before triggering LLM analysis.
MIN_PATTERNS_FOR_ANALYSIS: int = 3
@@ -1,6 +1,6 @@
"""LLM-powered incident analysis and duplicate detection for the error monitor.
Uses Gemini Flash via the google-generativeai library to:
Uses Gemini Flash via the Gemini Developer API (backend.services.gemini_client) to:
1. Group related error patterns into incidents with root-cause analysis.
2. Identify near-duplicate patterns that the regex normalizer missed.
@@ -17,13 +17,11 @@ import time
from collections import defaultdict
from dataclasses import dataclass, field
from google import genai
from google.genai import types as genai_types
from backend.services.gemini_client import get_genai_client, note_gemini_failure
from backend.services.error_monitor.config import (
GCP_PROJECT,
LLM_ANALYSIS_MODEL,
LLM_VERTEX_LOCATION,
MIN_PATTERNS_FOR_ANALYSIS,
SERVICE_DEPENDENCY_MAP,
)
@@ -96,11 +94,7 @@ def _call_llm(
for attempt in range(_RETRY_ATTEMPTS):
try:
client = genai.Client(
vertexai=True,
project=GCP_PROJECT,
location=LLM_VERTEX_LOCATION,
)
client = get_genai_client(timeout_ms=120_000)
response = client.models.generate_content(
model=LLM_ANALYSIS_MODEL,
contents=user_prompt,
@@ -111,6 +105,10 @@ def _call_llm(
)
return response.text
except Exception as exc: # noqa: BLE001
if note_gemini_failure("error_monitor", exc):
# Quota/credit/key problem: don't retry — callers fall back to
# alerts without the AI summary.
raise
last_exc = exc
delay = _RETRY_BASE_DELAY * (2**attempt)
logger.warning(
+208
View File
@@ -0,0 +1,208 @@
"""Shared Gemini client for the backend: Gemini Developer API (API key), not Vertex.
Every backend Gemini call goes through :func:`get_genai_client`. Vertex AI is
disabled in the nomadkaraoke GCP project so AI spend never lands on it; calls are
billed to whichever AI Studio key is stored in Secret Manager ``gemini-api-key``.
Key resolution (via :meth:`backend.config.Settings.get_secret`):
1. ``GEMINI_API_KEY`` env var — injected on Cloud Run from
``gemini-api-key:latest`` (CI ``--set-secrets`` for the service, Pulumi
for the jobs).
2. Secret Manager ``gemini-api-key`` (latest) — read directly with the
runtime service account / your ADC when the env var isn't set.
Rotating the key: add a new version of ``gemini-api-key``. New instances / job
executions pick it up; no code change.
Degradation: when the key's quota or prepaid credit is exhausted (or the key is
rejected), callers must degrade rather than fail jobs. Use
:func:`is_quota_or_billing_error` to classify and :func:`note_gemini_failure`
from the caller's ``except`` block — it also sends a Discord ops alert at most
once per ``GEMINI_QUOTA_ALERT_HOURS`` (default 6h) across all instances.
"""
from __future__ import annotations
import logging
import os
import re
import threading
import time
from typing import Any, Optional
logger = logging.getLogger(__name__)
SECRET_NAME = "gemini-api-key"
QUOTA_EXHAUSTED_MESSAGE = (
"Gemini quota/credit exhausted — top up AI Studio or rotate gemini-api-key "
"(add a new version of Secret Manager secret 'gemini-api-key' in project nomadkaraoke)."
)
# Firestore dedup doc id (in ops_alerts' collection) shared by every instance/job.
_ALERT_KEY = "gemini-quota-exhausted"
_DEFAULT_ALERT_HOURS = 6.0
class GeminiKeyUnavailableError(RuntimeError):
"""No Gemini API key could be resolved (env var unset and Secret Manager failed)."""
def get_api_key() -> str:
"""Return the Gemini Developer API key or raise GeminiKeyUnavailableError."""
env_key = os.environ.get("GEMINI_API_KEY", "").strip()
if env_key:
return env_key
from backend.config import get_settings
key = (get_settings().get_secret(SECRET_NAME) or "").strip()
if not key:
raise GeminiKeyUnavailableError(
"GEMINI_API_KEY is not set and Secret Manager secret 'gemini-api-key' "
"could not be read"
)
return key
def get_genai_client(timeout_ms: Optional[int] = None) -> Any:
"""Build a google-genai client for the Gemini Developer API.
``timeout_ms`` bounds each HTTP request (milliseconds). No project/location:
those only apply to Vertex AI.
"""
from google import genai
from google.genai import types
http_options = types.HttpOptions(timeout=int(timeout_ms)) if timeout_ms else None
# vertexai=False pins the Developer API even if GOOGLE_GENAI_USE_VERTEXAI is set.
return genai.Client(vertexai=False, api_key=get_api_key(), http_options=http_options)
# Message fragments that mean "the key/account can't pay or isn't valid" — a
# retry won't help; a human must top up credit or rotate the key.
_QUOTA_PATTERNS = re.compile(
r"RESOURCE_EXHAUSTED|exceeded your current quota|quota exceeded|"
r"billing (details|account)|prepa(id|yment)|credits? (are |is |have been )?"
r"(exhausted|depleted|used up)|insufficient (credit|funds|balance)|"
r"API_KEY_INVALID|API key not valid|API key expired|API_KEY_SERVICE_BLOCKED|"
r"\b429 Too Many Requests",
re.IGNORECASE,
)
# A 429 that names a per-minute quota / carries a retry delay is an ordinary
# rate limit (transient — retry with backoff), not exhausted credit.
_RATE_LIMIT_PATTERNS = re.compile(r"PerMinute|retryDelay|retry in \d", re.IGNORECASE)
# ...unless it names a per-day quota: that won't clear for hours — treat as exhausted.
_DAILY_QUOTA_PATTERN = re.compile(r"PerDay", re.IGNORECASE)
def _is_rate_limit(text: str) -> bool:
return bool(_RATE_LIMIT_PATTERNS.search(text)) and not _DAILY_QUOTA_PATTERN.search(text)
def is_quota_or_billing_error(exc: Optional[BaseException]) -> bool:
"""True if ``exc`` (or anything in its ``__cause__`` chain) is a Gemini quota,
prepaid-credit, billing or API-key error — i.e. needs a top-up/key rotation,
not a retry.
Matches google-genai ``APIError`` (``.code`` 429 / 403 or status
RESOURCE_EXHAUSTED / PERMISSION_DENIED, or a billing/key message on a 400),
a missing key, and wrapped/stringified errors from other layers (LangChain).
Per-minute rate limits (429 naming a ``PerMinute`` quota or a retry delay)
are NOT matched — those are transient; retry them.
"""
seen: set[int] = set()
while exc is not None and id(exc) not in seen:
seen.add(id(exc))
if isinstance(exc, GeminiKeyUnavailableError):
return True
code = getattr(exc, "code", None)
if code is None:
code = getattr(exc, "status_code", None)
status = str(getattr(exc, "status", "") or "")
if code == 429 or status == "RESOURCE_EXHAUSTED":
return not _is_rate_limit(str(exc))
if code == 403 or status == "PERMISSION_DENIED":
return True
text = str(exc)
if _QUOTA_PATTERNS.search(text) and not _is_rate_limit(text):
return True
# Only explicit chaining (`raise X from e`): implicit __context__ could
# attribute an unrelated earlier error (e.g. a Firestore 403) to Gemini.
exc = exc.__cause__
return False
def _alert_hours() -> float:
try:
return float(os.environ.get("GEMINI_QUOTA_ALERT_HOURS", _DEFAULT_ALERT_HOURS))
except (TypeError, ValueError):
return _DEFAULT_ALERT_HOURS
# In-process throttle so a burst of failures in one instance doesn't hit
# Firestore on every call; the Firestore doc throttles across instances/jobs.
_last_alert_attempt: float = 0.0
_alert_lock = threading.Lock()
# Alert I/O (Firestore + Secret Manager + Discord) runs on a daemon thread so it
# never blocks the caller — several callers are on the asyncio event loop.
# Tests flip this off to run it inline.
_ALERT_IN_BACKGROUND = True
def note_gemini_failure(caller: str, exc: BaseException) -> bool:
"""Classify a Gemini failure; on quota/billing/key errors log + alert.
Call from a caller's ``except`` block. Returns True when the error is a
quota/billing/key error (caller should skip retries and degrade). Never raises.
"""
try:
if not is_quota_or_billing_error(exc):
return False
logger.error("Gemini unavailable (quota/billing/key) in %s: %s", caller, exc)
_maybe_alert(caller, exc)
return True
except Exception: # noqa: BLE001 - classification/alerting must never raise
logger.warning("note_gemini_failure failed", exc_info=True)
return False
def _maybe_alert(caller: str, exc: BaseException) -> None:
global _last_alert_attempt
window = _alert_hours() * 3600
now = time.monotonic()
with _alert_lock:
if _last_alert_attempt and now - _last_alert_attempt < window:
return
_last_alert_attempt = now
if _ALERT_IN_BACKGROUND:
threading.Thread(
target=_send_alert, args=(caller, exc, window), name="gemini-quota-alert", daemon=True
).start()
else:
_send_alert(caller, exc, window)
def _send_alert(caller: str, exc: BaseException, window: float) -> None:
global _last_alert_attempt
from backend.services import ops_alerts
err = str(exc).strip()
if len(err) > 800:
err = err[:800] + " …[truncated]"
message = (
"🟠 **Gemini API unavailable — quota / prepaid credit / key problem**\n"
f"**First seen in:** `{caller}`\n"
"Callers are degrading (auto-correct keeps Opus only, match-judge/parse-titles "
"return no suggestion, credit evals go to manual review, translations are "
"skipped).\n"
f"**Fix:** {QUOTA_EXHAUSTED_MESSAGE}\n"
f"**Error:** ```{err}```"
)
try:
ops_alerts.send_throttled_ops_alert(_ALERT_KEY, message, throttle_minutes=int(window / 60))
except Exception: # noqa: BLE001 - alerting must never raise
logger.warning("Gemini quota alert failed", exc_info=True)
# Let the next failure in this instance try again.
with _alert_lock:
_last_alert_attempt = 0.0
+11 -7
View File
@@ -176,15 +176,11 @@ class LyricsTranslationService:
# ---- model ----
def _call_gemini(self, user_prompt: str) -> Any:
from google import genai
from google.genai import types
client = genai.Client(
vertexai=True,
project=self.settings.google_cloud_project,
location="global",
http_options=types.HttpOptions(timeout=90_000), # 2 attempts: render waits <= ~3 min
)
from backend.services.gemini_client import get_genai_client
client = get_genai_client(timeout_ms=90_000) # 2 attempts: render waits <= ~3 min
response = client.models.generate_content(
model=self.model,
contents=[user_prompt],
@@ -245,6 +241,14 @@ class LyricsTranslationService:
except (LyricsTranslationError, ValueError) as exc: # malformed output: retry
last_error = exc
except Exception as exc:
from backend.services.gemini_client import note_gemini_failure
if note_gemini_failure("lyrics_translation", exc):
# Quota/credit/key problem: don't retry; the render goes ahead
# without the translation row.
raise LyricsTranslationError(
"translation service unavailable (Gemini quota/billing) — rendered without translation"
) from exc
if not _is_transient(exc):
raise LyricsTranslationError(f"translation model error: {exc}") from exc
last_error = exc
+3 -7
View File
@@ -149,18 +149,14 @@ async def _default_generate(model: str, system_prompt: str, user_prompt: str) ->
def _blocking_generate(model: str, system_prompt: str, user_prompt: str) -> dict:
import copy
from google import genai
from google.genai import types
from backend.config import settings
from backend.services.gemini_client import get_genai_client
timeout_ms = int(getattr(settings, "match_judge_timeout_ms", 3000))
client = genai.Client(
vertexai=True,
project=settings.google_cloud_project,
location="global",
http_options=types.HttpOptions(timeout=timeout_ms),
)
client = get_genai_client(timeout_ms=timeout_ms)
response = client.models.generate_content(
model=model,
contents=[user_prompt],
+6 -7
View File
@@ -141,6 +141,9 @@ async def resolve_free_text(query: str, *, generate: Optional[Generate] = None,
data = await (generate or _default_generate)(
model or _model(), _SYSTEM_PROMPT, f'Singer typed: "{query}"')
except Exception as e:
from backend.services.gemini_client import note_gemini_failure
note_gemini_failure("match_judge_free_text", e)
logger.warning(f"resolve_free_text failed: {e}")
return _none(query, "unavailable")
return verdict_from_response(data, query)
@@ -200,17 +203,13 @@ async def _default_generate(model: str, system_prompt: str, user_prompt: str) ->
def _blocking_generate(model: str, system_prompt: str, user_prompt: str) -> dict:
from google import genai
from google.genai import types
from backend.config import settings
client = genai.Client(
vertexai=True,
project=settings.google_cloud_project,
location="global",
http_options=types.HttpOptions(timeout=int(getattr(settings, "match_judge_timeout_ms", 12000))),
)
from backend.services.gemini_client import get_genai_client
client = get_genai_client(timeout_ms=int(getattr(settings, "match_judge_timeout_ms", 12000)))
response = client.models.generate_content(
model=model,
contents=[user_prompt],
+4 -1
View File
@@ -126,7 +126,10 @@ async def judge_match(
try:
return await ai(artist, title, candidates, audio_tier)
except Exception:
except Exception as exc:
from backend.services.gemini_client import note_gemini_failure
note_gemini_failure("match_judge", exc)
# Graceful degradation: returning no-suggestion is a safe fallback (the
# user just gets no auto-match). Log at WARNING (with traceback) so a
# transient AI blip does not page as a red "new error pattern".
+35 -2
View File
@@ -134,7 +134,9 @@ def _signature(error_message: str) -> str:
return hashlib.sha256((error_message or "").encode("utf-8")).hexdigest()
def _should_alert(db: Any, signature: str, now: datetime) -> tuple[bool, bool, int, Any]:
def _should_alert(
db: Any, signature: str, now: datetime, throttle_minutes: Optional[int] = None
) -> tuple[bool, bool, int, Any]:
"""Decide whether to send, WITHOUT yet acknowledging delivery.
Returns ``(should_send, is_novel, suppressed_since_last, doc_ref)``:
@@ -182,7 +184,8 @@ def _should_alert(db: Any, signature: str, now: datetime) -> tuple[bool, bool, i
if last_alerted_raw:
try:
last_alerted = datetime.fromisoformat(last_alerted_raw)
within_window = (now - last_alerted) < timedelta(minutes=_throttle_minutes())
window = throttle_minutes if throttle_minutes is not None else _throttle_minutes()
within_window = (now - last_alerted) < timedelta(minutes=window)
except (TypeError, ValueError):
within_window = False
@@ -225,6 +228,36 @@ def _mark_alerted(doc_ref: Any, now: datetime) -> None:
logger.debug("ops_alerts: could not record alert delivery: %s", exc)
def send_throttled_ops_alert(key: str, message: str, throttle_minutes: int) -> bool:
"""Send an ops alert at most once per ``throttle_minutes`` for ``key``.
The throttle lives in Firestore (same dedup collection as job-failure
alerts), so it holds across every Cloud Run instance and job execution.
Fails open (sends) if Firestore is unreachable. Best-effort; never raises.
Returns True if an alert was sent.
"""
try:
if not _is_enabled():
return False
from backend.services.firestore_service import get_firestore_client
now = datetime.now(timezone.utc)
should_send, _novel, suppressed, doc_ref = _should_alert(
get_firestore_client(), key, now, throttle_minutes=throttle_minutes
)
if not should_send:
return False
if suppressed:
message += f"\n_(Collapsed {suppressed} earlier occurrence(s).)_"
sent = send_ops_alert(message)
if sent:
_mark_alerted(doc_ref, now)
return sent
except Exception as exc: # noqa: BLE001 - alerting must never raise
logger.warning("ops_alerts: throttled alert %s failed: %s", key, exc)
return False
def _retry_pending(state_data: dict) -> bool:
"""True if state_data.cloud_run_retry_pending is set and unexpired."""
pending = (state_data or {}).get("cloud_run_retry_pending")
+3 -7
View File
@@ -123,18 +123,14 @@ async def _default_generate(model: str, system: str, user: str) -> dict:
def _blocking_generate(model: str, system: str, user: str) -> dict:
from google import genai
from google.genai import types
from backend.config import settings
from backend.services.gemini_client import get_genai_client
timeout_ms = int(getattr(settings, "parse_titles_timeout_ms", 20000))
client = genai.Client(
vertexai=True,
project=settings.google_cloud_project,
location="global",
http_options=types.HttpOptions(timeout=timeout_ms),
)
client = get_genai_client(timeout_ms=timeout_ms)
response = client.models.generate_content(
model=model,
contents=[user],
+3
View File
@@ -56,6 +56,9 @@ async def _parse_chunk(chunk: list[dict], *, model, generate) -> list[dict]:
try:
results = await ai.ai_parse(chunk, model=model, generate=generate)
except Exception as exc:
from backend.services.gemini_client import note_gemini_failure
note_gemini_failure("parse_titles", exc)
logger.warning(
"parse_titles chunk degraded (%s); blanks for %d items", exc, len(chunk))
return _blanks(chunk)
+8 -9
View File
@@ -431,6 +431,9 @@ def analyze_filenames(
llm_rows, leftover = _llm_analyze(leftover, generate)
rows.extend(llm_rows)
except Exception as e: # pragma: no cover - resilience guard
from backend.services.gemini_client import note_gemini_failure
note_gemini_failure("tenant_bulk_analyze", e)
logger.warning("Bulk-analyze LLM pass failed, using regex result: %s", e)
unpaired = [_describe_unpaired(fn) for fn in leftover]
@@ -438,7 +441,7 @@ def analyze_filenames(
# ---------------------------------------------------------------------------
# Default Vertex-Gemini generate (mirrors backend.services.match_judge.ai)
# Default Gemini generate (mirrors backend.services.match_judge.ai)
# ---------------------------------------------------------------------------
def _default_model() -> str:
@@ -451,22 +454,18 @@ def _default_model() -> str:
def default_generate(system_prompt: str, user_prompt: str) -> dict:
"""Blocking Vertex-Gemini call producing schema-constrained JSON.
"""Blocking Gemini (Developer API) call producing schema-constrained JSON.
Wrap in ``asyncio.to_thread`` when calling from async code.
"""
from google import genai
from google.genai import types
from backend.config import settings
from backend.services.gemini_client import get_genai_client
timeout_ms = int(getattr(settings, "tenant_bulk_timeout_ms", 30000))
client = genai.Client(
vertexai=True,
project=settings.google_cloud_project,
location="global",
http_options=types.HttpOptions(timeout=timeout_ms),
)
client = get_genai_client(timeout_ms=timeout_ms)
response = client.models.generate_content(
model=_default_model(),
contents=[user_prompt],
+2
View File
@@ -18,6 +18,8 @@ if 'FIRESTORE_EMULATOR_HOST' not in os.environ:
os.environ.setdefault('GCS_BUCKET_NAME', 'test-bucket')
os.environ.setdefault('FIRESTORE_COLLECTION', 'jobs')
os.environ.setdefault('ENVIRONMENT', 'test')
# Gemini Developer API key: never let a test fall through to Secret Manager.
os.environ.setdefault('GEMINI_API_KEY', 'test-gemini-key')
import pytest
from unittest.mock import Mock, MagicMock, AsyncMock, patch
@@ -0,0 +1,304 @@
"""Tests for backend.services.gemini_client (Gemini Developer API helper) and the
per-caller graceful degradation when Gemini's quota/credit/key is unusable."""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from backend.services import gemini_client
from backend.services.gemini_client import (
GeminiKeyUnavailableError,
get_api_key,
get_genai_client,
is_quota_or_billing_error,
note_gemini_failure,
)
class FakeAPIError(Exception):
"""Mimics google.genai.errors.APIError (numeric ``code`` + ``status``)."""
def __init__(self, code, status, message=""):
super().__init__(f"{code} {status}. {message}")
self.code = code
self.status = status
QUOTA = FakeAPIError(429, "RESOURCE_EXHAUSTED", "Your prepayment credits are depleted")
RATE_LIMIT = FakeAPIError(
429, "RESOURCE_EXHAUSTED",
"quotaId: GenerateRequestsPerMinutePerProjectPerModel-PaidTier, retryDelay: 5s",
)
@pytest.fixture(autouse=True)
def _reset_alert_throttle(monkeypatch):
monkeypatch.setattr(gemini_client, "_last_alert_attempt", 0.0)
monkeypatch.setattr(gemini_client, "_ALERT_IN_BACKGROUND", False)
# ---- classifier -------------------------------------------------------------
@pytest.mark.parametrize(
"exc",
[
QUOTA,
FakeAPIError(429, "RESOURCE_EXHAUSTED", "You exceeded your current quota"),
FakeAPIError(403, "PERMISSION_DENIED", "Generative Language API has not been used"),
FakeAPIError(400, "INVALID_ARGUMENT", "API key not valid. Please pass a valid API key."),
FakeAPIError(400, "FAILED_PRECONDITION", "check your plan and billing details"),
RuntimeError("API key expired. Please renew the API key."),
GeminiKeyUnavailableError("no key"),
RuntimeError("429 RESOURCE_EXHAUSTED quotaId GenerateRequestsPerDayPerProjectPerModel, retryDelay 3600s"),
],
)
def test_classifier_positive(exc):
assert is_quota_or_billing_error(exc)
@pytest.mark.parametrize(
"exc",
[
RATE_LIMIT,
FakeAPIError(500, "INTERNAL", "Internal error"),
FakeAPIError(503, "UNAVAILABLE", "The model is overloaded"),
FakeAPIError(400, "INVALID_ARGUMENT", "Request contains an invalid argument."),
TimeoutError("timed out"),
ValueError("processed 429 tokens"),
None,
],
)
def test_classifier_negative(exc):
assert not is_quota_or_billing_error(exc)
def test_classifier_walks_cause_chain():
try:
try:
raise QUOTA
except FakeAPIError as inner:
raise RuntimeError("wrapped") from inner
except RuntimeError as outer:
assert is_quota_or_billing_error(outer)
# ---- key + client -------------------------------------------------------------
def test_env_key_wins(monkeypatch):
monkeypatch.setenv("GEMINI_API_KEY", " env-key ")
with patch("backend.config.Settings.get_secret") as get_secret:
assert get_api_key() == "env-key"
get_secret.assert_not_called()
def test_falls_back_to_secret_manager(monkeypatch):
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
with patch("backend.config.Settings.get_secret", return_value="sm-key") as get_secret:
assert get_api_key() == "sm-key"
get_secret.assert_called_once_with("gemini-api-key")
def test_missing_key_raises(monkeypatch):
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
with patch("backend.config.Settings.get_secret", return_value=None):
with pytest.raises(GeminiKeyUnavailableError):
get_api_key()
def test_client_uses_api_key_never_vertex(monkeypatch):
monkeypatch.setenv("GEMINI_API_KEY", "k")
with patch("google.genai.Client") as client_cls:
get_genai_client(timeout_ms=1234)
kwargs = client_cls.call_args.kwargs
assert kwargs["api_key"] == "k"
assert kwargs["http_options"].timeout == 1234
assert kwargs["vertexai"] is False and "project" not in kwargs and "location" not in kwargs
# ---- note_gemini_failure / alert throttle --------------------------------------
def test_note_failure_ignores_ordinary_errors():
with patch.object(gemini_client, "_maybe_alert") as alert:
assert note_gemini_failure("x", RATE_LIMIT) is False
alert.assert_not_called()
def test_note_failure_alerts_once_per_window():
with patch("backend.services.ops_alerts.send_throttled_ops_alert", return_value=True) as send:
assert note_gemini_failure("auto_correct", QUOTA) is True
assert note_gemini_failure("match_judge", QUOTA) is True
send.assert_called_once()
key, message = send.call_args.args
assert key == "gemini-quota-exhausted"
assert "auto_correct" in message and "rotate gemini-api-key" in message
assert send.call_args.kwargs["throttle_minutes"] == 360
def test_note_failure_never_raises_and_rearms_after_failed_send():
with patch("backend.services.ops_alerts.send_throttled_ops_alert", side_effect=RuntimeError) as send:
assert note_gemini_failure("x", QUOTA) is True
assert note_gemini_failure("x", QUOTA) is True
# A failed send doesn't silence the instance for 6h.
assert send.call_count == 2
def test_alert_runs_in_background_thread(monkeypatch):
import threading
monkeypatch.setattr(gemini_client, "_ALERT_IN_BACKGROUND", True)
done = threading.Event()
seen = {}
def fake_send(*_a, **_k):
seen["thread"] = threading.current_thread().name
done.set()
return True
with patch("backend.services.ops_alerts.send_throttled_ops_alert", side_effect=fake_send):
assert note_gemini_failure("x", QUOTA) is True
assert done.wait(5)
assert seen["thread"] == "gemini-quota-alert"
def test_throttled_ops_alert_respects_firestore_window(monkeypatch):
from backend.services import ops_alerts
monkeypatch.setenv("FAILURE_ALERTS_ENABLED", "true")
db = MagicMock()
with patch("backend.services.firestore_service.get_firestore_client", return_value=db), \
patch.object(ops_alerts, "_should_alert", return_value=(False, False, 0, None)) as should, \
patch.object(ops_alerts, "send_ops_alert") as send:
assert ops_alerts.send_throttled_ops_alert("k", "msg", throttle_minutes=360) is False
send.assert_not_called()
assert should.call_args.kwargs["throttle_minutes"] == 360
with patch("backend.services.firestore_service.get_firestore_client", return_value=db), \
patch.object(ops_alerts, "_should_alert", return_value=(True, False, 2, "ref")), \
patch.object(ops_alerts, "send_ops_alert", return_value=True) as send, \
patch.object(ops_alerts, "_mark_alerted") as mark:
assert ops_alerts.send_throttled_ops_alert("k", "msg", throttle_minutes=360) is True
assert "Collapsed 2" in send.call_args.args[0]
mark.assert_called_once()
# ---- per-caller degradation ----------------------------------------------------
def _quota_client():
client = MagicMock()
client.models.generate_content.side_effect = QUOTA
return client
def test_auto_correct_quota_is_non_retryable_and_skips_gemini_leg():
from backend.services.auto_correct.service import AutoCorrectService, AutoCorrectServiceError
service = AutoCorrectService()
with patch("backend.services.gemini_client.get_genai_client", return_value=_quota_client()), \
patch.object(gemini_client, "_maybe_alert"), \
patch("backend.services.auto_correct.service.time.sleep") as sleep:
with pytest.raises(AutoCorrectServiceError) as ei:
service._call_model("gemini-3.8-flash", "sys", "user", job_id="j")
assert ei.value.retryable is False and ei.value.status_code == 503
sleep.assert_not_called()
def test_auto_correct_compare_keeps_opus_when_gemini_quota_exhausted():
from backend.services.auto_correct.service import AutoCorrectService, AutoCorrectServiceError
from backend.services.auto_correct.settings import AutoCorrectSettings
segments = [{"id": "s1", "words": [{"id": "w0", "text": "glory"}]}]
refs = {"genius": {"segments": [{"text": "chlorine"}]}}
sugg = {"op": "replace", "start_idx": 0, "end_idx": 0, "new_text": "chlorine",
"reason": "r", "category": "mishearing", "confidence": 0.9}
def fake_call(model, *_a, **_k):
if model.startswith("gemini"):
raise AutoCorrectServiceError("Gemini is unavailable", status_code=503)
return {"suggestions": [sugg]}, None
service = AutoCorrectService()
with patch.object(service.settings, "auto_correct_compare_models", "claude-opus-5-5;gemini-3.8-flash"), \
patch.object(service, "_cache_get", return_value=None), \
patch.object(service, "_cache_put"), \
patch.object(service, "_call_model", side_effect=fake_call):
result = service.suggest(
job_id="j", segments=segments, reference_lyrics=refs, artist="A", title="T",
settings=AutoCorrectSettings(compare_models=True),
)
assert result.model == "claude-opus-5-5"
assert any("gemini-3.8-flash failed" in w for w in result.warnings)
@pytest.mark.asyncio
async def test_match_judge_quota_returns_no_suggestion_and_alerts():
from backend.services.match_judge import service as mj
async def no_candidates(*_a, **_k):
return []
async def boom(*_a, **_k):
raise QUOTA
with patch.object(gemini_client, "_maybe_alert") as alert:
verdict = await mj.judge_match(
"A", "T", search_tracks=no_candidates, ai_judge=boom, enabled=True
)
assert verdict.reason == "ai failed"
alert.assert_called_once()
def test_custom_lyrics_quota_is_clean_503():
from backend.services.custom_lyrics.settings import GenerationSettings
from backend.services.custom_lyrics_service import CustomLyricsService, CustomLyricsServiceError
service = CustomLyricsService()
with patch("backend.services.gemini_client.get_genai_client", return_value=_quota_client()), \
patch.object(gemini_client, "_maybe_alert"):
with pytest.raises(CustomLyricsServiceError) as ei:
service._call_gemini(system_prompt="s", user_prompt="u", pdf_bytes=None,
settings=GenerationSettings())
assert ei.value.status_code == 503
assert "temporarily unavailable" in str(ei.value)
def test_credit_eval_quota_goes_to_manual_review():
from backend.services.credit_evaluation_service import CreditEvaluationService
svc = CreditEvaluationService.__new__(CreditEvaluationService)
svc.settings = MagicMock(credit_eval_enabled=True, credit_eval_model="gemini-3.8-flash")
signals = {"fingerprint_matches": [{"x": 1}], "ip_matches": [], "recent_signups_ip": 0,
"recent_signups_fp": 0, "user_data": {}, "ip_geo": None, "user_agent": None}
with patch.object(svc, "_collect_signals", return_value=signals), \
patch.object(svc, "_log_evaluation"), \
patch("backend.services.gemini_client.get_genai_client", return_value=_quota_client()), \
patch.object(gemini_client, "_maybe_alert") as alert:
evaluation = svc.evaluate("a@b.c", "welcome")
assert evaluation.decision == "pending_review"
alert.assert_called_once()
def test_error_monitor_llm_does_not_retry_on_quota():
from backend.services.error_monitor import llm_analysis
with patch.object(llm_analysis, "get_genai_client", return_value=_quota_client()), \
patch.object(gemini_client, "_maybe_alert"), \
patch.object(llm_analysis.time, "sleep") as sleep:
with pytest.raises(FakeAPIError):
llm_analysis._call_llm("sys", "user")
sleep.assert_not_called()
def test_classifier_ignores_implicit_context():
"""A Gemini-unrelated 403 raised earlier must not be blamed on Gemini."""
try:
try:
raise FakeAPIError(403, "PERMISSION_DENIED", "firestore says no")
except FakeAPIError:
raise ValueError("later, unrelated failure") # implicit __context__ only
except ValueError as exc:
assert not is_quota_or_billing_error(exc)
+6 -5
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import io
import os
import json
from unittest.mock import MagicMock, patch
@@ -103,11 +104,11 @@ def test_text_only_happy_path(service: CustomLyricsService) -> None:
assert result.line_count_mismatch is False
assert result.iterations_used == 0
assert result.model == "gemini-3.1-pro-preview"
assert mock_client_cls.call_args.kwargs == {
"vertexai": True,
"project": "test-project",
"location": "global",
}
# Gemini Developer API (API key) — never Vertex AI.
kwargs = mock_client_cls.call_args.kwargs
assert kwargs["api_key"] == os.environ["GEMINI_API_KEY"]
assert kwargs["vertexai"] is False and "project" not in kwargs
assert kwargs["http_options"].timeout == 300_000
def test_docx_input_parsed_to_text(
+15 -1
View File
@@ -106,7 +106,8 @@ def test_gives_up_after_max_attempts():
def test_transient_error_retried_permanent_error_raised():
transient = Exception("429 RESOURCE_EXHAUSTED")
# Per-minute rate limit: transient, retried.
transient = Exception("429 RESOURCE_EXHAUSTED quotaId GenerateRequestsPerMinutePerProjectPerModel")
svc = make_service([transient, {"translations": ["uno"]}])
assert svc.translate_lines(["one"], "es") == ["uno"]
@@ -116,6 +117,19 @@ def test_transient_error_retried_permanent_error_raised():
assert svc._call_gemini.call_count == 1
def test_quota_exhausted_not_retried_and_alerts():
class QuotaError(Exception):
code = 403
status = "PERMISSION_DENIED"
svc = make_service([QuotaError("Your prepayment credits are depleted")])
with patch("backend.services.gemini_client._maybe_alert") as alert:
with pytest.raises(LyricsTranslationError, match="unavailable"):
svc.translate_lines(["one"], "es")
assert svc._call_gemini.call_count == 1
alert.assert_called_once()
def test_unsupported_language_raises():
with pytest.raises(LyricsTranslationError):
make_service([]).translate_lines(["one"], "xx")
+3 -3
View File
@@ -736,7 +736,7 @@ Content-Type: application/json
AI correction suggestions for the lyrics review UI. Stateless: one whole-song
LLM call per model (Claude models use the Anthropic API via the
`anthropic-api-key` secret, Gemini models use Vertex AI) compares the
`anthropic-api-key` secret, Gemini models use the Gemini Developer API, key `gemini-api-key`) compares the
client's current transcription against the reference sources and returns
word-id-keyed suggestions. Nothing is applied server-side — the reviewer
accepts/rejects each suggestion in the UI and persistence flows through the
@@ -1017,7 +1017,7 @@ Content-Type: application/json
```
Decides the official formatting for a typed artist/title and whether it matches a
real song. A deterministic normalizer + the catalog run first; a light Vertex
real song. A deterministic normalizer + the catalog run first; a light Gemini
Gemini model (`MATCH_JUDGE_MODEL`, default `gemini-3.8-flash`) is consulted
**only** when those aren't confident. `audio_confidence_tier` (1=strong..3=weak,
optional) lets the judge tell whether weak audio results hint at a typo. The call
@@ -1340,7 +1340,7 @@ Artist/Title. Pure analysis — filenames only, no uploads, no state written.
- **Auth:** tenant session required; gated on the tenant's `features.bulk_upload` flag (403 otherwise).
- **Limits:** max 100 audio files per request (non-audio files don't count toward the cap; 2000-filename payload guard).
- **Two passes:** deterministic regex for the `S<code>-<1|2> Artist - Title <Guide|BV|Instru>`
convention, then a Vertex-Gemini pass (`backend/services/tenant_bulk/analyze.py`) for
convention, then a Gemini pass (`backend/services/tenant_bulk/analyze.py`) for
whatever the regex couldn't confidently pair. Falls back to regex-only if the model errors.
Response:
+1 -1
View File
@@ -252,7 +252,7 @@ LyricsTranscriber LyricsTranscriber
|---------|---------|----------|
| Cloud Run GPU Job (L4, us-east4) | Audio stem separation — `Separator` runs directly in the audio worker Job (no HTTP hop) | Yes |
| AudioShake | Lyrics transcription | Yes |
| Vertex AI | Agentic AI correction (Gemini 3.8 Flash) | Default off (SKIP_CORRECTION=false to enable) |
| Gemini Developer API (`gemini-api-key`) | Agentic AI correction (Gemini 3.8 Flash) | Default off (SKIP_CORRECTION=false to enable) |
| Genius | Reference lyrics | Yes |
| Flacfetch | Audio downloads (YouTube, torrents) | Recommended* |
| YouTube API | Video upload | Optional |
+1 -1
View File
@@ -303,7 +303,7 @@ Some external services lock to redirect URIs or treat the old GCP project ID as
- YouTube OAuth (redirect URIs)
- Stripe webhook endpoint URL (secret regenerates)
- AudioShake API key (rotate as a precaution if old project was compromised)
- Gemini / Vertex API (project-scoped — needs new project enabled)
- Gemini Developer API key (`gemini-api-key` secret; key lives in an AI Studio project outside nomadkaraoke — just copy the secret)
- Discord webhook URLs (re-issue if old GCP exposure could have leaked them)
- Cloudflare API tokens (rotate)
+11 -5
View File
@@ -1403,11 +1403,17 @@ For indefinite loops (like polling), use outer timeout (total duration) + inner
### Secret Manager Access
Service account needs `roles/secretmanager.secretAccessor`. Mount as env var or volume. Use GCP workload identity, not key files.
### Vertex AI Auth
Use `project` parameter with `ChatGoogleGenerativeAI` to trigger Vertex AI backend with ADC. Don't require `GOOGLE_API_KEY`.
### Gemini 3 Requires Global Location
Gemini 3 models require `location="global"`, not regional endpoints like `us-central1`.
### Gemini: Developer API key, never Vertex (2026-10-01)
All Gemini calls use the **Gemini Developer API** (AI Studio key in Secret Manager
`gemini-api-key`, billed outside the nomadkaraoke project) via
`backend/services/gemini_client.get_genai_client()`; `aiplatform.googleapis.com` and
`generativelanguage.googleapis.com` are **disabled** in nomadkaraoke so no AI spend lands
on it. Key = `GEMINI_API_KEY` env (CI `--set-secrets` for the service, Pulumi for jobs),
else Secret Manager read at runtime. Rotate = add a secret version (new instances/job
executions pick it up). Classify failures with `note_gemini_failure(caller, exc)`:
quota/billing/key errors (not per-minute 429s) skip retries, degrade, and Discord-alert
at most once per 6h. (Supersedes the old "Vertex AI Auth" / "global location" notes —
the Developer API has no project/location.)
### LangChain: REST over gRPC
Use `langchain-google-genai` (REST) instead of `langchain-google-vertexai` (gRPC) to avoid silent hangs.
+2 -2
View File
@@ -132,7 +132,7 @@ Commercial producers pay for API access or custom integration to generate tracks
### What Works
- Audio upload and separation (Cloud Run GPU)
- Lyrics transcription (AudioShake API)
- Agentic AI correction (Gemini 3 Flash via Vertex AI)
- Agentic AI correction (Gemini 3 Flash via the Gemini Developer API)
- Human lyrics review (React UI)
- Preview video generation
- Instrumental selection (clean vs. with backing vocals)
@@ -190,7 +190,7 @@ Commercial producers pay for API access or custom integration to generate tracks
- **Secrets**: Google Secret Manager
- **Audio Separation**: Cloud Run GPU (L4, us-east4)
- **Lyrics Transcription**: AudioShake API
- **AI Correction**: Vertex AI (Gemini 3 Flash)
- **AI Correction**: Gemini Developer API (Gemini 3 Flash)
- **Payments**: Stripe
- **Email**: Postmark
- **IaC**: Pulumi
+2
View File
@@ -57,6 +57,8 @@
## Recent Changes
- **Gemini moved off Vertex AI** (2026-10-01, v0.262.0): every Gemini call (auto-correct compare leg, match-judge, parse-titles, credit eval, tenant bulk analyze, custom lyrics, lyrics translation, error-monitor LLM, agentic correction, translate.py) now uses the Gemini Developer API with the `gemini-api-key` secret via `backend/services/gemini_client.py`, so no AI spend lands on the nomadkaraoke project (Vertex APIs disabled there). Quota/billing/key failures degrade per caller and send one Discord ops alert per 6h. See LESSONS-LEARNED "Gemini: Developer API key, never Vertex".
- **Translated lyrics follow-ups + review waveform spinner** (2026-10-01, v0.261.0):
- The translation row now renders at 70% of the unsung colour's opacity, with outline and shadow faded to match, so it reads as secondary to the sung line (`subtitles.py:TRANSLATION_OPACITY`). A theme's `karaoke.translation_color` still wins.
- Published videos with translations get "(With Translation into X)" in the YouTube title, before "(Karaoke)". Only the artist/title get truncated, never the label. The description also gets a notice. This applies to the live upload and the quota queue, but only when `state_data.lyrics_translation.status == "translated"` (`youtube_description.build_youtube_title`).
+1 -1
View File
@@ -99,7 +99,7 @@ the same KaraokeNerds check the normal job-submission flow uses runs at two poin
- **Auth:** reuses gen's passwordless magic-link system. Board sign-in sends
`purpose="requests_board"`, which creates an identity **without** granting a welcome credit and
with a higher per-IP signup cap (so a whole venue on shared WiFi can sign in).
- **Auto-correct:** reuses `match_judge` (Vertex Gemini) — the same artist/title canonicalizer the
- **Auto-correct:** reuses `match_judge` (Gemini) — the same artist/title canonicalizer the
job-submission flow uses.
- **Storage:** two Firestore collections — `song_requests` (one doc per song, with a denormalized
`vote_count`) and `song_request_votes` (one doc per person per day, id `{email}__{YYYY-MM-DD}`,
+142
View File
@@ -0,0 +1,142 @@
"""Gemini Developer API (Google AI Studio) client helper for local scripts.
All Gemini calls go through the Developer API with an API key, NOT Vertex AI
(Vertex is disabled in the nomadkaraoke GCP project to keep AI spend off it).
Key resolution (first match wins):
1. ``GEMINI_API_KEY`` environment variable
2. ``gcloud secrets versions access latest --secret=gemini-api-key
--project=nomadkaraoke`` (your own gcloud login)
The key is cached in-process only; it is never written to disk or printed.
Rotating the key (e.g. prepaid credit exhausted): add a new secret version —
no code change needed:
printf '%s' "$NEW_KEY" | gcloud secrets versions add gemini-api-key \
--project=nomadkaraoke --data-file=-
"""
from __future__ import annotations
import os
import re
import subprocess
SECRET_NAME = "gemini-api-key"
SECRET_PROJECT = "nomadkaraoke"
QUOTA_EXHAUSTED_MESSAGE = (
"Gemini quota/credit exhausted — top up AI Studio or rotate gemini-api-key "
f"(add a new version of Secret Manager secret '{SECRET_NAME}' in project "
f"'{SECRET_PROJECT}')."
)
def quota_exhausted_message() -> str:
"""QUOTA_EXHAUSTED_MESSAGE plus a hint when the key came from the environment."""
if os.environ.get("GEMINI_API_KEY", "").strip():
return (
QUOTA_EXHAUSTED_MESSAGE
+ " Note: the key came from the GEMINI_API_KEY environment variable — "
"unset it to use the Secret Manager key instead."
)
return QUOTA_EXHAUSTED_MESSAGE
class GeminiKeyUnavailableError(RuntimeError):
"""The Gemini API key could not be resolved (env var unset and gcloud failed)."""
_cached_key: str | None = None
def get_api_key() -> str:
"""Return the Gemini Developer API key (env var, else Secret Manager via gcloud)."""
global _cached_key
env_key = os.environ.get("GEMINI_API_KEY", "").strip()
if env_key:
return env_key
if _cached_key:
return _cached_key
cmd = [
"gcloud", "secrets", "versions", "access", "latest",
f"--secret={SECRET_NAME}", f"--project={SECRET_PROJECT}",
]
try:
result = subprocess.run(cmd, capture_output=True, text=True, timeout=60, check=False)
except (OSError, subprocess.TimeoutExpired) as e:
raise GeminiKeyUnavailableError(
f"GEMINI_API_KEY is not set and `gcloud secrets versions access` failed: {e}"
) from e
key = (result.stdout or "").strip()
if result.returncode != 0 or not key:
stderr = (result.stderr or "").strip()[-300:]
raise GeminiKeyUnavailableError(
"GEMINI_API_KEY is not set and reading Secret Manager secret "
f"'{SECRET_NAME}' (project {SECRET_PROJECT}) failed — run `gcloud auth login` "
f"or export GEMINI_API_KEY. gcloud said: {stderr}"
)
_cached_key = key
return key
def get_genai_client():
"""Build a google-genai client for the Gemini Developer API."""
from google import genai
# vertexai=False pins the Developer API even if GOOGLE_GENAI_USE_VERTEXAI is set.
return genai.Client(vertexai=False, api_key=get_api_key())
# Message fragments that mean "the key/account can't pay or isn't valid" — a
# retry won't help; a human must top up credit or rotate the key.
_QUOTA_PATTERNS = re.compile(
r"RESOURCE_EXHAUSTED|exceeded your current quota|quota exceeded|"
r"billing (details|account)|prepa(id|yment)|credits? (are |is |have been )?"
r"(exhausted|depleted|used up)|insufficient (credit|funds|balance)|"
r"API_KEY_INVALID|API key not valid|API key expired|API_KEY_SERVICE_BLOCKED|"
r"\b429 Too Many Requests",
re.IGNORECASE,
)
# A 429 that names a per-minute quota / carries a retry delay is an ordinary
# rate limit (transient — retry with backoff), not exhausted credit.
_RATE_LIMIT_PATTERNS = re.compile(r"PerMinute|retryDelay|retry in \d", re.IGNORECASE)
# ...unless it names a per-day quota: that won't clear for hours — treat as exhausted.
_DAILY_QUOTA_PATTERN = re.compile(r"PerDay", re.IGNORECASE)
def _is_rate_limit(text: str) -> bool:
return bool(_RATE_LIMIT_PATTERNS.search(text)) and not _DAILY_QUOTA_PATTERN.search(text)
def is_quota_or_billing_error(exc: BaseException | None) -> bool:
"""True if ``exc`` (or anything in its ``__cause__`` chain) is a Gemini quota,
prepaid-credit, billing or API-key error — i.e. needs a top-up/key rotation,
not a retry.
Matches google-genai ``APIError`` (``.code`` 429 / 403, or 400 with a
billing/key message) and wrapped/stringified errors from other layers.
Per-minute rate limits (429 naming a ``PerMinute`` quota or a retry delay)
are NOT matched — those are transient; retry them.
"""
seen: set[int] = set()
while exc is not None and id(exc) not in seen:
seen.add(id(exc))
if isinstance(exc, GeminiKeyUnavailableError):
return True
code = getattr(exc, "code", None)
if code is None:
code = getattr(exc, "status_code", None)
status = str(getattr(exc, "status", "") or "")
if code == 429 or status == "RESOURCE_EXHAUSTED":
return not _is_rate_limit(str(exc))
if code == 403 or status == "PERMISSION_DENIED":
return True
text = str(exc)
if _QUOTA_PATTERNS.search(text) and not _is_rate_limit(text):
return True
# Only explicit chaining (`raise X from e`): implicit __context__ could
# attribute an unrelated earlier error (e.g. a Firestore 403) to Gemini.
exc = exc.__cause__
return False
+124
View File
@@ -0,0 +1,124 @@
"""Unit tests for scripts/gemini_client.py (run: python -m pytest scripts/test_gemini_client.py)."""
import subprocess
import sys
from pathlib import Path
from unittest import mock
import pytest
sys.path.insert(0, str(Path(__file__).parent))
import gemini_client # noqa: E402
from gemini_client import ( # noqa: E402
GeminiKeyUnavailableError,
get_api_key,
is_quota_or_billing_error,
)
class FakeAPIError(Exception):
"""Mimics google.genai.errors.APIError (numeric ``code`` + ``status``)."""
def __init__(self, code, status, message):
super().__init__(f"{code} {status}. {message}")
self.code = code
self.status = status
@pytest.fixture(autouse=True)
def _reset_cache(monkeypatch):
monkeypatch.setattr(gemini_client, "_cached_key", None)
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
class TestIsQuotaOrBillingError:
@pytest.mark.parametrize(
"exc",
[
FakeAPIError(429, "RESOURCE_EXHAUSTED", "You exceeded your current quota"),
FakeAPIError(403, "PERMISSION_DENIED", "Your prepayment credits are depleted"),
FakeAPIError(400, "INVALID_ARGUMENT", "API key not valid. Please pass a valid API key."),
FakeAPIError(400, "FAILED_PRECONDITION", "Please check your plan and billing details"),
RuntimeError("429 RESOURCE_EXHAUSTED quota"),
RuntimeError("Your credits have been exhausted"),
GeminiKeyUnavailableError("no key"),
RuntimeError("429 RESOURCE_EXHAUSTED quotaId GenerateRequestsPerDayPerProjectPerModel, retryDelay 3600s"),
],
)
def test_positive(self, exc):
assert is_quota_or_billing_error(exc)
@pytest.mark.parametrize(
"exc",
[
FakeAPIError(500, "INTERNAL", "Internal error"),
FakeAPIError(503, "UNAVAILABLE", "The model is overloaded"),
FakeAPIError(400, "INVALID_ARGUMENT", "Request contains an invalid argument."),
TimeoutError("timed out"),
ValueError("bad JSON from model, processed 429 tokens"),
FakeAPIError(
429,
"RESOURCE_EXHAUSTED",
"You exceeded your current quota. quotaId: "
"GenerateRequestsPerMinutePerProjectPerModel-PaidTier, retryDelay: 7s",
),
RuntimeError("429 RESOURCE_EXHAUSTED ... Please retry in 7.2s"),
None,
],
)
def test_negative(self, exc):
assert not is_quota_or_billing_error(exc)
def test_walks_cause_chain(self):
try:
try:
raise FakeAPIError(429, "RESOURCE_EXHAUSTED", "quota")
except FakeAPIError as inner:
raise RuntimeError("translation failed") from inner
except RuntimeError as outer:
assert is_quota_or_billing_error(outer)
class TestGetApiKey:
def test_env_var_wins(self, monkeypatch):
monkeypatch.setenv("GEMINI_API_KEY", " from-env ")
with mock.patch.object(subprocess, "run") as run:
assert get_api_key() == "from-env"
run.assert_not_called()
def test_falls_back_to_gcloud_and_caches(self):
done = subprocess.CompletedProcess([], 0, stdout="from-secret\n", stderr="")
with mock.patch.object(subprocess, "run", return_value=done) as run:
assert get_api_key() == "from-secret"
assert get_api_key() == "from-secret"
assert run.call_count == 1
cmd = run.call_args.args[0]
assert "--secret=gemini-api-key" in cmd and "--project=nomadkaraoke" in cmd
def test_gcloud_failure_raises(self):
failed = subprocess.CompletedProcess([], 1, stdout="", stderr="PERMISSION_DENIED")
with mock.patch.object(subprocess, "run", return_value=failed):
with pytest.raises(GeminiKeyUnavailableError):
get_api_key()
def test_gcloud_missing_raises(self):
with mock.patch.object(subprocess, "run", side_effect=FileNotFoundError("gcloud")):
with pytest.raises(GeminiKeyUnavailableError):
get_api_key()
def test_quota_message_hints_env_key(monkeypatch):
assert "environment" not in gemini_client.quota_exhausted_message()
monkeypatch.setenv("GEMINI_API_KEY", "x")
assert "GEMINI_API_KEY environment variable" in gemini_client.quota_exhausted_message()
def test_client_pins_developer_api(monkeypatch):
"""vertexai=False even if GOOGLE_GENAI_USE_VERTEXAI is set in the environment."""
monkeypatch.setenv("GEMINI_API_KEY", "k")
monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "true")
with mock.patch("google.genai.Client") as client_cls:
gemini_client.get_genai_client()
kwargs = client_cls.call_args.kwargs
assert kwargs["vertexai"] is False and kwargs["api_key"] == "k"
+26 -13
View File
@@ -2,7 +2,7 @@
"""
LLM Translation Pipeline for Nomad Karaoke i18n.
Two-pass translation using Gemini via Vertex AI:
Two-pass translation using Gemini (Developer API):
1. Translate English JSON to target language
2. Review and polish translations for fluency
@@ -24,7 +24,9 @@ Usage:
Requires:
- google-genai SDK (pip install google-genai)
- GCP Application Default Credentials (gcloud auth application-default login)
- Gemini Developer API key: GEMINI_API_KEY env var, else read from Secret
Manager (gemini-api-key) via your gcloud login — see gemini_client.py
- GCP Application Default Credentials for the GCS translation cache
"""
import argparse
@@ -34,14 +36,18 @@ import sys
import time
from pathlib import Path
from gemini_client import (
quota_exhausted_message,
GeminiKeyUnavailableError,
get_genai_client,
is_quota_or_billing_error,
)
from translation_cache import TranslationCache
from google import genai
from google.genai import types
MODEL = "gemini-3.8-flash"
PROJECT = "nomadkaraoke"
LOCATION = "global"
MAX_CONCURRENT = 5
MAX_RETRIES = 3
@@ -325,7 +331,8 @@ async def _call_with_retry(client: genai.Client, prompt: str) -> str:
)
return response.text
except Exception as e:
if attempt == MAX_RETRIES - 1:
# Quota/credit/key errors won't fix themselves — fail fast.
if attempt == MAX_RETRIES - 1 or is_quota_or_billing_error(e):
raise
wait = 2 ** (attempt + 1)
print(f" Retry {attempt + 1}/{MAX_RETRIES} after error: {e}")
@@ -576,12 +583,14 @@ async def async_main(args):
elif not args.full:
print("No snapshot found — will use missing-keys mode (use --full to force full retranslation)")
# Initialize Vertex AI client
client = genai.Client(
vertexai=True,
project=PROJECT,
location=LOCATION,
)
# Gemini Developer API client (API key; not needed for --dry-run)
client = None
if not args.dry_run:
try:
client = get_genai_client()
except GeminiKeyUnavailableError as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(2)
cache = TranslationCache(
bucket_name=args.cache_bucket,
@@ -589,7 +598,7 @@ async def async_main(args):
)
print(f"Using model: {MODEL}")
print(f"Project: {PROJECT}, Location: {LOCATION}")
print("Backend: Gemini Developer API (gemini-api-key)")
print(f"Locales: {', '.join(locales)} ({len(locales)} total)")
if args.full:
print("Mode: full (forced)")
@@ -655,10 +664,14 @@ async def async_main(args):
else:
print()
if any(isinstance(r, BaseException) and is_quota_or_billing_error(r) for r in results):
print(f"\nError: {quota_exhausted_message()}", file=sys.stderr)
sys.exit(2)
def main():
parser = argparse.ArgumentParser(
description="Translate i18n message files using Gemini via Vertex AI"
description="Translate i18n message files using the Gemini Developer API"
)
parser.add_argument(
"--messages-dir",
+22
View File
@@ -153,6 +153,17 @@ def create_lyrics_transcription_job(
name="GCS_BUCKET_NAME",
value=bucket.name,
),
# Gemini Developer API key (AI Studio; Vertex AI is
# disabled in this project). Rotate = new secret version.
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
name="GEMINI_API_KEY",
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
secret=f"projects/{PROJECT_ID}/secrets/gemini-api-key",
version="latest",
),
),
),
# Genius API token (for lyrics fetching)
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
name="GENIUS_API_TOKEN",
@@ -588,6 +599,17 @@ def create_video_encoding_job(
name="DEFAULT_BRAND_PREFIX",
value="NOMAD",
),
# Gemini Developer API key (AI Studio; Vertex AI is
# disabled in this project). Rotate = new secret version.
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
name="GEMINI_API_KEY",
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
secret=f"projects/{PROJECT_ID}/secrets/gemini-api-key",
version="latest",
),
),
),
# Discord webhook for release notifications
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
name="DEFAULT_DISCORD_WEBHOOK_URL",
+11
View File
@@ -91,6 +91,17 @@ def create_error_monitor(
name="ENVIRONMENT",
value="production",
),
# Gemini Developer API key (AI Studio; Vertex AI is
# disabled in this project). Rotate = new secret version.
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
name="GEMINI_API_KEY",
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
secret=f"projects/{PROJECT_ID}/secrets/gemini-api-key",
version="latest",
),
),
),
# Discord webhook for alert notifications
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
name="DISCORD_WEBHOOK_URL",
+3 -7
View File
@@ -80,13 +80,9 @@ def grant_backend_permissions(
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant Vertex AI User permissions (for Gemini agentic AI correction)
bindings["vertex_ai_iam"] = gcp.projects.IAMMember(
"karaoke-backend-vertexai-user",
project=PROJECT_ID,
role="roles/aiplatform.user",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# No Vertex AI role: Gemini is called via the Gemini Developer API with the
# `gemini-api-key` secret (read via the project-level secretAccessor role
# above), and aiplatform.googleapis.com is disabled in this project.
# Grant backend service account permission to enqueue Cloud Tasks
bindings["cloud_tasks_enqueuer"] = gcp.projects.IAMMember(
@@ -61,7 +61,7 @@ class ProviderConfig:
return ProviderConfig(
openai_api_key=os.getenv("OPENAI_API_KEY"),
anthropic_api_key=os.getenv("ANTHROPIC_API_KEY"),
google_api_key=os.getenv("GOOGLE_API_KEY"),
google_api_key=os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY"),
openrouter_api_key=os.getenv("OPENROUTER_API_KEY"),
privacy_mode=os.getenv("PRIVACY_MODE", "false").lower() in {"1", "true", "yes"},
cache_dir=cache_dir,
@@ -20,8 +20,9 @@ except ImportError:
# Error message constant for TRY003 compliance
GOOGLE_API_KEY_MISSING_ERROR = (
"GOOGLE_API_KEY environment variable is required for Google/Gemini models. "
"Get an API key from https://aistudio.google.com/app/apikey"
"GEMINI_API_KEY (or GOOGLE_API_KEY) environment variable is required for Gemini "
"models (Gemini Developer API; on Cloud Run it is injected from Secret Manager "
"`gemini-api-key`). Vertex AI is not used."
)
@@ -191,8 +192,10 @@ class ModelFactory:
return self._create_openai_model(model_name, callbacks, config)
elif provider == "anthropic":
return self._create_anthropic_model(model_name, callbacks, config)
elif provider in ("vertexai", "google"):
return self._create_vertexai_model(model_name, callbacks, config)
elif provider in ("gemini", "google", "vertexai"):
# "vertexai/..." is accepted for backward compatibility but is
# served by the Gemini Developer API (Vertex is disabled).
return self._create_gemini_model(model_name, callbacks, config)
else:
raise ValueError(f"Unsupported provider: {provider}")
except ImportError as e:
@@ -245,57 +248,39 @@ class ModelFactory:
logger.debug(f"🤖 Created Anthropic model: {model_name}")
return model
def _create_vertexai_model(
def _create_gemini_model(
self, model_name: str, callbacks: List[Any], config: ProviderConfig
) -> Any:
"""Create ChatGoogleGenerativeAI model for Google Gemini.
Uses the unified langchain-google-genai package which supports both:
- Vertex AI backend (service account / ADC auth) - when project is set
- Google AI Studio backend (API key auth) - when only api_key is set
Always uses the Gemini Developer API (API key auth) — never Vertex AI, so
no project/location is passed. The key comes from GEMINI_API_KEY (or
GOOGLE_API_KEY); on Cloud Run it's injected from Secret Manager
``gemini-api-key``.
On Cloud Run, ADC (Application Default Credentials) are used automatically
when the project parameter is provided, using the service account attached
to the Cloud Run service.
This is a REST-based API that avoids the gRPC connection issues
langchain-google-genai is REST-based, avoiding the gRPC connection issues
seen with the deprecated langchain-google-vertexai package.
"""
from langchain_google_genai import ChatGoogleGenerativeAI
start_time = time.time()
# Determine authentication method
api_key = config.google_api_key
project = config.gcp_project_id
# Prefer Vertex AI (service account) if project is set, otherwise require API key
if not project and not api_key:
if not api_key:
raise ValueError(GOOGLE_API_KEY_MISSING_ERROR)
if project:
logger.info(f"🤖 Creating Google Gemini model via Vertex AI (project={project}): {model_name}")
else:
logger.info(f"🤖 Creating Google Gemini model via AI Studio API: {model_name}")
logger.info(f"🤖 Creating Google Gemini model via Gemini Developer API: {model_name}")
# Build kwargs - only include api_key if set (otherwise ADC is used)
model_kwargs = {
"model": model_name,
"convert_system_message_to_human": True, # Gemini doesn't support system messages
"max_retries": config.max_retries,
"timeout": config.request_timeout_seconds,
"callbacks": callbacks,
}
# Add project to trigger Vertex AI backend with ADC
if project:
model_kwargs["project"] = project
# Add API key if available (can be used with or without project)
if api_key:
model_kwargs["google_api_key"] = api_key
model = ChatGoogleGenerativeAI(**model_kwargs)
model = ChatGoogleGenerativeAI(
model=model_name,
google_api_key=api_key,
# Pin the Developer API even if GOOGLE_GENAI_USE_VERTEXAI is set.
vertexai=False,
convert_system_message_to_human=True, # Gemini doesn't support system messages
max_retries=config.max_retries,
timeout=config.request_timeout_seconds,
callbacks=callbacks,
)
elapsed = time.time() - start_time
logger.info(f"🤖 Google Gemini model created in {elapsed:.2f}s: {model_name}")
@@ -5,9 +5,9 @@ from typing import Dict, Any
from .providers.config import ProviderConfig
# Default model for cloud deployments - Gemini 3 Flash via Vertex AI
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
DEFAULT_CLOUD_MODEL = "vertexai/gemini-3.8-flash"
# Default model for cloud deployments - Gemini 3.8 Flash via the Gemini
# Developer API (API key from GEMINI_API_KEY; Vertex AI is not used).
DEFAULT_CLOUD_MODEL = "gemini/gemini-3.8-flash"
class ModelRouter:
@@ -20,7 +20,7 @@ class ModelRouter:
"""Choose appropriate model based on gap characteristics.
Returns model identifier in format "provider/model" for LangChain:
- "vertexai/gemini-3.8-flash" for Gemini via Vertex AI (default)
- "gemini/gemini-3.8-flash" for Gemini via the Developer API (default)
- "ollama/llama3.2:latest" for local Ollama models
- "openai/gpt-4" for OpenAI models
- "anthropic/claude-3-sonnet-20240229" for Anthropic models
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.261.4"
version = "0.262.0"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+16 -1
View File
@@ -23,7 +23,9 @@ Usage:
Requires:
- ANTHROPIC_API_KEY in the environment (for claude-* models)
- GCP creds for Vertex/Firestore/GCS:
- Gemini Developer API key for gemini-* models: GEMINI_API_KEY, else read
from Secret Manager `gemini-api-key` via your gcloud login (in-process only)
- GCP creds for Firestore/GCS:
gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS)
- GOOGLE_CLOUD_PROJECT=nomadkaraoke (set automatically below if unset)
@@ -58,6 +60,17 @@ GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
DEFAULT_MODELS = ["claude-opus-5-5", "gemini-3.8-flash"]
def _ensure_gemini_key() -> None:
"""Populate GEMINI_API_KEY (this process only) from Secret Manager via gcloud."""
if os.environ.get("GEMINI_API_KEY"):
return
repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(repo_root, "frontend", "scripts"))
from gemini_client import get_api_key # noqa: E402 - scripts helper (gcloud fallback)
os.environ["GEMINI_API_KEY"] = get_api_key()
# ---------------------------------------------------------------------------
# Job + corrections loading
# ---------------------------------------------------------------------------
@@ -284,6 +297,8 @@ def main() -> int:
parser.add_argument("--models", nargs="+", default=DEFAULT_MODELS,
help=f"Models to measure (default: {' '.join(DEFAULT_MODELS)})")
args = parser.parse_args()
if any(m.startswith("gemini") for m in args.models):
_ensure_gemini_key()
if any(m.startswith("claude") for m in args.models) and not os.environ.get("ANTHROPIC_API_KEY"):
logger.error("ANTHROPIC_API_KEY is not set — required for claude-* models")
+12 -9
View File
@@ -6,7 +6,8 @@ This script allows testing the agentic correction locally without deploying
to Cloud Run or running a full karaoke job.
Usage:
# Test with default Vertex AI (requires gcloud auth):
# Test with the default Gemini model (Developer API; GEMINI_API_KEY, else
# read from Secret Manager `gemini-api-key` via your gcloud login):
python scripts/test_agentic_correction.py
# Test with a specific model:
@@ -17,8 +18,7 @@ Usage:
PRIVACY_MODE=1 python scripts/test_agentic_correction.py
Environment variables:
GOOGLE_CLOUD_PROJECT: GCP project ID (required for Vertex AI)
GCP_LOCATION: Vertex AI location (default: global)
GEMINI_API_KEY: Gemini Developer API key (else fetched via gcloud)
AGENTIC_AI_MODEL: Override the model (e.g., openai/gpt-4o)
PRIVACY_MODE: Set to 1 to use local Ollama
LANGFUSE_PUBLIC_KEY: Langfuse public key for tracing
@@ -209,12 +209,15 @@ def main():
model = router.choose_model("unknown", 0.5)
logger.info(f"Selected model: {model}")
# Validate environment for Vertex AI
if model.startswith("vertexai/") and not project_id:
logger.error("GOOGLE_CLOUD_PROJECT environment variable is required for Vertex AI")
logger.error("Set it with: export GOOGLE_CLOUD_PROJECT=your-project-id")
logger.error("Or use a different model: AGENTIC_AI_MODEL=openai/gpt-4o")
sys.exit(1)
# Gemini Developer API key (this process only; never written to disk)
if model.split("/", 1)[0] in ("gemini", "google", "vertexai") and not (
os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
):
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend", "scripts"))
from gemini_client import get_api_key
os.environ["GEMINI_API_KEY"] = get_api_key()
config = ProviderConfig.from_env()
# Create the agent
logger.info(f"\nCreating AgenticCorrector with model: {model}")
+3
View File
@@ -1,5 +1,8 @@
import os
import pytest
# Gemini Developer API key: never let a test fall through to Secret Manager.
os.environ.setdefault("GEMINI_API_KEY", "test-gemini-key")
import tempfile
import logging
from unittest.mock import MagicMock
@@ -0,0 +1,46 @@
"""Agentic correction's Gemini provider uses the Gemini Developer API key, never Vertex."""
from unittest.mock import patch
import pytest
from karaoke_gen.lyrics_transcriber.correction.agentic.providers.config import ProviderConfig
from karaoke_gen.lyrics_transcriber.correction.agentic.providers.model_factory import ModelFactory
from karaoke_gen.lyrics_transcriber.correction.agentic.router import DEFAULT_CLOUD_MODEL
def _config(monkeypatch, **env):
for k in ("GEMINI_API_KEY", "GOOGLE_API_KEY"):
monkeypatch.delenv(k, raising=False)
for k, v in env.items():
monkeypatch.setenv(k, v)
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "nomadkaraoke") # must NOT trigger Vertex
return ProviderConfig.from_env()
def test_default_model_is_developer_api():
assert DEFAULT_CLOUD_MODEL == "gemini/gemini-3.8-flash"
@pytest.mark.parametrize("spec", ["gemini/gemini-3.8-flash", "vertexai/gemini-3.8-flash"])
def test_gemini_model_uses_api_key_not_project(monkeypatch, spec):
config = _config(monkeypatch, GEMINI_API_KEY="dev-key")
with patch("langchain_google_genai.ChatGoogleGenerativeAI") as chat:
factory = ModelFactory()
provider, model_name = factory._parse_model_spec(spec)
factory._instantiate_model(provider, model_name, [], config)
kwargs = chat.call_args.kwargs
assert kwargs["google_api_key"] == "dev-key"
assert kwargs["model"] == "gemini-3.8-flash"
assert kwargs["vertexai"] is False
assert "project" not in kwargs and "location" not in kwargs
def test_google_api_key_still_accepted(monkeypatch):
config = _config(monkeypatch, GOOGLE_API_KEY="legacy-key")
assert config.google_api_key == "legacy-key"
def test_missing_key_raises_clear_error(monkeypatch):
config = _config(monkeypatch)
with pytest.raises(ValueError, match="GEMINI_API_KEY"):
ModelFactory()._create_gemini_model("gemini-3.8-flash", [], config)
@@ -181,11 +181,6 @@ class TestDefaultValues:
assert isinstance(config.LLM_ANALYSIS_MODEL, str)
assert len(config.LLM_ANALYSIS_MODEL) > 0
def test_llm_vertex_location_is_string(self):
from backend.services.error_monitor import config
assert isinstance(config.LLM_VERTEX_LOCATION, str)
assert len(config.LLM_VERTEX_LOCATION) > 0
class TestEnvVarOverrides: