mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bdca8f5542 | ||
|
|
b9f9c90c6d | ||
|
|
b8c6c6d796 | ||
|
|
50b3b7db8e |
No files matched your search
@@ -2022,7 +2022,7 @@ jobs:
|
||||
--min-instances 0 \
|
||||
--clear-vpc-connector \
|
||||
--set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,USE_CLOUD_RUN_JOBS_FOR_RENDER=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-5-5,AUTO_CORRECT_COMPARE_MODELS=claude-opus-5-5;gemini-3.8-flash,AUTO_CORRECT_PROACTIVE_ENABLED=true,AUTO_APPROVAL_BACKING_KEEP_ENABLED=true,COMMUNITY_DAILY_PICK_ENABLED=true,EDGE_AUTH_MODE=enforce,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \
|
||||
--set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest"
|
||||
--set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest"
|
||||
|
||||
# R3/R4 — smoke the tagged candidate revision BEFORE it serves any traffic.
|
||||
- name: Smoke candidate revision
|
||||
|
||||
@@ -112,7 +112,7 @@ See `docs/TESTING.md` § "Ad-Hoc Production Debugging" for full details.
|
||||
- Only `/admin/*` is English-only — it has no `[locale]/admin` counterpart and inherits `DefaultIntlProvider` from the root layout
|
||||
- Internal links use `Link` from `@/i18n/routing` (locale-aware)
|
||||
- After adding/changing English strings: `python frontend/scripts/translate.py --messages-dir frontend/messages --target all`
|
||||
- Translation uses Gemini 3.8 Flash via Vertex AI with GCS cache for efficiency
|
||||
- Translation uses Gemini 3.8 Flash via the **Gemini Developer API** (key: `GEMINI_API_KEY`, else Secret Manager `gemini-api-key` via your gcloud login — `frontend/scripts/gemini_client.py`) with GCS cache for efficiency. **Vertex AI is disabled in the GCP project — never use `vertexai=True`; backend code uses `backend/services/gemini_client.get_genai_client()`.**
|
||||
- CI validates all locale files have matching keys — **PR will fail if translations are missing**
|
||||
- Pre-commit hook auto-translates when en.json changes (enable: `git config core.hooksPath .githooks`)
|
||||
- Don't hardcode user-facing strings — add to `messages/en.json` and use `t('key')`
|
||||
|
||||
+2
-2
@@ -82,9 +82,9 @@ class Settings(BaseSettings):
|
||||
|
||||
# Agentic AI Correction (for lyrics correction via LLM)
|
||||
# Only used when skip_correction=false
|
||||
# When enabled, uses Gemini via Vertex AI for intelligent lyrics correction
|
||||
# When enabled, uses Gemini (Developer API, key GEMINI_API_KEY) for intelligent lyrics correction
|
||||
use_agentic_ai: bool = os.getenv("USE_AGENTIC_AI", "true").lower() in ("true", "1", "yes")
|
||||
agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "vertexai/gemini-3.8-flash")
|
||||
agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "gemini/gemini-3.8-flash")
|
||||
# Timeout for agentic correction in seconds. If correction takes longer, abort and
|
||||
# use uncorrected transcription - human review will fix any issues.
|
||||
agentic_correction_timeout_seconds: int = int(os.getenv("AGENTIC_CORRECTION_TIMEOUT_SECONDS", "180"))
|
||||
|
||||
@@ -54,8 +54,10 @@ MODEL_PRICING: dict[str, _Rate] = {
|
||||
"claude-opus-4-8": {"input": 5.0, "output": 25.0, "cache_read": 0.5},
|
||||
"claude-sonnet-4-6": {"input": 3.0, "output": 15.0, "cache_read": 0.3},
|
||||
"claude-haiku-4-5": {"input": 1.0, "output": 5.0, "cache_read": 0.1},
|
||||
# --- Gemini (ESTIMATE — Vertex 3.x Pro preview, <=200K-token context
|
||||
# tier; confirm against current Google pricing before trusting USD) ---
|
||||
# --- Gemini (ESTIMATE — 3.x Pro preview, <=200K-token context tier;
|
||||
# confirm against current Google pricing before trusting USD). Calls go
|
||||
# via the Gemini Developer API (paid tier), assumed at Vertex list-price
|
||||
# parity. ---
|
||||
"gemini-3.1-pro-preview": {"input": 2.0, "output": 12.0, "cache_read": 0.2},
|
||||
# Promo rate through 2026-12-31; see _RATE_CHANGES for the standard rate.
|
||||
"gemini-3.8-flash": {"input": 0.75, "output": 3.75, "cache_read": 0.075},
|
||||
|
||||
@@ -710,22 +710,19 @@ class AutoCorrectService:
|
||||
def _call_gemini(
|
||||
self, model: str, system_prompt: str, user_prompt: str, *, job_id: str
|
||||
) -> tuple[Any, Optional[TokenUsage]]:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=self.settings.google_cloud_project,
|
||||
location="global",
|
||||
# Milliseconds; bounds the call so a hung model never strands the UI.
|
||||
http_options=types.HttpOptions(timeout=120_000),
|
||||
)
|
||||
from backend.services.gemini_client import get_genai_client, note_gemini_failure
|
||||
|
||||
# Deep-copy: the google-genai SDK mutates the schema dict in place,
|
||||
# which would poison the shared module-level constant for other
|
||||
# providers running in the same process.
|
||||
import copy
|
||||
|
||||
try:
|
||||
# Gemini Developer API (API key). Timeout in milliseconds bounds the
|
||||
# call so a hung model never strands the UI.
|
||||
client = get_genai_client(timeout_ms=120_000)
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=[user_prompt],
|
||||
@@ -736,6 +733,15 @@ class AutoCorrectService:
|
||||
),
|
||||
)
|
||||
except Exception as exc: # surface as 502, never a stuck job
|
||||
if note_gemini_failure("auto_correct", exc):
|
||||
# Quota / prepaid credit / key problem: retrying won't help.
|
||||
# Non-retryable so compare mode drops the Gemini leg at once
|
||||
# and keeps the other model's (Opus) suggestions.
|
||||
raise AutoCorrectServiceError(
|
||||
"Gemini is unavailable (quota/billing/key) — skipped",
|
||||
status_code=503,
|
||||
retryable=False,
|
||||
) from exc
|
||||
if _is_transient_model_error(exc):
|
||||
logger.warning(
|
||||
"auto-correct model rate-limited/overloaded job=%s model=%s (%s)",
|
||||
|
||||
@@ -295,11 +295,9 @@ class CreditEvaluationService:
|
||||
"""Call Gemini and return the response text."""
|
||||
from google import genai
|
||||
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=self.settings.google_cloud_project,
|
||||
location="global",
|
||||
)
|
||||
from backend.services.gemini_client import get_genai_client
|
||||
|
||||
client = get_genai_client(timeout_ms=60_000)
|
||||
response = client.models.generate_content(
|
||||
model=self.settings.credit_eval_model,
|
||||
contents=prompt,
|
||||
@@ -405,7 +403,13 @@ class CreditEvaluationService:
|
||||
feedback_content=feedback_content,
|
||||
)
|
||||
|
||||
response_text = self._call_gemini(prompt)
|
||||
try:
|
||||
response_text = self._call_gemini(prompt)
|
||||
except Exception as gemini_exc:
|
||||
from backend.services.gemini_client import note_gemini_failure
|
||||
|
||||
note_gemini_failure("credit_evaluation", gemini_exc)
|
||||
raise
|
||||
evaluation = _parse_gemini_response(response_text)
|
||||
evaluation.signals = signals
|
||||
|
||||
|
||||
@@ -562,36 +562,43 @@ class CustomLyricsService:
|
||||
pdf_bytes: Optional[bytes],
|
||||
settings: GenerationSettings,
|
||||
) -> list[str]:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=self.settings.google_cloud_project,
|
||||
location="global",
|
||||
)
|
||||
from backend.services.gemini_client import get_genai_client, note_gemini_failure
|
||||
|
||||
contents: list = [user_prompt]
|
||||
if pdf_bytes is not None:
|
||||
contents.append(
|
||||
types.Part.from_bytes(mime_type="application/pdf", data=pdf_bytes)
|
||||
)
|
||||
|
||||
response = client.models.generate_content(
|
||||
model=self.settings.custom_lyrics_model,
|
||||
contents=contents,
|
||||
config=types.GenerateContentConfig(
|
||||
system_instruction=system_prompt,
|
||||
temperature=0.4,
|
||||
response_mime_type="application/json",
|
||||
response_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"lines": {"type": "array", "items": {"type": "string"}}
|
||||
try:
|
||||
# Bounded so a stalled call can't hang the request (Gemini Developer API).
|
||||
client = get_genai_client(timeout_ms=300_000)
|
||||
response = client.models.generate_content(
|
||||
model=self.settings.custom_lyrics_model,
|
||||
contents=contents,
|
||||
config=types.GenerateContentConfig(
|
||||
system_instruction=system_prompt,
|
||||
temperature=0.4,
|
||||
response_mime_type="application/json",
|
||||
response_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"lines": {"type": "array", "items": {"type": "string"}}
|
||||
},
|
||||
"required": ["lines"],
|
||||
},
|
||||
"required": ["lines"],
|
||||
},
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
if note_gemini_failure("custom_lyrics", exc):
|
||||
raise CustomLyricsServiceError(
|
||||
"The AI lyrics service is temporarily unavailable. Please try "
|
||||
"again later, or edit the lyrics manually.",
|
||||
status_code=503,
|
||||
) from exc
|
||||
raise
|
||||
# The model can return no text (safety block, MAX_TOKENS with no content,
|
||||
# empty candidate). `response.text` is then None and json.loads(None) would
|
||||
# raise a raw TypeError -> uncaught 500 without CORS headers -> the browser
|
||||
|
||||
@@ -110,7 +110,6 @@ Shared infrastructure:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LLM_ANALYSIS_MODEL: str = os.environ.get("LLM_ANALYSIS_MODEL", "gemini-3.8-flash")
|
||||
LLM_VERTEX_LOCATION: str = os.environ.get("LLM_VERTEX_LOCATION", "global")
|
||||
|
||||
#: Minimum number of active error patterns required before triggering LLM analysis.
|
||||
MIN_PATTERNS_FOR_ANALYSIS: int = 3
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""LLM-powered incident analysis and duplicate detection for the error monitor.
|
||||
|
||||
Uses Gemini Flash via the google-generativeai library to:
|
||||
Uses Gemini Flash via the Gemini Developer API (backend.services.gemini_client) to:
|
||||
1. Group related error patterns into incidents with root-cause analysis.
|
||||
2. Identify near-duplicate patterns that the regex normalizer missed.
|
||||
|
||||
@@ -17,13 +17,11 @@ import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from google import genai
|
||||
from google.genai import types as genai_types
|
||||
|
||||
from backend.services.gemini_client import get_genai_client, note_gemini_failure
|
||||
from backend.services.error_monitor.config import (
|
||||
GCP_PROJECT,
|
||||
LLM_ANALYSIS_MODEL,
|
||||
LLM_VERTEX_LOCATION,
|
||||
MIN_PATTERNS_FOR_ANALYSIS,
|
||||
SERVICE_DEPENDENCY_MAP,
|
||||
)
|
||||
@@ -96,11 +94,7 @@ def _call_llm(
|
||||
|
||||
for attempt in range(_RETRY_ATTEMPTS):
|
||||
try:
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=GCP_PROJECT,
|
||||
location=LLM_VERTEX_LOCATION,
|
||||
)
|
||||
client = get_genai_client(timeout_ms=120_000)
|
||||
response = client.models.generate_content(
|
||||
model=LLM_ANALYSIS_MODEL,
|
||||
contents=user_prompt,
|
||||
@@ -111,6 +105,10 @@ def _call_llm(
|
||||
)
|
||||
return response.text
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if note_gemini_failure("error_monitor", exc):
|
||||
# Quota/credit/key problem: don't retry — callers fall back to
|
||||
# alerts without the AI summary.
|
||||
raise
|
||||
last_exc = exc
|
||||
delay = _RETRY_BASE_DELAY * (2**attempt)
|
||||
logger.warning(
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Shared Gemini client for the backend: Gemini Developer API (API key), not Vertex.
|
||||
|
||||
Every backend Gemini call goes through :func:`get_genai_client`. Vertex AI is
|
||||
disabled in the nomadkaraoke GCP project so AI spend never lands on it; calls are
|
||||
billed to whichever AI Studio key is stored in Secret Manager ``gemini-api-key``.
|
||||
|
||||
Key resolution (via :meth:`backend.config.Settings.get_secret`):
|
||||
1. ``GEMINI_API_KEY`` env var — injected on Cloud Run from
|
||||
``gemini-api-key:latest`` (CI ``--set-secrets`` for the service, Pulumi
|
||||
for the jobs).
|
||||
2. Secret Manager ``gemini-api-key`` (latest) — read directly with the
|
||||
runtime service account / your ADC when the env var isn't set.
|
||||
|
||||
Rotating the key: add a new version of ``gemini-api-key``. New instances / job
|
||||
executions pick it up; no code change.
|
||||
|
||||
Degradation: when the key's quota or prepaid credit is exhausted (or the key is
|
||||
rejected), callers must degrade rather than fail jobs. Use
|
||||
:func:`is_quota_or_billing_error` to classify and :func:`note_gemini_failure`
|
||||
from the caller's ``except`` block — it also sends a Discord ops alert at most
|
||||
once per ``GEMINI_QUOTA_ALERT_HOURS`` (default 6h) across all instances.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SECRET_NAME = "gemini-api-key"
|
||||
|
||||
QUOTA_EXHAUSTED_MESSAGE = (
|
||||
"Gemini quota/credit exhausted — top up AI Studio or rotate gemini-api-key "
|
||||
"(add a new version of Secret Manager secret 'gemini-api-key' in project nomadkaraoke)."
|
||||
)
|
||||
|
||||
# Firestore dedup doc id (in ops_alerts' collection) shared by every instance/job.
|
||||
_ALERT_KEY = "gemini-quota-exhausted"
|
||||
_DEFAULT_ALERT_HOURS = 6.0
|
||||
|
||||
|
||||
class GeminiKeyUnavailableError(RuntimeError):
|
||||
"""No Gemini API key could be resolved (env var unset and Secret Manager failed)."""
|
||||
|
||||
|
||||
def get_api_key() -> str:
|
||||
"""Return the Gemini Developer API key or raise GeminiKeyUnavailableError."""
|
||||
env_key = os.environ.get("GEMINI_API_KEY", "").strip()
|
||||
if env_key:
|
||||
return env_key
|
||||
from backend.config import get_settings
|
||||
|
||||
key = (get_settings().get_secret(SECRET_NAME) or "").strip()
|
||||
if not key:
|
||||
raise GeminiKeyUnavailableError(
|
||||
"GEMINI_API_KEY is not set and Secret Manager secret 'gemini-api-key' "
|
||||
"could not be read"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def get_genai_client(timeout_ms: Optional[int] = None) -> Any:
|
||||
"""Build a google-genai client for the Gemini Developer API.
|
||||
|
||||
``timeout_ms`` bounds each HTTP request (milliseconds). No project/location:
|
||||
those only apply to Vertex AI.
|
||||
"""
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
http_options = types.HttpOptions(timeout=int(timeout_ms)) if timeout_ms else None
|
||||
# vertexai=False pins the Developer API even if GOOGLE_GENAI_USE_VERTEXAI is set.
|
||||
return genai.Client(vertexai=False, api_key=get_api_key(), http_options=http_options)
|
||||
|
||||
|
||||
# Message fragments that mean "the key/account can't pay or isn't valid" — a
|
||||
# retry won't help; a human must top up credit or rotate the key.
|
||||
_QUOTA_PATTERNS = re.compile(
|
||||
r"RESOURCE_EXHAUSTED|exceeded your current quota|quota exceeded|"
|
||||
r"billing (details|account)|prepa(id|yment)|credits? (are |is |have been )?"
|
||||
r"(exhausted|depleted|used up)|insufficient (credit|funds|balance)|"
|
||||
r"API_KEY_INVALID|API key not valid|API key expired|API_KEY_SERVICE_BLOCKED|"
|
||||
r"\b429 Too Many Requests",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# A 429 that names a per-minute quota / carries a retry delay is an ordinary
|
||||
# rate limit (transient — retry with backoff), not exhausted credit.
|
||||
_RATE_LIMIT_PATTERNS = re.compile(r"PerMinute|retryDelay|retry in \d", re.IGNORECASE)
|
||||
# ...unless it names a per-day quota: that won't clear for hours — treat as exhausted.
|
||||
_DAILY_QUOTA_PATTERN = re.compile(r"PerDay", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_rate_limit(text: str) -> bool:
|
||||
return bool(_RATE_LIMIT_PATTERNS.search(text)) and not _DAILY_QUOTA_PATTERN.search(text)
|
||||
|
||||
|
||||
def is_quota_or_billing_error(exc: Optional[BaseException]) -> bool:
|
||||
"""True if ``exc`` (or anything in its ``__cause__`` chain) is a Gemini quota,
|
||||
prepaid-credit, billing or API-key error — i.e. needs a top-up/key rotation,
|
||||
not a retry.
|
||||
|
||||
Matches google-genai ``APIError`` (``.code`` 429 / 403 or status
|
||||
RESOURCE_EXHAUSTED / PERMISSION_DENIED, or a billing/key message on a 400),
|
||||
a missing key, and wrapped/stringified errors from other layers (LangChain).
|
||||
Per-minute rate limits (429 naming a ``PerMinute`` quota or a retry delay)
|
||||
are NOT matched — those are transient; retry them.
|
||||
"""
|
||||
seen: set[int] = set()
|
||||
while exc is not None and id(exc) not in seen:
|
||||
seen.add(id(exc))
|
||||
if isinstance(exc, GeminiKeyUnavailableError):
|
||||
return True
|
||||
code = getattr(exc, "code", None)
|
||||
if code is None:
|
||||
code = getattr(exc, "status_code", None)
|
||||
status = str(getattr(exc, "status", "") or "")
|
||||
if code == 429 or status == "RESOURCE_EXHAUSTED":
|
||||
return not _is_rate_limit(str(exc))
|
||||
if code == 403 or status == "PERMISSION_DENIED":
|
||||
return True
|
||||
text = str(exc)
|
||||
if _QUOTA_PATTERNS.search(text) and not _is_rate_limit(text):
|
||||
return True
|
||||
# Only explicit chaining (`raise X from e`): implicit __context__ could
|
||||
# attribute an unrelated earlier error (e.g. a Firestore 403) to Gemini.
|
||||
exc = exc.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def _alert_hours() -> float:
|
||||
try:
|
||||
return float(os.environ.get("GEMINI_QUOTA_ALERT_HOURS", _DEFAULT_ALERT_HOURS))
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_ALERT_HOURS
|
||||
|
||||
|
||||
# In-process throttle so a burst of failures in one instance doesn't hit
|
||||
# Firestore on every call; the Firestore doc throttles across instances/jobs.
|
||||
_last_alert_attempt: float = 0.0
|
||||
_alert_lock = threading.Lock()
|
||||
# Alert I/O (Firestore + Secret Manager + Discord) runs on a daemon thread so it
|
||||
# never blocks the caller — several callers are on the asyncio event loop.
|
||||
# Tests flip this off to run it inline.
|
||||
_ALERT_IN_BACKGROUND = True
|
||||
|
||||
|
||||
def note_gemini_failure(caller: str, exc: BaseException) -> bool:
|
||||
"""Classify a Gemini failure; on quota/billing/key errors log + alert.
|
||||
|
||||
Call from a caller's ``except`` block. Returns True when the error is a
|
||||
quota/billing/key error (caller should skip retries and degrade). Never raises.
|
||||
"""
|
||||
try:
|
||||
if not is_quota_or_billing_error(exc):
|
||||
return False
|
||||
logger.error("Gemini unavailable (quota/billing/key) in %s: %s", caller, exc)
|
||||
_maybe_alert(caller, exc)
|
||||
return True
|
||||
except Exception: # noqa: BLE001 - classification/alerting must never raise
|
||||
logger.warning("note_gemini_failure failed", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def _maybe_alert(caller: str, exc: BaseException) -> None:
|
||||
global _last_alert_attempt
|
||||
window = _alert_hours() * 3600
|
||||
now = time.monotonic()
|
||||
with _alert_lock:
|
||||
if _last_alert_attempt and now - _last_alert_attempt < window:
|
||||
return
|
||||
_last_alert_attempt = now
|
||||
if _ALERT_IN_BACKGROUND:
|
||||
threading.Thread(
|
||||
target=_send_alert, args=(caller, exc, window), name="gemini-quota-alert", daemon=True
|
||||
).start()
|
||||
else:
|
||||
_send_alert(caller, exc, window)
|
||||
|
||||
|
||||
def _send_alert(caller: str, exc: BaseException, window: float) -> None:
|
||||
global _last_alert_attempt
|
||||
from backend.services import ops_alerts
|
||||
|
||||
err = str(exc).strip()
|
||||
if len(err) > 800:
|
||||
err = err[:800] + " …[truncated]"
|
||||
message = (
|
||||
"🟠 **Gemini API unavailable — quota / prepaid credit / key problem**\n"
|
||||
f"**First seen in:** `{caller}`\n"
|
||||
"Callers are degrading (auto-correct keeps Opus only, match-judge/parse-titles "
|
||||
"return no suggestion, credit evals go to manual review, translations are "
|
||||
"skipped).\n"
|
||||
f"**Fix:** {QUOTA_EXHAUSTED_MESSAGE}\n"
|
||||
f"**Error:** ```{err}```"
|
||||
)
|
||||
try:
|
||||
ops_alerts.send_throttled_ops_alert(_ALERT_KEY, message, throttle_minutes=int(window / 60))
|
||||
except Exception: # noqa: BLE001 - alerting must never raise
|
||||
logger.warning("Gemini quota alert failed", exc_info=True)
|
||||
# Let the next failure in this instance try again.
|
||||
with _alert_lock:
|
||||
_last_alert_attempt = 0.0
|
||||
@@ -176,15 +176,11 @@ class LyricsTranslationService:
|
||||
# ---- model ----
|
||||
|
||||
def _call_gemini(self, user_prompt: str) -> Any:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=self.settings.google_cloud_project,
|
||||
location="global",
|
||||
http_options=types.HttpOptions(timeout=90_000), # 2 attempts: render waits <= ~3 min
|
||||
)
|
||||
from backend.services.gemini_client import get_genai_client
|
||||
|
||||
client = get_genai_client(timeout_ms=90_000) # 2 attempts: render waits <= ~3 min
|
||||
response = client.models.generate_content(
|
||||
model=self.model,
|
||||
contents=[user_prompt],
|
||||
@@ -245,6 +241,14 @@ class LyricsTranslationService:
|
||||
except (LyricsTranslationError, ValueError) as exc: # malformed output: retry
|
||||
last_error = exc
|
||||
except Exception as exc:
|
||||
from backend.services.gemini_client import note_gemini_failure
|
||||
|
||||
if note_gemini_failure("lyrics_translation", exc):
|
||||
# Quota/credit/key problem: don't retry; the render goes ahead
|
||||
# without the translation row.
|
||||
raise LyricsTranslationError(
|
||||
"translation service unavailable (Gemini quota/billing) — rendered without translation"
|
||||
) from exc
|
||||
if not _is_transient(exc):
|
||||
raise LyricsTranslationError(f"translation model error: {exc}") from exc
|
||||
last_error = exc
|
||||
|
||||
@@ -149,18 +149,14 @@ async def _default_generate(model: str, system_prompt: str, user_prompt: str) ->
|
||||
def _blocking_generate(model: str, system_prompt: str, user_prompt: str) -> dict:
|
||||
import copy
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
from backend.services.gemini_client import get_genai_client
|
||||
|
||||
timeout_ms = int(getattr(settings, "match_judge_timeout_ms", 3000))
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=settings.google_cloud_project,
|
||||
location="global",
|
||||
http_options=types.HttpOptions(timeout=timeout_ms),
|
||||
)
|
||||
client = get_genai_client(timeout_ms=timeout_ms)
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=[user_prompt],
|
||||
|
||||
@@ -141,6 +141,9 @@ async def resolve_free_text(query: str, *, generate: Optional[Generate] = None,
|
||||
data = await (generate or _default_generate)(
|
||||
model or _model(), _SYSTEM_PROMPT, f'Singer typed: "{query}"')
|
||||
except Exception as e:
|
||||
from backend.services.gemini_client import note_gemini_failure
|
||||
|
||||
note_gemini_failure("match_judge_free_text", e)
|
||||
logger.warning(f"resolve_free_text failed: {e}")
|
||||
return _none(query, "unavailable")
|
||||
return verdict_from_response(data, query)
|
||||
@@ -200,17 +203,13 @@ async def _default_generate(model: str, system_prompt: str, user_prompt: str) ->
|
||||
|
||||
|
||||
def _blocking_generate(model: str, system_prompt: str, user_prompt: str) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=settings.google_cloud_project,
|
||||
location="global",
|
||||
http_options=types.HttpOptions(timeout=int(getattr(settings, "match_judge_timeout_ms", 12000))),
|
||||
)
|
||||
from backend.services.gemini_client import get_genai_client
|
||||
|
||||
client = get_genai_client(timeout_ms=int(getattr(settings, "match_judge_timeout_ms", 12000)))
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=[user_prompt],
|
||||
|
||||
@@ -126,7 +126,10 @@ async def judge_match(
|
||||
|
||||
try:
|
||||
return await ai(artist, title, candidates, audio_tier)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
from backend.services.gemini_client import note_gemini_failure
|
||||
|
||||
note_gemini_failure("match_judge", exc)
|
||||
# Graceful degradation: returning no-suggestion is a safe fallback (the
|
||||
# user just gets no auto-match). Log at WARNING (with traceback) so a
|
||||
# transient AI blip does not page as a red "new error pattern".
|
||||
|
||||
@@ -134,7 +134,9 @@ def _signature(error_message: str) -> str:
|
||||
return hashlib.sha256((error_message or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _should_alert(db: Any, signature: str, now: datetime) -> tuple[bool, bool, int, Any]:
|
||||
def _should_alert(
|
||||
db: Any, signature: str, now: datetime, throttle_minutes: Optional[int] = None
|
||||
) -> tuple[bool, bool, int, Any]:
|
||||
"""Decide whether to send, WITHOUT yet acknowledging delivery.
|
||||
|
||||
Returns ``(should_send, is_novel, suppressed_since_last, doc_ref)``:
|
||||
@@ -182,7 +184,8 @@ def _should_alert(db: Any, signature: str, now: datetime) -> tuple[bool, bool, i
|
||||
if last_alerted_raw:
|
||||
try:
|
||||
last_alerted = datetime.fromisoformat(last_alerted_raw)
|
||||
within_window = (now - last_alerted) < timedelta(minutes=_throttle_minutes())
|
||||
window = throttle_minutes if throttle_minutes is not None else _throttle_minutes()
|
||||
within_window = (now - last_alerted) < timedelta(minutes=window)
|
||||
except (TypeError, ValueError):
|
||||
within_window = False
|
||||
|
||||
@@ -225,6 +228,36 @@ def _mark_alerted(doc_ref: Any, now: datetime) -> None:
|
||||
logger.debug("ops_alerts: could not record alert delivery: %s", exc)
|
||||
|
||||
|
||||
def send_throttled_ops_alert(key: str, message: str, throttle_minutes: int) -> bool:
|
||||
"""Send an ops alert at most once per ``throttle_minutes`` for ``key``.
|
||||
|
||||
The throttle lives in Firestore (same dedup collection as job-failure
|
||||
alerts), so it holds across every Cloud Run instance and job execution.
|
||||
Fails open (sends) if Firestore is unreachable. Best-effort; never raises.
|
||||
Returns True if an alert was sent.
|
||||
"""
|
||||
try:
|
||||
if not _is_enabled():
|
||||
return False
|
||||
from backend.services.firestore_service import get_firestore_client
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
should_send, _novel, suppressed, doc_ref = _should_alert(
|
||||
get_firestore_client(), key, now, throttle_minutes=throttle_minutes
|
||||
)
|
||||
if not should_send:
|
||||
return False
|
||||
if suppressed:
|
||||
message += f"\n_(Collapsed {suppressed} earlier occurrence(s).)_"
|
||||
sent = send_ops_alert(message)
|
||||
if sent:
|
||||
_mark_alerted(doc_ref, now)
|
||||
return sent
|
||||
except Exception as exc: # noqa: BLE001 - alerting must never raise
|
||||
logger.warning("ops_alerts: throttled alert %s failed: %s", key, exc)
|
||||
return False
|
||||
|
||||
|
||||
def _retry_pending(state_data: dict) -> bool:
|
||||
"""True if state_data.cloud_run_retry_pending is set and unexpired."""
|
||||
pending = (state_data or {}).get("cloud_run_retry_pending")
|
||||
|
||||
@@ -123,18 +123,14 @@ async def _default_generate(model: str, system: str, user: str) -> dict:
|
||||
|
||||
|
||||
def _blocking_generate(model: str, system: str, user: str) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
from backend.services.gemini_client import get_genai_client
|
||||
|
||||
timeout_ms = int(getattr(settings, "parse_titles_timeout_ms", 20000))
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=settings.google_cloud_project,
|
||||
location="global",
|
||||
http_options=types.HttpOptions(timeout=timeout_ms),
|
||||
)
|
||||
client = get_genai_client(timeout_ms=timeout_ms)
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=[user],
|
||||
|
||||
@@ -56,6 +56,9 @@ async def _parse_chunk(chunk: list[dict], *, model, generate) -> list[dict]:
|
||||
try:
|
||||
results = await ai.ai_parse(chunk, model=model, generate=generate)
|
||||
except Exception as exc:
|
||||
from backend.services.gemini_client import note_gemini_failure
|
||||
|
||||
note_gemini_failure("parse_titles", exc)
|
||||
logger.warning(
|
||||
"parse_titles chunk degraded (%s); blanks for %d items", exc, len(chunk))
|
||||
return _blanks(chunk)
|
||||
|
||||
@@ -431,6 +431,9 @@ def analyze_filenames(
|
||||
llm_rows, leftover = _llm_analyze(leftover, generate)
|
||||
rows.extend(llm_rows)
|
||||
except Exception as e: # pragma: no cover - resilience guard
|
||||
from backend.services.gemini_client import note_gemini_failure
|
||||
|
||||
note_gemini_failure("tenant_bulk_analyze", e)
|
||||
logger.warning("Bulk-analyze LLM pass failed, using regex result: %s", e)
|
||||
|
||||
unpaired = [_describe_unpaired(fn) for fn in leftover]
|
||||
@@ -438,7 +441,7 @@ def analyze_filenames(
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default Vertex-Gemini generate (mirrors backend.services.match_judge.ai)
|
||||
# Default Gemini generate (mirrors backend.services.match_judge.ai)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _default_model() -> str:
|
||||
@@ -451,22 +454,18 @@ def _default_model() -> str:
|
||||
|
||||
|
||||
def default_generate(system_prompt: str, user_prompt: str) -> dict:
|
||||
"""Blocking Vertex-Gemini call producing schema-constrained JSON.
|
||||
"""Blocking Gemini (Developer API) call producing schema-constrained JSON.
|
||||
|
||||
Wrap in ``asyncio.to_thread`` when calling from async code.
|
||||
"""
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
from backend.services.gemini_client import get_genai_client
|
||||
|
||||
timeout_ms = int(getattr(settings, "tenant_bulk_timeout_ms", 30000))
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=settings.google_cloud_project,
|
||||
location="global",
|
||||
http_options=types.HttpOptions(timeout=timeout_ms),
|
||||
)
|
||||
client = get_genai_client(timeout_ms=timeout_ms)
|
||||
response = client.models.generate_content(
|
||||
model=_default_model(),
|
||||
contents=[user_prompt],
|
||||
|
||||
@@ -18,6 +18,8 @@ if 'FIRESTORE_EMULATOR_HOST' not in os.environ:
|
||||
os.environ.setdefault('GCS_BUCKET_NAME', 'test-bucket')
|
||||
os.environ.setdefault('FIRESTORE_COLLECTION', 'jobs')
|
||||
os.environ.setdefault('ENVIRONMENT', 'test')
|
||||
# Gemini Developer API key: never let a test fall through to Secret Manager.
|
||||
os.environ.setdefault('GEMINI_API_KEY', 'test-gemini-key')
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, AsyncMock, patch
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Tests for backend.services.gemini_client (Gemini Developer API helper) and the
|
||||
per-caller graceful degradation when Gemini's quota/credit/key is unusable."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.services import gemini_client
|
||||
from backend.services.gemini_client import (
|
||||
GeminiKeyUnavailableError,
|
||||
get_api_key,
|
||||
get_genai_client,
|
||||
is_quota_or_billing_error,
|
||||
note_gemini_failure,
|
||||
)
|
||||
|
||||
|
||||
class FakeAPIError(Exception):
|
||||
"""Mimics google.genai.errors.APIError (numeric ``code`` + ``status``)."""
|
||||
|
||||
def __init__(self, code, status, message=""):
|
||||
super().__init__(f"{code} {status}. {message}")
|
||||
self.code = code
|
||||
self.status = status
|
||||
|
||||
|
||||
QUOTA = FakeAPIError(429, "RESOURCE_EXHAUSTED", "Your prepayment credits are depleted")
|
||||
RATE_LIMIT = FakeAPIError(
|
||||
429, "RESOURCE_EXHAUSTED",
|
||||
"quotaId: GenerateRequestsPerMinutePerProjectPerModel-PaidTier, retryDelay: 5s",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_alert_throttle(monkeypatch):
|
||||
monkeypatch.setattr(gemini_client, "_last_alert_attempt", 0.0)
|
||||
monkeypatch.setattr(gemini_client, "_ALERT_IN_BACKGROUND", False)
|
||||
|
||||
|
||||
# ---- classifier -------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
QUOTA,
|
||||
FakeAPIError(429, "RESOURCE_EXHAUSTED", "You exceeded your current quota"),
|
||||
FakeAPIError(403, "PERMISSION_DENIED", "Generative Language API has not been used"),
|
||||
FakeAPIError(400, "INVALID_ARGUMENT", "API key not valid. Please pass a valid API key."),
|
||||
FakeAPIError(400, "FAILED_PRECONDITION", "check your plan and billing details"),
|
||||
RuntimeError("API key expired. Please renew the API key."),
|
||||
GeminiKeyUnavailableError("no key"),
|
||||
RuntimeError("429 RESOURCE_EXHAUSTED quotaId GenerateRequestsPerDayPerProjectPerModel, retryDelay 3600s"),
|
||||
],
|
||||
)
|
||||
def test_classifier_positive(exc):
|
||||
assert is_quota_or_billing_error(exc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
RATE_LIMIT,
|
||||
FakeAPIError(500, "INTERNAL", "Internal error"),
|
||||
FakeAPIError(503, "UNAVAILABLE", "The model is overloaded"),
|
||||
FakeAPIError(400, "INVALID_ARGUMENT", "Request contains an invalid argument."),
|
||||
TimeoutError("timed out"),
|
||||
ValueError("processed 429 tokens"),
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_classifier_negative(exc):
|
||||
assert not is_quota_or_billing_error(exc)
|
||||
|
||||
|
||||
def test_classifier_walks_cause_chain():
|
||||
try:
|
||||
try:
|
||||
raise QUOTA
|
||||
except FakeAPIError as inner:
|
||||
raise RuntimeError("wrapped") from inner
|
||||
except RuntimeError as outer:
|
||||
assert is_quota_or_billing_error(outer)
|
||||
|
||||
|
||||
# ---- key + client -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_env_key_wins(monkeypatch):
|
||||
monkeypatch.setenv("GEMINI_API_KEY", " env-key ")
|
||||
with patch("backend.config.Settings.get_secret") as get_secret:
|
||||
assert get_api_key() == "env-key"
|
||||
get_secret.assert_not_called()
|
||||
|
||||
|
||||
def test_falls_back_to_secret_manager(monkeypatch):
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
with patch("backend.config.Settings.get_secret", return_value="sm-key") as get_secret:
|
||||
assert get_api_key() == "sm-key"
|
||||
get_secret.assert_called_once_with("gemini-api-key")
|
||||
|
||||
|
||||
def test_missing_key_raises(monkeypatch):
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
with patch("backend.config.Settings.get_secret", return_value=None):
|
||||
with pytest.raises(GeminiKeyUnavailableError):
|
||||
get_api_key()
|
||||
|
||||
|
||||
def test_client_uses_api_key_never_vertex(monkeypatch):
|
||||
monkeypatch.setenv("GEMINI_API_KEY", "k")
|
||||
with patch("google.genai.Client") as client_cls:
|
||||
get_genai_client(timeout_ms=1234)
|
||||
kwargs = client_cls.call_args.kwargs
|
||||
assert kwargs["api_key"] == "k"
|
||||
assert kwargs["http_options"].timeout == 1234
|
||||
assert kwargs["vertexai"] is False and "project" not in kwargs and "location" not in kwargs
|
||||
|
||||
|
||||
# ---- note_gemini_failure / alert throttle --------------------------------------
|
||||
|
||||
|
||||
def test_note_failure_ignores_ordinary_errors():
|
||||
with patch.object(gemini_client, "_maybe_alert") as alert:
|
||||
assert note_gemini_failure("x", RATE_LIMIT) is False
|
||||
alert.assert_not_called()
|
||||
|
||||
|
||||
def test_note_failure_alerts_once_per_window():
|
||||
with patch("backend.services.ops_alerts.send_throttled_ops_alert", return_value=True) as send:
|
||||
assert note_gemini_failure("auto_correct", QUOTA) is True
|
||||
assert note_gemini_failure("match_judge", QUOTA) is True
|
||||
send.assert_called_once()
|
||||
key, message = send.call_args.args
|
||||
assert key == "gemini-quota-exhausted"
|
||||
assert "auto_correct" in message and "rotate gemini-api-key" in message
|
||||
assert send.call_args.kwargs["throttle_minutes"] == 360
|
||||
|
||||
|
||||
def test_note_failure_never_raises_and_rearms_after_failed_send():
|
||||
with patch("backend.services.ops_alerts.send_throttled_ops_alert", side_effect=RuntimeError) as send:
|
||||
assert note_gemini_failure("x", QUOTA) is True
|
||||
assert note_gemini_failure("x", QUOTA) is True
|
||||
# A failed send doesn't silence the instance for 6h.
|
||||
assert send.call_count == 2
|
||||
|
||||
|
||||
def test_alert_runs_in_background_thread(monkeypatch):
|
||||
import threading
|
||||
|
||||
monkeypatch.setattr(gemini_client, "_ALERT_IN_BACKGROUND", True)
|
||||
done = threading.Event()
|
||||
seen = {}
|
||||
|
||||
def fake_send(*_a, **_k):
|
||||
seen["thread"] = threading.current_thread().name
|
||||
done.set()
|
||||
return True
|
||||
|
||||
with patch("backend.services.ops_alerts.send_throttled_ops_alert", side_effect=fake_send):
|
||||
assert note_gemini_failure("x", QUOTA) is True
|
||||
assert done.wait(5)
|
||||
assert seen["thread"] == "gemini-quota-alert"
|
||||
|
||||
|
||||
def test_throttled_ops_alert_respects_firestore_window(monkeypatch):
|
||||
from backend.services import ops_alerts
|
||||
|
||||
monkeypatch.setenv("FAILURE_ALERTS_ENABLED", "true")
|
||||
db = MagicMock()
|
||||
with patch("backend.services.firestore_service.get_firestore_client", return_value=db), \
|
||||
patch.object(ops_alerts, "_should_alert", return_value=(False, False, 0, None)) as should, \
|
||||
patch.object(ops_alerts, "send_ops_alert") as send:
|
||||
assert ops_alerts.send_throttled_ops_alert("k", "msg", throttle_minutes=360) is False
|
||||
send.assert_not_called()
|
||||
assert should.call_args.kwargs["throttle_minutes"] == 360
|
||||
|
||||
with patch("backend.services.firestore_service.get_firestore_client", return_value=db), \
|
||||
patch.object(ops_alerts, "_should_alert", return_value=(True, False, 2, "ref")), \
|
||||
patch.object(ops_alerts, "send_ops_alert", return_value=True) as send, \
|
||||
patch.object(ops_alerts, "_mark_alerted") as mark:
|
||||
assert ops_alerts.send_throttled_ops_alert("k", "msg", throttle_minutes=360) is True
|
||||
assert "Collapsed 2" in send.call_args.args[0]
|
||||
mark.assert_called_once()
|
||||
|
||||
|
||||
# ---- per-caller degradation ----------------------------------------------------
|
||||
|
||||
|
||||
def _quota_client():
|
||||
client = MagicMock()
|
||||
client.models.generate_content.side_effect = QUOTA
|
||||
return client
|
||||
|
||||
|
||||
def test_auto_correct_quota_is_non_retryable_and_skips_gemini_leg():
|
||||
from backend.services.auto_correct.service import AutoCorrectService, AutoCorrectServiceError
|
||||
|
||||
service = AutoCorrectService()
|
||||
with patch("backend.services.gemini_client.get_genai_client", return_value=_quota_client()), \
|
||||
patch.object(gemini_client, "_maybe_alert"), \
|
||||
patch("backend.services.auto_correct.service.time.sleep") as sleep:
|
||||
with pytest.raises(AutoCorrectServiceError) as ei:
|
||||
service._call_model("gemini-3.8-flash", "sys", "user", job_id="j")
|
||||
assert ei.value.retryable is False and ei.value.status_code == 503
|
||||
sleep.assert_not_called()
|
||||
|
||||
|
||||
def test_auto_correct_compare_keeps_opus_when_gemini_quota_exhausted():
|
||||
from backend.services.auto_correct.service import AutoCorrectService, AutoCorrectServiceError
|
||||
from backend.services.auto_correct.settings import AutoCorrectSettings
|
||||
|
||||
segments = [{"id": "s1", "words": [{"id": "w0", "text": "glory"}]}]
|
||||
refs = {"genius": {"segments": [{"text": "chlorine"}]}}
|
||||
sugg = {"op": "replace", "start_idx": 0, "end_idx": 0, "new_text": "chlorine",
|
||||
"reason": "r", "category": "mishearing", "confidence": 0.9}
|
||||
|
||||
def fake_call(model, *_a, **_k):
|
||||
if model.startswith("gemini"):
|
||||
raise AutoCorrectServiceError("Gemini is unavailable", status_code=503)
|
||||
return {"suggestions": [sugg]}, None
|
||||
|
||||
service = AutoCorrectService()
|
||||
with patch.object(service.settings, "auto_correct_compare_models", "claude-opus-5-5;gemini-3.8-flash"), \
|
||||
patch.object(service, "_cache_get", return_value=None), \
|
||||
patch.object(service, "_cache_put"), \
|
||||
patch.object(service, "_call_model", side_effect=fake_call):
|
||||
result = service.suggest(
|
||||
job_id="j", segments=segments, reference_lyrics=refs, artist="A", title="T",
|
||||
settings=AutoCorrectSettings(compare_models=True),
|
||||
)
|
||||
assert result.model == "claude-opus-5-5"
|
||||
assert any("gemini-3.8-flash failed" in w for w in result.warnings)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_match_judge_quota_returns_no_suggestion_and_alerts():
|
||||
from backend.services.match_judge import service as mj
|
||||
|
||||
async def no_candidates(*_a, **_k):
|
||||
return []
|
||||
|
||||
async def boom(*_a, **_k):
|
||||
raise QUOTA
|
||||
|
||||
with patch.object(gemini_client, "_maybe_alert") as alert:
|
||||
verdict = await mj.judge_match(
|
||||
"A", "T", search_tracks=no_candidates, ai_judge=boom, enabled=True
|
||||
)
|
||||
assert verdict.reason == "ai failed"
|
||||
alert.assert_called_once()
|
||||
|
||||
|
||||
def test_custom_lyrics_quota_is_clean_503():
|
||||
from backend.services.custom_lyrics.settings import GenerationSettings
|
||||
from backend.services.custom_lyrics_service import CustomLyricsService, CustomLyricsServiceError
|
||||
|
||||
service = CustomLyricsService()
|
||||
with patch("backend.services.gemini_client.get_genai_client", return_value=_quota_client()), \
|
||||
patch.object(gemini_client, "_maybe_alert"):
|
||||
with pytest.raises(CustomLyricsServiceError) as ei:
|
||||
service._call_gemini(system_prompt="s", user_prompt="u", pdf_bytes=None,
|
||||
settings=GenerationSettings())
|
||||
assert ei.value.status_code == 503
|
||||
assert "temporarily unavailable" in str(ei.value)
|
||||
|
||||
|
||||
def test_credit_eval_quota_goes_to_manual_review():
|
||||
from backend.services.credit_evaluation_service import CreditEvaluationService
|
||||
|
||||
svc = CreditEvaluationService.__new__(CreditEvaluationService)
|
||||
svc.settings = MagicMock(credit_eval_enabled=True, credit_eval_model="gemini-3.8-flash")
|
||||
signals = {"fingerprint_matches": [{"x": 1}], "ip_matches": [], "recent_signups_ip": 0,
|
||||
"recent_signups_fp": 0, "user_data": {}, "ip_geo": None, "user_agent": None}
|
||||
with patch.object(svc, "_collect_signals", return_value=signals), \
|
||||
patch.object(svc, "_log_evaluation"), \
|
||||
patch("backend.services.gemini_client.get_genai_client", return_value=_quota_client()), \
|
||||
patch.object(gemini_client, "_maybe_alert") as alert:
|
||||
evaluation = svc.evaluate("a@b.c", "welcome")
|
||||
assert evaluation.decision == "pending_review"
|
||||
alert.assert_called_once()
|
||||
|
||||
|
||||
def test_error_monitor_llm_does_not_retry_on_quota():
|
||||
from backend.services.error_monitor import llm_analysis
|
||||
|
||||
with patch.object(llm_analysis, "get_genai_client", return_value=_quota_client()), \
|
||||
patch.object(gemini_client, "_maybe_alert"), \
|
||||
patch.object(llm_analysis.time, "sleep") as sleep:
|
||||
with pytest.raises(FakeAPIError):
|
||||
llm_analysis._call_llm("sys", "user")
|
||||
sleep.assert_not_called()
|
||||
|
||||
|
||||
def test_classifier_ignores_implicit_context():
|
||||
"""A Gemini-unrelated 403 raised earlier must not be blamed on Gemini."""
|
||||
try:
|
||||
try:
|
||||
raise FakeAPIError(403, "PERMISSION_DENIED", "firestore says no")
|
||||
except FakeAPIError:
|
||||
raise ValueError("later, unrelated failure") # implicit __context__ only
|
||||
except ValueError as exc:
|
||||
assert not is_quota_or_billing_error(exc)
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -103,11 +104,11 @@ def test_text_only_happy_path(service: CustomLyricsService) -> None:
|
||||
assert result.line_count_mismatch is False
|
||||
assert result.iterations_used == 0
|
||||
assert result.model == "gemini-3.1-pro-preview"
|
||||
assert mock_client_cls.call_args.kwargs == {
|
||||
"vertexai": True,
|
||||
"project": "test-project",
|
||||
"location": "global",
|
||||
}
|
||||
# Gemini Developer API (API key) — never Vertex AI.
|
||||
kwargs = mock_client_cls.call_args.kwargs
|
||||
assert kwargs["api_key"] == os.environ["GEMINI_API_KEY"]
|
||||
assert kwargs["vertexai"] is False and "project" not in kwargs
|
||||
assert kwargs["http_options"].timeout == 300_000
|
||||
|
||||
|
||||
def test_docx_input_parsed_to_text(
|
||||
|
||||
@@ -106,7 +106,8 @@ def test_gives_up_after_max_attempts():
|
||||
|
||||
|
||||
def test_transient_error_retried_permanent_error_raised():
|
||||
transient = Exception("429 RESOURCE_EXHAUSTED")
|
||||
# Per-minute rate limit: transient, retried.
|
||||
transient = Exception("429 RESOURCE_EXHAUSTED quotaId GenerateRequestsPerMinutePerProjectPerModel")
|
||||
svc = make_service([transient, {"translations": ["uno"]}])
|
||||
assert svc.translate_lines(["one"], "es") == ["uno"]
|
||||
|
||||
@@ -116,6 +117,19 @@ def test_transient_error_retried_permanent_error_raised():
|
||||
assert svc._call_gemini.call_count == 1
|
||||
|
||||
|
||||
def test_quota_exhausted_not_retried_and_alerts():
|
||||
class QuotaError(Exception):
|
||||
code = 403
|
||||
status = "PERMISSION_DENIED"
|
||||
|
||||
svc = make_service([QuotaError("Your prepayment credits are depleted")])
|
||||
with patch("backend.services.gemini_client._maybe_alert") as alert:
|
||||
with pytest.raises(LyricsTranslationError, match="unavailable"):
|
||||
svc.translate_lines(["one"], "es")
|
||||
assert svc._call_gemini.call_count == 1
|
||||
alert.assert_called_once()
|
||||
|
||||
|
||||
def test_unsupported_language_raises():
|
||||
with pytest.raises(LyricsTranslationError):
|
||||
make_service([]).translate_lines(["one"], "xx")
|
||||
|
||||
+3
-3
@@ -736,7 +736,7 @@ Content-Type: application/json
|
||||
|
||||
AI correction suggestions for the lyrics review UI. Stateless: one whole-song
|
||||
LLM call per model (Claude models use the Anthropic API via the
|
||||
`anthropic-api-key` secret, Gemini models use Vertex AI) compares the
|
||||
`anthropic-api-key` secret, Gemini models use the Gemini Developer API, key `gemini-api-key`) compares the
|
||||
client's current transcription against the reference sources and returns
|
||||
word-id-keyed suggestions. Nothing is applied server-side — the reviewer
|
||||
accepts/rejects each suggestion in the UI and persistence flows through the
|
||||
@@ -1017,7 +1017,7 @@ Content-Type: application/json
|
||||
```
|
||||
|
||||
Decides the official formatting for a typed artist/title and whether it matches a
|
||||
real song. A deterministic normalizer + the catalog run first; a light Vertex
|
||||
real song. A deterministic normalizer + the catalog run first; a light Gemini
|
||||
Gemini model (`MATCH_JUDGE_MODEL`, default `gemini-3.8-flash`) is consulted
|
||||
**only** when those aren't confident. `audio_confidence_tier` (1=strong..3=weak,
|
||||
optional) lets the judge tell whether weak audio results hint at a typo. The call
|
||||
@@ -1340,7 +1340,7 @@ Artist/Title. Pure analysis — filenames only, no uploads, no state written.
|
||||
- **Auth:** tenant session required; gated on the tenant's `features.bulk_upload` flag (403 otherwise).
|
||||
- **Limits:** max 100 audio files per request (non-audio files don't count toward the cap; 2000-filename payload guard).
|
||||
- **Two passes:** deterministic regex for the `S<code>-<1|2> Artist - Title <Guide|BV|Instru>`
|
||||
convention, then a Vertex-Gemini pass (`backend/services/tenant_bulk/analyze.py`) for
|
||||
convention, then a Gemini pass (`backend/services/tenant_bulk/analyze.py`) for
|
||||
whatever the regex couldn't confidently pair. Falls back to regex-only if the model errors.
|
||||
|
||||
Response:
|
||||
|
||||
@@ -252,7 +252,7 @@ LyricsTranscriber LyricsTranscriber
|
||||
|---------|---------|----------|
|
||||
| Cloud Run GPU Job (L4, us-east4) | Audio stem separation — `Separator` runs directly in the audio worker Job (no HTTP hop) | Yes |
|
||||
| AudioShake | Lyrics transcription | Yes |
|
||||
| Vertex AI | Agentic AI correction (Gemini 3.8 Flash) | Default off (SKIP_CORRECTION=false to enable) |
|
||||
| Gemini Developer API (`gemini-api-key`) | Agentic AI correction (Gemini 3.8 Flash) | Default off (SKIP_CORRECTION=false to enable) |
|
||||
| Genius | Reference lyrics | Yes |
|
||||
| Flacfetch | Audio downloads (YouTube, torrents) | Recommended* |
|
||||
| YouTube API | Video upload | Optional |
|
||||
|
||||
@@ -303,7 +303,7 @@ Some external services lock to redirect URIs or treat the old GCP project ID as
|
||||
- YouTube OAuth (redirect URIs)
|
||||
- Stripe webhook endpoint URL (secret regenerates)
|
||||
- AudioShake API key (rotate as a precaution if old project was compromised)
|
||||
- Gemini / Vertex API (project-scoped — needs new project enabled)
|
||||
- Gemini Developer API key (`gemini-api-key` secret; key lives in an AI Studio project outside nomadkaraoke — just copy the secret)
|
||||
- Discord webhook URLs (re-issue if old GCP exposure could have leaked them)
|
||||
- Cloudflare API tokens (rotate)
|
||||
|
||||
|
||||
+11
-5
@@ -1403,11 +1403,17 @@ For indefinite loops (like polling), use outer timeout (total duration) + inner
|
||||
### Secret Manager Access
|
||||
Service account needs `roles/secretmanager.secretAccessor`. Mount as env var or volume. Use GCP workload identity, not key files.
|
||||
|
||||
### Vertex AI Auth
|
||||
Use `project` parameter with `ChatGoogleGenerativeAI` to trigger Vertex AI backend with ADC. Don't require `GOOGLE_API_KEY`.
|
||||
|
||||
### Gemini 3 Requires Global Location
|
||||
Gemini 3 models require `location="global"`, not regional endpoints like `us-central1`.
|
||||
### Gemini: Developer API key, never Vertex (2026-10-01)
|
||||
All Gemini calls use the **Gemini Developer API** (AI Studio key in Secret Manager
|
||||
`gemini-api-key`, billed outside the nomadkaraoke project) via
|
||||
`backend/services/gemini_client.get_genai_client()`; `aiplatform.googleapis.com` and
|
||||
`generativelanguage.googleapis.com` are **disabled** in nomadkaraoke so no AI spend lands
|
||||
on it. Key = `GEMINI_API_KEY` env (CI `--set-secrets` for the service, Pulumi for jobs),
|
||||
else Secret Manager read at runtime. Rotate = add a secret version (new instances/job
|
||||
executions pick it up). Classify failures with `note_gemini_failure(caller, exc)`:
|
||||
quota/billing/key errors (not per-minute 429s) skip retries, degrade, and Discord-alert
|
||||
at most once per 6h. (Supersedes the old "Vertex AI Auth" / "global location" notes —
|
||||
the Developer API has no project/location.)
|
||||
|
||||
### LangChain: REST over gRPC
|
||||
Use `langchain-google-genai` (REST) instead of `langchain-google-vertexai` (gRPC) to avoid silent hangs.
|
||||
|
||||
@@ -132,7 +132,7 @@ Commercial producers pay for API access or custom integration to generate tracks
|
||||
### What Works
|
||||
- Audio upload and separation (Cloud Run GPU)
|
||||
- Lyrics transcription (AudioShake API)
|
||||
- Agentic AI correction (Gemini 3 Flash via Vertex AI)
|
||||
- Agentic AI correction (Gemini 3 Flash via the Gemini Developer API)
|
||||
- Human lyrics review (React UI)
|
||||
- Preview video generation
|
||||
- Instrumental selection (clean vs. with backing vocals)
|
||||
@@ -190,7 +190,7 @@ Commercial producers pay for API access or custom integration to generate tracks
|
||||
- **Secrets**: Google Secret Manager
|
||||
- **Audio Separation**: Cloud Run GPU (L4, us-east4)
|
||||
- **Lyrics Transcription**: AudioShake API
|
||||
- **AI Correction**: Vertex AI (Gemini 3 Flash)
|
||||
- **AI Correction**: Gemini Developer API (Gemini 3 Flash)
|
||||
- **Payments**: Stripe
|
||||
- **Email**: Postmark
|
||||
- **IaC**: Pulumi
|
||||
|
||||
@@ -57,6 +57,8 @@
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- **Gemini moved off Vertex AI** (2026-10-01, v0.262.0): every Gemini call (auto-correct compare leg, match-judge, parse-titles, credit eval, tenant bulk analyze, custom lyrics, lyrics translation, error-monitor LLM, agentic correction, translate.py) now uses the Gemini Developer API with the `gemini-api-key` secret via `backend/services/gemini_client.py`, so no AI spend lands on the nomadkaraoke project (Vertex APIs disabled there). Quota/billing/key failures degrade per caller and send one Discord ops alert per 6h. See LESSONS-LEARNED "Gemini: Developer API key, never Vertex".
|
||||
|
||||
- **Translated lyrics follow-ups + review waveform spinner** (2026-10-01, v0.261.0):
|
||||
- The translation row now renders at 70% of the unsung colour's opacity, with outline and shadow faded to match, so it reads as secondary to the sung line (`subtitles.py:TRANSLATION_OPACITY`). A theme's `karaoke.translation_color` still wins.
|
||||
- Published videos with translations get "(With Translation into X)" in the YouTube title, before "(Karaoke)". Only the artist/title get truncated, never the label. The description also gets a notice. This applies to the live upload and the quota queue, but only when `state_data.lyrics_translation.status == "translated"` (`youtube_description.build_youtube_title`).
|
||||
|
||||
@@ -99,7 +99,7 @@ the same KaraokeNerds check the normal job-submission flow uses runs at two poin
|
||||
- **Auth:** reuses gen's passwordless magic-link system. Board sign-in sends
|
||||
`purpose="requests_board"`, which creates an identity **without** granting a welcome credit and
|
||||
with a higher per-IP signup cap (so a whole venue on shared WiFi can sign in).
|
||||
- **Auto-correct:** reuses `match_judge` (Vertex Gemini) — the same artist/title canonicalizer the
|
||||
- **Auto-correct:** reuses `match_judge` (Gemini) — the same artist/title canonicalizer the
|
||||
job-submission flow uses.
|
||||
- **Storage:** two Firestore collections — `song_requests` (one doc per song, with a denormalized
|
||||
`vote_count`) and `song_request_votes` (one doc per person per day, id `{email}__{YYYY-MM-DD}`,
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Gemini Developer API (Google AI Studio) client helper for local scripts.
|
||||
|
||||
All Gemini calls go through the Developer API with an API key, NOT Vertex AI
|
||||
(Vertex is disabled in the nomadkaraoke GCP project to keep AI spend off it).
|
||||
|
||||
Key resolution (first match wins):
|
||||
1. ``GEMINI_API_KEY`` environment variable
|
||||
2. ``gcloud secrets versions access latest --secret=gemini-api-key
|
||||
--project=nomadkaraoke`` (your own gcloud login)
|
||||
|
||||
The key is cached in-process only; it is never written to disk or printed.
|
||||
|
||||
Rotating the key (e.g. prepaid credit exhausted): add a new secret version —
|
||||
no code change needed:
|
||||
printf '%s' "$NEW_KEY" | gcloud secrets versions add gemini-api-key \
|
||||
--project=nomadkaraoke --data-file=-
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
SECRET_NAME = "gemini-api-key"
|
||||
SECRET_PROJECT = "nomadkaraoke"
|
||||
|
||||
QUOTA_EXHAUSTED_MESSAGE = (
|
||||
"Gemini quota/credit exhausted — top up AI Studio or rotate gemini-api-key "
|
||||
f"(add a new version of Secret Manager secret '{SECRET_NAME}' in project "
|
||||
f"'{SECRET_PROJECT}')."
|
||||
)
|
||||
|
||||
|
||||
def quota_exhausted_message() -> str:
|
||||
"""QUOTA_EXHAUSTED_MESSAGE plus a hint when the key came from the environment."""
|
||||
if os.environ.get("GEMINI_API_KEY", "").strip():
|
||||
return (
|
||||
QUOTA_EXHAUSTED_MESSAGE
|
||||
+ " Note: the key came from the GEMINI_API_KEY environment variable — "
|
||||
"unset it to use the Secret Manager key instead."
|
||||
)
|
||||
return QUOTA_EXHAUSTED_MESSAGE
|
||||
|
||||
|
||||
class GeminiKeyUnavailableError(RuntimeError):
|
||||
"""The Gemini API key could not be resolved (env var unset and gcloud failed)."""
|
||||
|
||||
|
||||
_cached_key: str | None = None
|
||||
|
||||
|
||||
def get_api_key() -> str:
|
||||
"""Return the Gemini Developer API key (env var, else Secret Manager via gcloud)."""
|
||||
global _cached_key
|
||||
env_key = os.environ.get("GEMINI_API_KEY", "").strip()
|
||||
if env_key:
|
||||
return env_key
|
||||
if _cached_key:
|
||||
return _cached_key
|
||||
cmd = [
|
||||
"gcloud", "secrets", "versions", "access", "latest",
|
||||
f"--secret={SECRET_NAME}", f"--project={SECRET_PROJECT}",
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=60, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired) as e:
|
||||
raise GeminiKeyUnavailableError(
|
||||
f"GEMINI_API_KEY is not set and `gcloud secrets versions access` failed: {e}"
|
||||
) from e
|
||||
key = (result.stdout or "").strip()
|
||||
if result.returncode != 0 or not key:
|
||||
stderr = (result.stderr or "").strip()[-300:]
|
||||
raise GeminiKeyUnavailableError(
|
||||
"GEMINI_API_KEY is not set and reading Secret Manager secret "
|
||||
f"'{SECRET_NAME}' (project {SECRET_PROJECT}) failed — run `gcloud auth login` "
|
||||
f"or export GEMINI_API_KEY. gcloud said: {stderr}"
|
||||
)
|
||||
_cached_key = key
|
||||
return key
|
||||
|
||||
|
||||
def get_genai_client():
|
||||
"""Build a google-genai client for the Gemini Developer API."""
|
||||
from google import genai
|
||||
|
||||
# vertexai=False pins the Developer API even if GOOGLE_GENAI_USE_VERTEXAI is set.
|
||||
return genai.Client(vertexai=False, api_key=get_api_key())
|
||||
|
||||
|
||||
# Message fragments that mean "the key/account can't pay or isn't valid" — a
|
||||
# retry won't help; a human must top up credit or rotate the key.
|
||||
_QUOTA_PATTERNS = re.compile(
|
||||
r"RESOURCE_EXHAUSTED|exceeded your current quota|quota exceeded|"
|
||||
r"billing (details|account)|prepa(id|yment)|credits? (are |is |have been )?"
|
||||
r"(exhausted|depleted|used up)|insufficient (credit|funds|balance)|"
|
||||
r"API_KEY_INVALID|API key not valid|API key expired|API_KEY_SERVICE_BLOCKED|"
|
||||
r"\b429 Too Many Requests",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# A 429 that names a per-minute quota / carries a retry delay is an ordinary
|
||||
# rate limit (transient — retry with backoff), not exhausted credit.
|
||||
_RATE_LIMIT_PATTERNS = re.compile(r"PerMinute|retryDelay|retry in \d", re.IGNORECASE)
|
||||
# ...unless it names a per-day quota: that won't clear for hours — treat as exhausted.
|
||||
_DAILY_QUOTA_PATTERN = re.compile(r"PerDay", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_rate_limit(text: str) -> bool:
|
||||
return bool(_RATE_LIMIT_PATTERNS.search(text)) and not _DAILY_QUOTA_PATTERN.search(text)
|
||||
|
||||
|
||||
def is_quota_or_billing_error(exc: BaseException | None) -> bool:
|
||||
"""True if ``exc`` (or anything in its ``__cause__`` chain) is a Gemini quota,
|
||||
prepaid-credit, billing or API-key error — i.e. needs a top-up/key rotation,
|
||||
not a retry.
|
||||
|
||||
Matches google-genai ``APIError`` (``.code`` 429 / 403, or 400 with a
|
||||
billing/key message) and wrapped/stringified errors from other layers.
|
||||
Per-minute rate limits (429 naming a ``PerMinute`` quota or a retry delay)
|
||||
are NOT matched — those are transient; retry them.
|
||||
"""
|
||||
seen: set[int] = set()
|
||||
while exc is not None and id(exc) not in seen:
|
||||
seen.add(id(exc))
|
||||
if isinstance(exc, GeminiKeyUnavailableError):
|
||||
return True
|
||||
code = getattr(exc, "code", None)
|
||||
if code is None:
|
||||
code = getattr(exc, "status_code", None)
|
||||
status = str(getattr(exc, "status", "") or "")
|
||||
if code == 429 or status == "RESOURCE_EXHAUSTED":
|
||||
return not _is_rate_limit(str(exc))
|
||||
if code == 403 or status == "PERMISSION_DENIED":
|
||||
return True
|
||||
text = str(exc)
|
||||
if _QUOTA_PATTERNS.search(text) and not _is_rate_limit(text):
|
||||
return True
|
||||
# Only explicit chaining (`raise X from e`): implicit __context__ could
|
||||
# attribute an unrelated earlier error (e.g. a Firestore 403) to Gemini.
|
||||
exc = exc.__cause__
|
||||
return False
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Unit tests for scripts/gemini_client.py (run: python -m pytest scripts/test_gemini_client.py)."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
import gemini_client # noqa: E402
|
||||
from gemini_client import ( # noqa: E402
|
||||
GeminiKeyUnavailableError,
|
||||
get_api_key,
|
||||
is_quota_or_billing_error,
|
||||
)
|
||||
|
||||
|
||||
class FakeAPIError(Exception):
|
||||
"""Mimics google.genai.errors.APIError (numeric ``code`` + ``status``)."""
|
||||
|
||||
def __init__(self, code, status, message):
|
||||
super().__init__(f"{code} {status}. {message}")
|
||||
self.code = code
|
||||
self.status = status
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_cache(monkeypatch):
|
||||
monkeypatch.setattr(gemini_client, "_cached_key", None)
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
|
||||
|
||||
class TestIsQuotaOrBillingError:
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
FakeAPIError(429, "RESOURCE_EXHAUSTED", "You exceeded your current quota"),
|
||||
FakeAPIError(403, "PERMISSION_DENIED", "Your prepayment credits are depleted"),
|
||||
FakeAPIError(400, "INVALID_ARGUMENT", "API key not valid. Please pass a valid API key."),
|
||||
FakeAPIError(400, "FAILED_PRECONDITION", "Please check your plan and billing details"),
|
||||
RuntimeError("429 RESOURCE_EXHAUSTED quota"),
|
||||
RuntimeError("Your credits have been exhausted"),
|
||||
GeminiKeyUnavailableError("no key"),
|
||||
RuntimeError("429 RESOURCE_EXHAUSTED quotaId GenerateRequestsPerDayPerProjectPerModel, retryDelay 3600s"),
|
||||
],
|
||||
)
|
||||
def test_positive(self, exc):
|
||||
assert is_quota_or_billing_error(exc)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
FakeAPIError(500, "INTERNAL", "Internal error"),
|
||||
FakeAPIError(503, "UNAVAILABLE", "The model is overloaded"),
|
||||
FakeAPIError(400, "INVALID_ARGUMENT", "Request contains an invalid argument."),
|
||||
TimeoutError("timed out"),
|
||||
ValueError("bad JSON from model, processed 429 tokens"),
|
||||
FakeAPIError(
|
||||
429,
|
||||
"RESOURCE_EXHAUSTED",
|
||||
"You exceeded your current quota. quotaId: "
|
||||
"GenerateRequestsPerMinutePerProjectPerModel-PaidTier, retryDelay: 7s",
|
||||
),
|
||||
RuntimeError("429 RESOURCE_EXHAUSTED ... Please retry in 7.2s"),
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_negative(self, exc):
|
||||
assert not is_quota_or_billing_error(exc)
|
||||
|
||||
def test_walks_cause_chain(self):
|
||||
try:
|
||||
try:
|
||||
raise FakeAPIError(429, "RESOURCE_EXHAUSTED", "quota")
|
||||
except FakeAPIError as inner:
|
||||
raise RuntimeError("translation failed") from inner
|
||||
except RuntimeError as outer:
|
||||
assert is_quota_or_billing_error(outer)
|
||||
|
||||
|
||||
class TestGetApiKey:
|
||||
def test_env_var_wins(self, monkeypatch):
|
||||
monkeypatch.setenv("GEMINI_API_KEY", " from-env ")
|
||||
with mock.patch.object(subprocess, "run") as run:
|
||||
assert get_api_key() == "from-env"
|
||||
run.assert_not_called()
|
||||
|
||||
def test_falls_back_to_gcloud_and_caches(self):
|
||||
done = subprocess.CompletedProcess([], 0, stdout="from-secret\n", stderr="")
|
||||
with mock.patch.object(subprocess, "run", return_value=done) as run:
|
||||
assert get_api_key() == "from-secret"
|
||||
assert get_api_key() == "from-secret"
|
||||
assert run.call_count == 1
|
||||
cmd = run.call_args.args[0]
|
||||
assert "--secret=gemini-api-key" in cmd and "--project=nomadkaraoke" in cmd
|
||||
|
||||
def test_gcloud_failure_raises(self):
|
||||
failed = subprocess.CompletedProcess([], 1, stdout="", stderr="PERMISSION_DENIED")
|
||||
with mock.patch.object(subprocess, "run", return_value=failed):
|
||||
with pytest.raises(GeminiKeyUnavailableError):
|
||||
get_api_key()
|
||||
|
||||
def test_gcloud_missing_raises(self):
|
||||
with mock.patch.object(subprocess, "run", side_effect=FileNotFoundError("gcloud")):
|
||||
with pytest.raises(GeminiKeyUnavailableError):
|
||||
get_api_key()
|
||||
|
||||
|
||||
def test_quota_message_hints_env_key(monkeypatch):
|
||||
assert "environment" not in gemini_client.quota_exhausted_message()
|
||||
monkeypatch.setenv("GEMINI_API_KEY", "x")
|
||||
assert "GEMINI_API_KEY environment variable" in gemini_client.quota_exhausted_message()
|
||||
|
||||
|
||||
def test_client_pins_developer_api(monkeypatch):
|
||||
"""vertexai=False even if GOOGLE_GENAI_USE_VERTEXAI is set in the environment."""
|
||||
monkeypatch.setenv("GEMINI_API_KEY", "k")
|
||||
monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "true")
|
||||
with mock.patch("google.genai.Client") as client_cls:
|
||||
gemini_client.get_genai_client()
|
||||
kwargs = client_cls.call_args.kwargs
|
||||
assert kwargs["vertexai"] is False and kwargs["api_key"] == "k"
|
||||
@@ -2,7 +2,7 @@
|
||||
"""
|
||||
LLM Translation Pipeline for Nomad Karaoke i18n.
|
||||
|
||||
Two-pass translation using Gemini via Vertex AI:
|
||||
Two-pass translation using Gemini (Developer API):
|
||||
1. Translate English JSON to target language
|
||||
2. Review and polish translations for fluency
|
||||
|
||||
@@ -24,7 +24,9 @@ Usage:
|
||||
|
||||
Requires:
|
||||
- google-genai SDK (pip install google-genai)
|
||||
- GCP Application Default Credentials (gcloud auth application-default login)
|
||||
- Gemini Developer API key: GEMINI_API_KEY env var, else read from Secret
|
||||
Manager (gemini-api-key) via your gcloud login — see gemini_client.py
|
||||
- GCP Application Default Credentials for the GCS translation cache
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -34,14 +36,18 @@ import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from gemini_client import (
|
||||
quota_exhausted_message,
|
||||
GeminiKeyUnavailableError,
|
||||
get_genai_client,
|
||||
is_quota_or_billing_error,
|
||||
)
|
||||
from translation_cache import TranslationCache
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
MODEL = "gemini-3.8-flash"
|
||||
PROJECT = "nomadkaraoke"
|
||||
LOCATION = "global"
|
||||
|
||||
MAX_CONCURRENT = 5
|
||||
MAX_RETRIES = 3
|
||||
@@ -325,7 +331,8 @@ async def _call_with_retry(client: genai.Client, prompt: str) -> str:
|
||||
)
|
||||
return response.text
|
||||
except Exception as e:
|
||||
if attempt == MAX_RETRIES - 1:
|
||||
# Quota/credit/key errors won't fix themselves — fail fast.
|
||||
if attempt == MAX_RETRIES - 1 or is_quota_or_billing_error(e):
|
||||
raise
|
||||
wait = 2 ** (attempt + 1)
|
||||
print(f" Retry {attempt + 1}/{MAX_RETRIES} after error: {e}")
|
||||
@@ -576,12 +583,14 @@ async def async_main(args):
|
||||
elif not args.full:
|
||||
print("No snapshot found — will use missing-keys mode (use --full to force full retranslation)")
|
||||
|
||||
# Initialize Vertex AI client
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=PROJECT,
|
||||
location=LOCATION,
|
||||
)
|
||||
# Gemini Developer API client (API key; not needed for --dry-run)
|
||||
client = None
|
||||
if not args.dry_run:
|
||||
try:
|
||||
client = get_genai_client()
|
||||
except GeminiKeyUnavailableError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
cache = TranslationCache(
|
||||
bucket_name=args.cache_bucket,
|
||||
@@ -589,7 +598,7 @@ async def async_main(args):
|
||||
)
|
||||
|
||||
print(f"Using model: {MODEL}")
|
||||
print(f"Project: {PROJECT}, Location: {LOCATION}")
|
||||
print("Backend: Gemini Developer API (gemini-api-key)")
|
||||
print(f"Locales: {', '.join(locales)} ({len(locales)} total)")
|
||||
if args.full:
|
||||
print("Mode: full (forced)")
|
||||
@@ -655,10 +664,14 @@ async def async_main(args):
|
||||
else:
|
||||
print()
|
||||
|
||||
if any(isinstance(r, BaseException) and is_quota_or_billing_error(r) for r in results):
|
||||
print(f"\nError: {quota_exhausted_message()}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Translate i18n message files using Gemini via Vertex AI"
|
||||
description="Translate i18n message files using the Gemini Developer API"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--messages-dir",
|
||||
|
||||
@@ -153,6 +153,17 @@ def create_lyrics_transcription_job(
|
||||
name="GCS_BUCKET_NAME",
|
||||
value=bucket.name,
|
||||
),
|
||||
# Gemini Developer API key (AI Studio; Vertex AI is
|
||||
# disabled in this project). Rotate = new secret version.
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="GEMINI_API_KEY",
|
||||
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
||||
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
||||
secret=f"projects/{PROJECT_ID}/secrets/gemini-api-key",
|
||||
version="latest",
|
||||
),
|
||||
),
|
||||
),
|
||||
# Genius API token (for lyrics fetching)
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="GENIUS_API_TOKEN",
|
||||
@@ -588,6 +599,17 @@ def create_video_encoding_job(
|
||||
name="DEFAULT_BRAND_PREFIX",
|
||||
value="NOMAD",
|
||||
),
|
||||
# Gemini Developer API key (AI Studio; Vertex AI is
|
||||
# disabled in this project). Rotate = new secret version.
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="GEMINI_API_KEY",
|
||||
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
||||
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
||||
secret=f"projects/{PROJECT_ID}/secrets/gemini-api-key",
|
||||
version="latest",
|
||||
),
|
||||
),
|
||||
),
|
||||
# Discord webhook for release notifications
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="DEFAULT_DISCORD_WEBHOOK_URL",
|
||||
|
||||
@@ -91,6 +91,17 @@ def create_error_monitor(
|
||||
name="ENVIRONMENT",
|
||||
value="production",
|
||||
),
|
||||
# Gemini Developer API key (AI Studio; Vertex AI is
|
||||
# disabled in this project). Rotate = new secret version.
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="GEMINI_API_KEY",
|
||||
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
||||
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
||||
secret=f"projects/{PROJECT_ID}/secrets/gemini-api-key",
|
||||
version="latest",
|
||||
),
|
||||
),
|
||||
),
|
||||
# Discord webhook for alert notifications
|
||||
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
||||
name="DISCORD_WEBHOOK_URL",
|
||||
|
||||
@@ -80,13 +80,9 @@ def grant_backend_permissions(
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Vertex AI User permissions (for Gemini agentic AI correction)
|
||||
bindings["vertex_ai_iam"] = gcp.projects.IAMMember(
|
||||
"karaoke-backend-vertexai-user",
|
||||
project=PROJECT_ID,
|
||||
role="roles/aiplatform.user",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
# No Vertex AI role: Gemini is called via the Gemini Developer API with the
|
||||
# `gemini-api-key` secret (read via the project-level secretAccessor role
|
||||
# above), and aiplatform.googleapis.com is disabled in this project.
|
||||
|
||||
# Grant backend service account permission to enqueue Cloud Tasks
|
||||
bindings["cloud_tasks_enqueuer"] = gcp.projects.IAMMember(
|
||||
|
||||
@@ -61,7 +61,7 @@ class ProviderConfig:
|
||||
return ProviderConfig(
|
||||
openai_api_key=os.getenv("OPENAI_API_KEY"),
|
||||
anthropic_api_key=os.getenv("ANTHROPIC_API_KEY"),
|
||||
google_api_key=os.getenv("GOOGLE_API_KEY"),
|
||||
google_api_key=os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY"),
|
||||
openrouter_api_key=os.getenv("OPENROUTER_API_KEY"),
|
||||
privacy_mode=os.getenv("PRIVACY_MODE", "false").lower() in {"1", "true", "yes"},
|
||||
cache_dir=cache_dir,
|
||||
|
||||
@@ -20,8 +20,9 @@ except ImportError:
|
||||
|
||||
# Error message constant for TRY003 compliance
|
||||
GOOGLE_API_KEY_MISSING_ERROR = (
|
||||
"GOOGLE_API_KEY environment variable is required for Google/Gemini models. "
|
||||
"Get an API key from https://aistudio.google.com/app/apikey"
|
||||
"GEMINI_API_KEY (or GOOGLE_API_KEY) environment variable is required for Gemini "
|
||||
"models (Gemini Developer API; on Cloud Run it is injected from Secret Manager "
|
||||
"`gemini-api-key`). Vertex AI is not used."
|
||||
)
|
||||
|
||||
|
||||
@@ -191,8 +192,10 @@ class ModelFactory:
|
||||
return self._create_openai_model(model_name, callbacks, config)
|
||||
elif provider == "anthropic":
|
||||
return self._create_anthropic_model(model_name, callbacks, config)
|
||||
elif provider in ("vertexai", "google"):
|
||||
return self._create_vertexai_model(model_name, callbacks, config)
|
||||
elif provider in ("gemini", "google", "vertexai"):
|
||||
# "vertexai/..." is accepted for backward compatibility but is
|
||||
# served by the Gemini Developer API (Vertex is disabled).
|
||||
return self._create_gemini_model(model_name, callbacks, config)
|
||||
else:
|
||||
raise ValueError(f"Unsupported provider: {provider}")
|
||||
except ImportError as e:
|
||||
@@ -245,57 +248,39 @@ class ModelFactory:
|
||||
logger.debug(f"🤖 Created Anthropic model: {model_name}")
|
||||
return model
|
||||
|
||||
def _create_vertexai_model(
|
||||
def _create_gemini_model(
|
||||
self, model_name: str, callbacks: List[Any], config: ProviderConfig
|
||||
) -> Any:
|
||||
"""Create ChatGoogleGenerativeAI model for Google Gemini.
|
||||
|
||||
Uses the unified langchain-google-genai package which supports both:
|
||||
- Vertex AI backend (service account / ADC auth) - when project is set
|
||||
- Google AI Studio backend (API key auth) - when only api_key is set
|
||||
Always uses the Gemini Developer API (API key auth) — never Vertex AI, so
|
||||
no project/location is passed. The key comes from GEMINI_API_KEY (or
|
||||
GOOGLE_API_KEY); on Cloud Run it's injected from Secret Manager
|
||||
``gemini-api-key``.
|
||||
|
||||
On Cloud Run, ADC (Application Default Credentials) are used automatically
|
||||
when the project parameter is provided, using the service account attached
|
||||
to the Cloud Run service.
|
||||
|
||||
This is a REST-based API that avoids the gRPC connection issues
|
||||
langchain-google-genai is REST-based, avoiding the gRPC connection issues
|
||||
seen with the deprecated langchain-google-vertexai package.
|
||||
"""
|
||||
from langchain_google_genai import ChatGoogleGenerativeAI
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Determine authentication method
|
||||
api_key = config.google_api_key
|
||||
project = config.gcp_project_id
|
||||
|
||||
# Prefer Vertex AI (service account) if project is set, otherwise require API key
|
||||
if not project and not api_key:
|
||||
if not api_key:
|
||||
raise ValueError(GOOGLE_API_KEY_MISSING_ERROR)
|
||||
|
||||
if project:
|
||||
logger.info(f"🤖 Creating Google Gemini model via Vertex AI (project={project}): {model_name}")
|
||||
else:
|
||||
logger.info(f"🤖 Creating Google Gemini model via AI Studio API: {model_name}")
|
||||
logger.info(f"🤖 Creating Google Gemini model via Gemini Developer API: {model_name}")
|
||||
|
||||
# Build kwargs - only include api_key if set (otherwise ADC is used)
|
||||
model_kwargs = {
|
||||
"model": model_name,
|
||||
"convert_system_message_to_human": True, # Gemini doesn't support system messages
|
||||
"max_retries": config.max_retries,
|
||||
"timeout": config.request_timeout_seconds,
|
||||
"callbacks": callbacks,
|
||||
}
|
||||
|
||||
# Add project to trigger Vertex AI backend with ADC
|
||||
if project:
|
||||
model_kwargs["project"] = project
|
||||
|
||||
# Add API key if available (can be used with or without project)
|
||||
if api_key:
|
||||
model_kwargs["google_api_key"] = api_key
|
||||
|
||||
model = ChatGoogleGenerativeAI(**model_kwargs)
|
||||
model = ChatGoogleGenerativeAI(
|
||||
model=model_name,
|
||||
google_api_key=api_key,
|
||||
# Pin the Developer API even if GOOGLE_GENAI_USE_VERTEXAI is set.
|
||||
vertexai=False,
|
||||
convert_system_message_to_human=True, # Gemini doesn't support system messages
|
||||
max_retries=config.max_retries,
|
||||
timeout=config.request_timeout_seconds,
|
||||
callbacks=callbacks,
|
||||
)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"🤖 Google Gemini model created in {elapsed:.2f}s: {model_name}")
|
||||
|
||||
@@ -5,9 +5,9 @@ from typing import Dict, Any
|
||||
|
||||
from .providers.config import ProviderConfig
|
||||
|
||||
# Default model for cloud deployments - Gemini 3 Flash via Vertex AI
|
||||
# Note: Gemini 3 models require 'global' location (not regional like us-central1)
|
||||
DEFAULT_CLOUD_MODEL = "vertexai/gemini-3.8-flash"
|
||||
# Default model for cloud deployments - Gemini 3.8 Flash via the Gemini
|
||||
# Developer API (API key from GEMINI_API_KEY; Vertex AI is not used).
|
||||
DEFAULT_CLOUD_MODEL = "gemini/gemini-3.8-flash"
|
||||
|
||||
|
||||
class ModelRouter:
|
||||
@@ -20,7 +20,7 @@ class ModelRouter:
|
||||
"""Choose appropriate model based on gap characteristics.
|
||||
|
||||
Returns model identifier in format "provider/model" for LangChain:
|
||||
- "vertexai/gemini-3.8-flash" for Gemini via Vertex AI (default)
|
||||
- "gemini/gemini-3.8-flash" for Gemini via the Developer API (default)
|
||||
- "ollama/llama3.2:latest" for local Ollama models
|
||||
- "openai/gpt-4" for OpenAI models
|
||||
- "anthropic/claude-3-sonnet-20240229" for Anthropic models
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.261.4"
|
||||
version = "0.262.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -23,7 +23,9 @@ Usage:
|
||||
|
||||
Requires:
|
||||
- ANTHROPIC_API_KEY in the environment (for claude-* models)
|
||||
- GCP creds for Vertex/Firestore/GCS:
|
||||
- Gemini Developer API key for gemini-* models: GEMINI_API_KEY, else read
|
||||
from Secret Manager `gemini-api-key` via your gcloud login (in-process only)
|
||||
- GCP creds for Firestore/GCS:
|
||||
gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS)
|
||||
- GOOGLE_CLOUD_PROJECT=nomadkaraoke (set automatically below if unset)
|
||||
|
||||
@@ -58,6 +60,17 @@ GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
|
||||
DEFAULT_MODELS = ["claude-opus-5-5", "gemini-3.8-flash"]
|
||||
|
||||
|
||||
def _ensure_gemini_key() -> None:
|
||||
"""Populate GEMINI_API_KEY (this process only) from Secret Manager via gcloud."""
|
||||
if os.environ.get("GEMINI_API_KEY"):
|
||||
return
|
||||
repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, os.path.join(repo_root, "frontend", "scripts"))
|
||||
from gemini_client import get_api_key # noqa: E402 - scripts helper (gcloud fallback)
|
||||
|
||||
os.environ["GEMINI_API_KEY"] = get_api_key()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job + corrections loading
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -284,6 +297,8 @@ def main() -> int:
|
||||
parser.add_argument("--models", nargs="+", default=DEFAULT_MODELS,
|
||||
help=f"Models to measure (default: {' '.join(DEFAULT_MODELS)})")
|
||||
args = parser.parse_args()
|
||||
if any(m.startswith("gemini") for m in args.models):
|
||||
_ensure_gemini_key()
|
||||
|
||||
if any(m.startswith("claude") for m in args.models) and not os.environ.get("ANTHROPIC_API_KEY"):
|
||||
logger.error("ANTHROPIC_API_KEY is not set — required for claude-* models")
|
||||
|
||||
@@ -6,7 +6,8 @@ This script allows testing the agentic correction locally without deploying
|
||||
to Cloud Run or running a full karaoke job.
|
||||
|
||||
Usage:
|
||||
# Test with default Vertex AI (requires gcloud auth):
|
||||
# Test with the default Gemini model (Developer API; GEMINI_API_KEY, else
|
||||
# read from Secret Manager `gemini-api-key` via your gcloud login):
|
||||
python scripts/test_agentic_correction.py
|
||||
|
||||
# Test with a specific model:
|
||||
@@ -17,8 +18,7 @@ Usage:
|
||||
PRIVACY_MODE=1 python scripts/test_agentic_correction.py
|
||||
|
||||
Environment variables:
|
||||
GOOGLE_CLOUD_PROJECT: GCP project ID (required for Vertex AI)
|
||||
GCP_LOCATION: Vertex AI location (default: global)
|
||||
GEMINI_API_KEY: Gemini Developer API key (else fetched via gcloud)
|
||||
AGENTIC_AI_MODEL: Override the model (e.g., openai/gpt-4o)
|
||||
PRIVACY_MODE: Set to 1 to use local Ollama
|
||||
LANGFUSE_PUBLIC_KEY: Langfuse public key for tracing
|
||||
@@ -209,12 +209,15 @@ def main():
|
||||
model = router.choose_model("unknown", 0.5)
|
||||
logger.info(f"Selected model: {model}")
|
||||
|
||||
# Validate environment for Vertex AI
|
||||
if model.startswith("vertexai/") and not project_id:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT environment variable is required for Vertex AI")
|
||||
logger.error("Set it with: export GOOGLE_CLOUD_PROJECT=your-project-id")
|
||||
logger.error("Or use a different model: AGENTIC_AI_MODEL=openai/gpt-4o")
|
||||
sys.exit(1)
|
||||
# Gemini Developer API key (this process only; never written to disk)
|
||||
if model.split("/", 1)[0] in ("gemini", "google", "vertexai") and not (
|
||||
os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
|
||||
):
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend", "scripts"))
|
||||
from gemini_client import get_api_key
|
||||
|
||||
os.environ["GEMINI_API_KEY"] = get_api_key()
|
||||
config = ProviderConfig.from_env()
|
||||
|
||||
# Create the agent
|
||||
logger.info(f"\nCreating AgenticCorrector with model: {model}")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import os
|
||||
import pytest
|
||||
|
||||
# Gemini Developer API key: never let a test fall through to Secret Manager.
|
||||
os.environ.setdefault("GEMINI_API_KEY", "test-gemini-key")
|
||||
import tempfile
|
||||
import logging
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Agentic correction's Gemini provider uses the Gemini Developer API key, never Vertex."""
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from karaoke_gen.lyrics_transcriber.correction.agentic.providers.config import ProviderConfig
|
||||
from karaoke_gen.lyrics_transcriber.correction.agentic.providers.model_factory import ModelFactory
|
||||
from karaoke_gen.lyrics_transcriber.correction.agentic.router import DEFAULT_CLOUD_MODEL
|
||||
|
||||
|
||||
def _config(monkeypatch, **env):
|
||||
for k in ("GEMINI_API_KEY", "GOOGLE_API_KEY"):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
for k, v in env.items():
|
||||
monkeypatch.setenv(k, v)
|
||||
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "nomadkaraoke") # must NOT trigger Vertex
|
||||
return ProviderConfig.from_env()
|
||||
|
||||
|
||||
def test_default_model_is_developer_api():
|
||||
assert DEFAULT_CLOUD_MODEL == "gemini/gemini-3.8-flash"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec", ["gemini/gemini-3.8-flash", "vertexai/gemini-3.8-flash"])
|
||||
def test_gemini_model_uses_api_key_not_project(monkeypatch, spec):
|
||||
config = _config(monkeypatch, GEMINI_API_KEY="dev-key")
|
||||
with patch("langchain_google_genai.ChatGoogleGenerativeAI") as chat:
|
||||
factory = ModelFactory()
|
||||
provider, model_name = factory._parse_model_spec(spec)
|
||||
factory._instantiate_model(provider, model_name, [], config)
|
||||
kwargs = chat.call_args.kwargs
|
||||
assert kwargs["google_api_key"] == "dev-key"
|
||||
assert kwargs["model"] == "gemini-3.8-flash"
|
||||
assert kwargs["vertexai"] is False
|
||||
assert "project" not in kwargs and "location" not in kwargs
|
||||
|
||||
|
||||
def test_google_api_key_still_accepted(monkeypatch):
|
||||
config = _config(monkeypatch, GOOGLE_API_KEY="legacy-key")
|
||||
assert config.google_api_key == "legacy-key"
|
||||
|
||||
|
||||
def test_missing_key_raises_clear_error(monkeypatch):
|
||||
config = _config(monkeypatch)
|
||||
with pytest.raises(ValueError, match="GEMINI_API_KEY"):
|
||||
ModelFactory()._create_gemini_model("gemini-3.8-flash", [], config)
|
||||
@@ -181,11 +181,6 @@ class TestDefaultValues:
|
||||
assert isinstance(config.LLM_ANALYSIS_MODEL, str)
|
||||
assert len(config.LLM_ANALYSIS_MODEL) > 0
|
||||
|
||||
def test_llm_vertex_location_is_string(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert isinstance(config.LLM_VERTEX_LOCATION, str)
|
||||
assert len(config.LLM_VERTEX_LOCATION) > 0
|
||||
|
||||
|
||||
class TestEnvVarOverrides:
|
||||
|
||||
Reference in new issue
Block a user