diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 862bb1e5..46f0f29c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2176,7 +2176,7 @@ jobs: --min-instances 0 \ --clear-vpc-connector \ --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,USE_CLOUD_RUN_JOBS_FOR_RENDER=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-5-5,AUTO_CORRECT_COMPARE_MODELS=claude-opus-5-5;gemini-3.8-flash,AUTO_CORRECT_PROACTIVE_ENABLED=true,AUTO_APPROVAL_BACKING_KEEP_ENABLED=true,COMMUNITY_DAILY_PICK_ENABLED=true,STORAGE_RETENTION_DRY_RUN=false,STORAGE_RETENTION_MAX_JOBS_PER_RUN=50,CONVERSION_EMAILS_ENABLED=true,REVIEW_LAST_CHANCE_ENABLED=true,CONVERSION_OFFER_PERCENT=25,EDGE_AUTH_MODE=enforce,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \ - --set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest" + --set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,LRCLIB_APP_ID=lrclib-app-id:latest,LRCLIB_SECRET_KEY=lrclib-api-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest" # R3/R4 — smoke the tagged candidate revision BEFORE it serves any traffic. - name: Smoke candidate revision diff --git a/backend/services/quick_version/renderer.py b/backend/services/quick_version/renderer.py index 4d168ac2..5387e4ce 100644 --- a/backend/services/quick_version/renderer.py +++ b/backend/services/quick_version/renderer.py @@ -31,6 +31,8 @@ from typing import List, Optional, Tuple import requests +from karaoke_gen.lyrics_transcriber.lyrics import lrclib_client + logger = logging.getLogger(__name__) LRCLIB_BASE = "https://lrclib.net/api" @@ -124,7 +126,7 @@ def fetch_lrclib_lyrics( if duration: params["duration"] = int(round(duration)) try: - r = session.get(f"{LRCLIB_BASE}/get", params=params, timeout=timeout) + r = lrclib_client.get(f"{LRCLIB_BASE}/get", params, user_agent=USER_AGENT, timeout=timeout, session=session, retry=False) if r.status_code == 200: got = lyrics_from_lrclib_payload(r.json()) if got and got.kind == "synced": @@ -137,7 +139,7 @@ def fetch_lrclib_lyrics( exact_plain = None try: - r = session.get(f"{LRCLIB_BASE}/search", params={"q": f"{artist} {title}"}, timeout=timeout) + r = lrclib_client.get(f"{LRCLIB_BASE}/search", {"q": f"{artist} {title}"}, user_agent=USER_AGENT, timeout=timeout, session=session, retry=False) if r.status_code == 200: hits = r.json() or [] for hit in sorted(hits, key=lambda h: 0 if h.get("syncedLyrics") else 1): diff --git a/infrastructure/modules/cloud_run.py b/infrastructure/modules/cloud_run.py index b4bf548d..a405a348 100644 --- a/infrastructure/modules/cloud_run.py +++ b/infrastructure/modules/cloud_run.py @@ -178,6 +178,25 @@ def create_lyrics_transcription_job( name="GOOGLE_CLOUD_PROJECT", value=PROJECT_ID, ), + # LRCLIB registered-app credentials (lrclib_client.py) + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="LRCLIB_APP_ID", + value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs( + secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs( + secret=f"projects/{PROJECT_ID}/secrets/lrclib-app-id", + version="latest", + ), + ), + ), + cloudrunv2.JobTemplateTemplateContainerEnvArgs( + name="LRCLIB_SECRET_KEY", + value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs( + secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs( + secret=f"projects/{PROJECT_ID}/secrets/lrclib-api-key", + version="latest", + ), + ), + ), # RapidAPI key (for Musixmatch lyrics) cloudrunv2.JobTemplateTemplateContainerEnvArgs( name="RAPIDAPI_KEY", diff --git a/infrastructure/modules/secrets.py b/infrastructure/modules/secrets.py index a08a9ae6..5a6b7531 100644 --- a/infrastructure/modules/secrets.py +++ b/infrastructure/modules/secrets.py @@ -21,6 +21,9 @@ def create_secrets() -> dict[str, secretmanager.Secret]: # Audio processing "audioshake-api-key", "genius-api-key", + # LRCLIB registered server-side app (higher rate limits; lrclib.net developer portal) + "lrclib-app-id", + "lrclib-api-key", "audio-separator-api-url", # Payment and email diff --git a/karaoke_gen/lyrics_transcriber/lyrics/lrclib.py b/karaoke_gen/lyrics_transcriber/lyrics/lrclib.py index 16192ed1..cad38a65 100644 --- a/karaoke_gen/lyrics_transcriber/lyrics/lrclib.py +++ b/karaoke_gen/lyrics_transcriber/lyrics/lrclib.py @@ -5,6 +5,7 @@ import requests from karaoke_gen.lyrics_transcriber.types import LyricsData, LyricsMetadata, LyricsSegment, Word from karaoke_gen.lyrics_transcriber.lyrics.base_lyrics_provider import BaseLyricsProvider, LyricsProviderConfig from karaoke_gen.lyrics_transcriber.utils.word_utils import WordUtils +from karaoke_gen.lyrics_transcriber.lyrics import lrclib_client class LRCLIBProvider(BaseLyricsProvider): @@ -66,12 +67,8 @@ class LRCLIBProvider(BaseLyricsProvider): "duration": duration } - headers = { - "User-Agent": self.USER_AGENT - } - self.logger.debug(f"Making LRCLIB request with duration {duration}s") - response = requests.get(url, headers=headers, params=params, timeout=15) + response = lrclib_client.get(url, params, user_agent=self.USER_AGENT, logger=self.logger) if response.status_code == 404: self.logger.debug("Track not found with exact duration") @@ -99,12 +96,8 @@ class LRCLIBProvider(BaseLyricsProvider): "artist_name": artist } - headers = { - "User-Agent": self.USER_AGENT - } - - self.logger.debug(f"Making LRCLIB search request") - response = requests.get(url, headers=headers, params=params, timeout=15) + self.logger.debug("Making LRCLIB search request") + response = lrclib_client.get(url, params, user_agent=self.USER_AGENT, logger=self.logger) response.raise_for_status() results = response.json() diff --git a/karaoke_gen/lyrics_transcriber/lyrics/lrclib_client.py b/karaoke_gen/lyrics_transcriber/lyrics/lrclib_client.py new file mode 100644 index 00000000..bfa685a0 --- /dev/null +++ b/karaoke_gen/lyrics_transcriber/lyrics/lrclib_client.py @@ -0,0 +1,112 @@ +"""HTTP access to LRCLIB (https://lrclib.net/docs#implementation-requirements-rate-limiting). + +Unregistered clients get ~45-90 requests/min per IP, and Cloud Run shares egress IPs, so +since Oct 2026 our anonymous requests were intermittently rejected (503s). We are a +registered server-side application: send ``Lrclib-App-Id`` + the secret ``Lrclib-Api-Key`` +(env ``LRCLIB_APP_ID`` / ``LRCLIB_SECRET_KEY``) for the app-wide allowance. + +LRCLIB requires clients to honour ``Retry-After`` on 429 (plus up to 30% jitter) or risk a +temporary ban. ``get`` records a process-wide cooldown from every 429 and sends nothing +until it has passed (callers get the 429 back instead); it waits and retries once only +when the wait fits ``MAX_WAIT_S``. It also retries 5xx with a short backoff, and drops the +credentials for the rest of the process if they are rejected (401) so a revoked key +degrades to anonymous access instead of losing LRCLIB entirely. +""" +import logging +import os +import random +import threading +import time +from typing import Any, Dict, Optional + +import requests + +DEFAULT_USER_AGENT = "karaoke-gen (https://github.com/nomadkaraoke/karaoke-gen)" +MAX_WAIT_S = 30.0 # longest we block one call for a Retry-After; longer cooldowns skip LRCLIB +SERVER_ERROR_BACKOFF_S = (1.0, 2.0) + +_credentials_rejected = False +_cooldown_until = 0.0 # time.monotonic() before which we must not call LRCLIB (Retry-After) +_cooldown_response: Optional[requests.Response] = None +_cooldown_lock = threading.Lock() + + +def auth_headers() -> Dict[str, str]: + """Registered-app headers, or {} when not configured or rejected earlier.""" + if _credentials_rejected: + return {} + headers = {} + app_id = os.environ.get("LRCLIB_APP_ID") + api_key = os.environ.get("LRCLIB_SECRET_KEY") + if app_id: + headers["Lrclib-App-Id"] = app_id + if api_key: + headers["Lrclib-Api-Key"] = api_key + return headers + + +def _retry_after_seconds(response: requests.Response) -> float: + """Retry-After plus up to 30% jitter (never shortened).""" + try: + base = float(response.headers.get("Retry-After", "1")) + except ValueError: + base = 1.0 + return max(base, 0.0) * (1 + random.uniform(0, 0.3)) + + +def get( + url: str, + params: Dict[str, Any], + user_agent: str = DEFAULT_USER_AGENT, + timeout: float = 15, + logger: Optional[logging.Logger] = None, + session: Optional[requests.Session] = None, + retry: bool = True, +) -> requests.Response: + """GET an LRCLIB endpoint with auth, Retry-After cooldown and (if ``retry``) 5xx retries. + + Returns the final response (callers check the status); raises only on transport errors. + While a Retry-After cooldown is active, returns the 429 that started it without sending. + """ + global _credentials_rejected, _cooldown_until, _cooldown_response + log = logger or logging.getLogger(__name__) + http = session or requests + server_error_retries = list(SERVER_ERROR_BACKOFF_S) if retry else [] + rate_limit_retried = not retry + while True: + wait = _cooldown_until - time.monotonic() + if wait > 0: + if rate_limit_retried or wait > MAX_WAIT_S: + log.info(f"LRCLIB cooldown active ({wait:.0f}s left after a 429); skipping request") + return _cooldown_response + rate_limit_retried = True + time.sleep(wait) + sent_auth = auth_headers() + response = http.get(url, params=params, timeout=timeout, + headers={"User-Agent": user_agent, **sent_auth}) + status = response.status_code + if status == 401 and sent_auth: + _credentials_rejected = True + log.warning("LRCLIB rejected our app credentials (401); continuing anonymously. " + "Check LRCLIB_APP_ID / LRCLIB_SECRET_KEY in the LRCLIB developer portal.") + continue + if status == 429: + delay = _retry_after_seconds(response) + with _cooldown_lock: # overlapping 429s: keep the latest expiry, never shorten it + if time.monotonic() + delay > _cooldown_until: + _cooldown_until = time.monotonic() + delay + _cooldown_response = response + log.info(f"LRCLIB rate limited (429); cooling down {delay:.1f}s") + if rate_limit_retried or delay > MAX_WAIT_S: + return response + continue # the cooldown check at the top of the loop waits, then retries once + if status >= 500 and server_error_retries: + delay = server_error_retries.pop(0) + if "Retry-After" in response.headers: # an overloaded 503 may say how long to wait + delay = max(delay, _retry_after_seconds(response)) + if delay > MAX_WAIT_S: + return response + log.info(f"LRCLIB returned {status}; retrying in {delay:.0f}s") + time.sleep(delay) + continue + return response diff --git a/pyproject.toml b/pyproject.toml index 71b59197..2403b31a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.279.1" +version = "0.279.2" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/test_lrclib_client.py b/tests/unit/test_lrclib_client.py new file mode 100644 index 00000000..206025b5 --- /dev/null +++ b/tests/unit/test_lrclib_client.py @@ -0,0 +1,139 @@ +"""lrclib_client: registered-app auth headers + LRCLIB's rate-limit rules.""" +import logging +from unittest.mock import MagicMock, patch + +import pytest + +from karaoke_gen.lyrics_transcriber.lyrics import lrclib_client + + +@pytest.fixture(autouse=True) +def _reset(monkeypatch): + monkeypatch.setattr(lrclib_client, "_credentials_rejected", False) + monkeypatch.setattr(lrclib_client, "_cooldown_until", 0.0) + monkeypatch.setattr(lrclib_client, "_cooldown_response", None) + monkeypatch.setenv("LRCLIB_APP_ID", "app-1") + monkeypatch.setenv("LRCLIB_SECRET_KEY", "secret-1") + sleeps = [] + monkeypatch.setattr(lrclib_client.time, "sleep", sleeps.append) + return sleeps + + +def _resp(status, headers=None): + r = MagicMock() + r.status_code = status + r.headers = headers or {} + return r + + +def _get(*responses): + session = MagicMock() + session.get.side_effect = list(responses) + response = lrclib_client.get("https://lrclib.net/api/get", {"q": "x"}, user_agent="ua", + logger=logging.getLogger("t"), session=session) + return response, session + + +def test_sends_app_id_and_secret_key_with_user_agent(): + response, session = _get(_resp(200)) + assert response.status_code == 200 + headers = session.get.call_args.kwargs["headers"] + assert headers == {"User-Agent": "ua", "Lrclib-App-Id": "app-1", "Lrclib-Api-Key": "secret-1"} + + +def test_anonymous_when_not_configured(monkeypatch): + monkeypatch.delenv("LRCLIB_APP_ID") + monkeypatch.delenv("LRCLIB_SECRET_KEY") + _, session = _get(_resp(200)) + assert session.get.call_args.kwargs["headers"] == {"User-Agent": "ua"} + + +def test_honours_retry_after_with_jitter_on_429(_reset): + response, session = _get(_resp(429, {"Retry-After": "4"}), _resp(200)) + assert response.status_code == 200 and session.get.call_count == 2 + assert 4.0 <= _reset[0] <= 4.0 * 1.3 + + +def test_long_retry_after_is_not_shortened_and_blocks_further_calls(_reset): + """Never retry before Retry-After: a 600s wait returns the 429 and the cooldown holds.""" + response, session = _get(_resp(429, {"Retry-After": "600"})) + assert response.status_code == 429 and session.get.call_count == 1 and _reset == [] + # e.g. the provider's /search fallback right after a rate-limited /get: nothing is sent + later, later_session = _get(_resp(200)) + assert later.status_code == 429 and later_session.get.call_count == 0 + + +def test_no_retry_mode_respects_cooldown_without_waiting(_reset): + session = MagicMock() + session.get.side_effect = [_resp(429, {"Retry-After": "5"})] + r = lrclib_client.get("https://lrclib.net/api/get", {}, session=session, retry=False) + assert r.status_code == 429 and _reset == [] + r2 = lrclib_client.get("https://lrclib.net/api/search", {}, session=session, retry=False) + assert r2.status_code == 429 and session.get.call_count == 1 + + +def test_second_429_is_returned_not_retried_forever(): + response, session = _get(_resp(429, {"Retry-After": "1"}), _resp(429, {"Retry-After": "1"})) + assert response.status_code == 429 and session.get.call_count == 2 + + +def test_server_errors_retry_twice_with_backoff(_reset): + response, session = _get(_resp(503), _resp(503), _resp(200)) + assert response.status_code == 200 and session.get.call_count == 3 + assert _reset == list(lrclib_client.SERVER_ERROR_BACKOFF_S) + + +def test_persistent_server_error_is_returned(): + response, session = _get(_resp(503), _resp(502), _resp(503)) + assert response.status_code == 503 and session.get.call_count == 3 + + +def test_rejected_credentials_fall_back_to_anonymous_for_the_process(caplog): + with caplog.at_level(logging.WARNING, logger="t"): + response, session = _get(_resp(401), _resp(200)) + assert response.status_code == 200 + assert "Lrclib-Api-Key" not in session.get.call_args.kwargs["headers"] + assert lrclib_client.auth_headers() == {} + assert "rejected our app credentials" in caplog.text + + +def test_anonymous_401_is_returned_not_looped(monkeypatch): + monkeypatch.delenv("LRCLIB_APP_ID") + monkeypatch.delenv("LRCLIB_SECRET_KEY") + response, session = _get(_resp(401)) + assert response.status_code == 401 and session.get.call_count == 1 + + +def test_provider_requests_go_through_the_client(): + from karaoke_gen.lyrics_transcriber.lyrics.base_lyrics_provider import LyricsProviderConfig + from karaoke_gen.lyrics_transcriber.lyrics.lrclib import LRCLIBProvider + + hit = _resp(200) + hit.json.return_value = [{"trackName": "Iowa", "artistName": "Dar Williams", "plainLyrics": "x"}] + with patch.object(lrclib_client, "requests") as req: + req.get.return_value = hit + result = LRCLIBProvider(LyricsProviderConfig(), logger=logging.getLogger("t"))._fetch_from_search("Dar Williams", "Iowa") + assert result["trackName"] == "Iowa" + assert req.get.call_args.kwargs["headers"]["Lrclib-Api-Key"] == "secret-1" + + +def test_shorter_overlapping_429_does_not_shorten_the_cooldown(): + def in_flight(*a, **k): + # a concurrent request got Retry-After: 600 while this one was in flight + lrclib_client._cooldown_until = lrclib_client.time.monotonic() + 600 + return _resp(429, {"Retry-After": "1"}) + + session = MagicMock() + session.get.side_effect = in_flight + lrclib_client.get("https://lrclib.net/api/get", {}, session=session, retry=False) + assert lrclib_client._cooldown_until - lrclib_client.time.monotonic() > 590 + + +def test_503_retry_after_is_honoured(_reset): + response, session = _get(_resp(503, {"Retry-After": "7"}), _resp(200)) + assert response.status_code == 200 and 7.0 <= _reset[0] <= 7.0 * 1.3 + + +def test_503_with_long_retry_after_is_returned_without_retry(): + response, session = _get(_resp(503, {"Retry-After": "120"})) + assert response.status_code == 503 and session.get.call_count == 1