mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(youtube): TornadoAPI first for YouTube downloads, flacfetch as fallback (v0.291.0)
YouTube downloads depended on one logged-in cookie session on one VM. Tornado depends on neither, and returned the audio that failed on 11-03 in about 5s. Audio is packet-identical to flacfetch's Opus. Off until the tornado-api-key secret has a version; CI then wires it on the next deploy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
This commit is contained in:
1 parent
3a563ab3e6
commit
95fd867e9f
8 files changed
+861
-22
No files matched your search
@@ -2197,6 +2197,14 @@ jobs:
|
||||
# --cpu-throttling (request-based billing) a warm-but-idle non-min
|
||||
# instance costs nothing. Concurrency stays far below the wedge-era 80. --clear-vpc-connector: the
|
||||
# serverless VPC connector was deleted (flacfetch is public now).
|
||||
# YouTube downloads try TornadoAPI first once this secret has a version. Until then the
|
||||
# backend uses flacfetch, and referencing a missing secret would fail the deploy.
|
||||
TORNADO_SECRET=""
|
||||
if gcloud secrets versions describe latest --secret=tornado-api-key --project ${{ env.PROJECT_ID }} >/dev/null 2>&1; then
|
||||
TORNADO_SECRET=",TORNADO_API_KEY=tornado-api-key:latest"
|
||||
fi
|
||||
echo "tornado-api-key wired: ${TORNADO_SECRET:+yes}"
|
||||
|
||||
gcloud run deploy karaoke-backend \
|
||||
--image $IMAGE \
|
||||
--no-traffic \
|
||||
@@ -2215,7 +2223,7 @@ jobs:
|
||||
--min-instances 0 \
|
||||
--clear-vpc-connector \
|
||||
--set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,USE_CLOUD_RUN_JOBS_FOR_RENDER=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-5-5,AUTO_CORRECT_COMPARE_MODELS=claude-opus-5-5;gemini-3.8-flash,AUTO_CORRECT_PROACTIVE_ENABLED=true,AUTO_APPROVAL_BACKING_KEEP_ENABLED=true,COMMUNITY_DAILY_PICK_ENABLED=true,STORAGE_RETENTION_DRY_RUN=false,STORAGE_RETENTION_MAX_JOBS_PER_RUN=50,CONVERSION_EMAILS_ENABLED=true,REVIEW_LAST_CHANCE_ENABLED=true,CONVERSION_OFFER_PERCENT=25,YOUTUBE_COMMENT_AUTOREPLY_ENABLED=true,YOUTUBE_COMMENT_AUTOREPLY_START=2026-04-22T00:00:00Z,EDGE_AUTH_MODE=enforce,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \
|
||||
--set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,LRCLIB_APP_ID=lrclib-app-id:latest,LRCLIB_SECRET_KEY=lrclib-api-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest,DECIDE_GEN_PARTNER_SECRET=decide-gen-partner-secret:latest,REVENUECAT_WEBHOOK_AUTH=revenuecat-webhook-auth:latest,REVENUECAT_SECRET_API_KEY=revenuecat-secret-api-key:latest,APP_REVIEW_DEMO_CODE=app-review-demo-code:latest"
|
||||
--set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,LRCLIB_APP_ID=lrclib-app-id:latest,LRCLIB_SECRET_KEY=lrclib-api-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest,DECIDE_GEN_PARTNER_SECRET=decide-gen-partner-secret:latest,REVENUECAT_WEBHOOK_AUTH=revenuecat-webhook-auth:latest,REVENUECAT_SECRET_API_KEY=revenuecat-secret-api-key:latest,APP_REVIEW_DEMO_CODE=app-review-demo-code:latest${TORNADO_SECRET}"
|
||||
|
||||
# R3/R4 — smoke the tagged candidate revision BEFORE it serves any traffic.
|
||||
- name: Smoke candidate revision
|
||||
|
||||
@@ -297,6 +297,11 @@ class Settings(BaseSettings):
|
||||
flacfetch_api_url: Optional[str] = os.getenv("FLACFETCH_API_URL") # e.g., http://10.0.0.5:8080
|
||||
flacfetch_api_key: Optional[str] = os.getenv("FLACFETCH_API_KEY")
|
||||
|
||||
# TornadoAPI: tried first for YouTube downloads when the key is set, with flacfetch as fallback.
|
||||
# Set YOUTUBE_DOWNLOAD_PROVIDER=flacfetch to switch it off without a code change.
|
||||
tornado_api_key: Optional[str] = os.getenv("TORNADO_API_KEY")
|
||||
youtube_download_provider: str = os.getenv("YOUTUBE_DOWNLOAD_PROVIDER", "tornado")
|
||||
|
||||
# Flacfetch client retry settings
|
||||
# Retries help handle transient network failures when calling the flacfetch service
|
||||
flacfetch_retry_max_attempts: int = int(os.getenv("FLACFETCH_RETRY_MAX_ATTEMPTS", "9")) # Total attempts (1 initial + 8 retries)
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
TornadoAPI client: downloads YouTube audio through a hosted media API.
|
||||
|
||||
Tornado doesn't depend on our cookies or IP, unlike flacfetch, so YouTubeDownloadService tries it
|
||||
first. Docs: https://docs.tornadoapi.io
|
||||
|
||||
Three things that are easy to get wrong:
|
||||
|
||||
- Only ever send a canonical ``watch?v=<id>`` URL. A URL carrying a real playlist id downloads and
|
||||
bills the whole playlist, returning ``batch_id`` rather than ``job_id``.
|
||||
- ``format: "opus"`` without ``audio_codec`` returns the source's Opus stream, packet-identical to what
|
||||
flacfetch delivers (checked 2026-11-03). Adding ``audio_codec: "copy"`` or leaving ``format`` out
|
||||
returns YouTube's 128k AAC instead.
|
||||
- Delivered files are only kept for 24 hours and count toward a storage limit, so copy them
|
||||
straight away and delete them. The delivery URL is signed: never send it the API key or log it.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BASE_URL = "https://api.tornadoapi.io"
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
_NONTERMINAL = {"Pending", "Processing"}
|
||||
_TERMINAL = {"Completed", "Failed", "Warning", "Skipped", "Cancelled", "CancelledByAdmin"}
|
||||
|
||||
_CREATE_ATTEMPTS = 3
|
||||
_MAX_RETRY_AFTER_SECONDS = 30.0
|
||||
_MAX_FILE_BYTES = 1024 * 1024 * 1024 # audio only; anything this big isn't a song
|
||||
|
||||
|
||||
class TornadoError(Exception):
|
||||
"""A download through Tornado failed. The caller can try another route."""
|
||||
|
||||
|
||||
class TornadoAccountError(TornadoError):
|
||||
"""The key or account was rejected, so every download will fail until a person fixes it."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TornadoDownload:
|
||||
tornado_job_id: str
|
||||
extension: str
|
||||
file_size: Optional[int]
|
||||
billable_bytes: Optional[int]
|
||||
|
||||
|
||||
class TornadoClient:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
*,
|
||||
base_url: str = BASE_URL,
|
||||
deadline_seconds: float = 180.0,
|
||||
poll_interval_seconds: float = 3.0,
|
||||
transport: Optional[httpx.AsyncBaseTransport] = None,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
clock: Callable[[], float] = lambda: asyncio.get_running_loop().time(),
|
||||
):
|
||||
self._api_key = api_key
|
||||
self._base_url = base_url
|
||||
self._deadline_seconds = deadline_seconds
|
||||
self._poll_interval_seconds = poll_interval_seconds
|
||||
self._transport = transport
|
||||
self._sleep = sleep
|
||||
self._clock = clock
|
||||
|
||||
def _http(self, timeout: float = 30.0) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(timeout=timeout, transport=self._transport)
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
return {"x-api-key": self._api_key, "Content-Type": "application/json"}
|
||||
|
||||
@staticmethod
|
||||
def _watch_url(video_id: str) -> str:
|
||||
if not _VIDEO_ID.match(video_id or ""):
|
||||
raise TornadoError(f"Not a YouTube video id: {video_id!r}")
|
||||
return f"https://www.youtube.com/watch?v={video_id}"
|
||||
|
||||
async def get_metadata(self, video_id: str) -> Dict[str, Any]:
|
||||
"""Video metadata, without downloading. Raises TornadoError if the lookup fails."""
|
||||
url = self._watch_url(video_id)
|
||||
try:
|
||||
async with self._http(timeout=15.0) as client:
|
||||
resp = await client.post(
|
||||
f"{self._base_url}/metadata", headers=self._headers(), json={"url": url}
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
raise TornadoError(f"Metadata request failed: {type(e).__name__}") from e
|
||||
self._raise_for_account(resp)
|
||||
if resp.status_code != 200:
|
||||
raise TornadoError(f"Metadata lookup returned {resp.status_code}")
|
||||
data = self._json_object(resp)
|
||||
if data is None:
|
||||
raise TornadoError("Metadata response was not a JSON object")
|
||||
return data
|
||||
|
||||
async def download_audio(self, video_id: str, dest_path: str, submission: str) -> TornadoDownload:
|
||||
"""Download a video's audio to ``dest_path``.
|
||||
|
||||
``submission`` identifies this logical request (our job id). Raises TornadoError on any
|
||||
failure, including a job that doesn't finish before the deadline.
|
||||
"""
|
||||
url = self._watch_url(video_id)
|
||||
# Unique per call, stable across this call's retries: a POST that timed out may have created
|
||||
# a job, and the key lets the retry get that job back instead of paying for a second one.
|
||||
idempotency_key = f"nomad-{submission}-{video_id}-{uuid.uuid4().hex}"
|
||||
job_id = await self._create_job(url, idempotency_key)
|
||||
logger.info(f"Tornado job {job_id} created for video_id={video_id}, submission={submission}")
|
||||
|
||||
job = await self._wait_for_job(job_id)
|
||||
delivery_url = job.get("s3_url")
|
||||
if not delivery_url:
|
||||
raise TornadoError(f"Tornado job {job_id} completed without a delivery URL")
|
||||
|
||||
await self._fetch_file(delivery_url, dest_path)
|
||||
await self._delete_file(job_id)
|
||||
return TornadoDownload(
|
||||
tornado_job_id=job_id,
|
||||
extension=self._extension(delivery_url),
|
||||
file_size=job.get("file_size"),
|
||||
billable_bytes=job.get("billable_bytes"),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _json_object(resp: httpx.Response) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
@staticmethod
|
||||
def _raise_for_account(resp: httpx.Response) -> None:
|
||||
if resp.status_code in (401, 402, 403):
|
||||
raise TornadoAccountError(f"Tornado rejected the account: HTTP {resp.status_code}")
|
||||
|
||||
async def _create_job(self, url: str, idempotency_key: str) -> str:
|
||||
body = {"url": url, "audio_only": True, "format": "opus"}
|
||||
headers = {**self._headers(), "x-idempotency-key": idempotency_key}
|
||||
last_problem = "no attempt made"
|
||||
retry_after: Optional[str] = None
|
||||
for attempt in range(_CREATE_ATTEMPTS):
|
||||
if attempt:
|
||||
await self._sleep(self._retry_delay(attempt, retry_after))
|
||||
retry_after = None
|
||||
try:
|
||||
async with self._http() as client:
|
||||
resp = await client.post(f"{self._base_url}/jobs", headers=headers, json=body)
|
||||
except httpx.HTTPError as e:
|
||||
last_problem = type(e).__name__
|
||||
continue
|
||||
|
||||
self._raise_for_account(resp)
|
||||
if resp.status_code == 429 or resp.status_code >= 500:
|
||||
last_problem = f"HTTP {resp.status_code}"
|
||||
retry_after = resp.headers.get("Retry-After")
|
||||
continue
|
||||
if resp.status_code >= 400:
|
||||
raise TornadoError(f"Tornado rejected the job: HTTP {resp.status_code}")
|
||||
|
||||
data = self._json_object(resp)
|
||||
if data is None:
|
||||
raise TornadoError("Job response was not a JSON object")
|
||||
if data.get("batch_id"):
|
||||
# Should be impossible with a canonical URL, and it would bill a whole playlist.
|
||||
logger.error(f"Tornado created batch {data['batch_id']} for {url}; cancel it in the dashboard")
|
||||
raise TornadoError("Tornado created a batch for a single video")
|
||||
job_id = data.get("job_id")
|
||||
if not job_id:
|
||||
raise TornadoError("Job response had no job_id")
|
||||
return job_id
|
||||
raise TornadoError(f"Could not create a Tornado job: {last_problem}")
|
||||
|
||||
@staticmethod
|
||||
def _retry_delay(attempt: int, retry_after: Optional[str]) -> float:
|
||||
try:
|
||||
if retry_after is not None:
|
||||
return min(max(float(retry_after), 0.0), _MAX_RETRY_AFTER_SECONDS)
|
||||
except ValueError:
|
||||
pass
|
||||
return float(2**attempt) + random.uniform(0, 1)
|
||||
|
||||
async def _wait_for_job(self, job_id: str) -> Dict[str, Any]:
|
||||
deadline = self._clock() + self._deadline_seconds
|
||||
last_status = "unknown"
|
||||
while self._clock() < deadline:
|
||||
status_code, job = await self._get_job(job_id)
|
||||
if job is not None:
|
||||
last_status = str(job.get("status"))
|
||||
if last_status == "Completed":
|
||||
return job
|
||||
if last_status in _TERMINAL:
|
||||
raise TornadoError(
|
||||
f"Tornado job {job_id} ended {last_status} "
|
||||
f"({job.get('error_type')}): {job.get('error')}"
|
||||
)
|
||||
if last_status not in _NONTERMINAL:
|
||||
logger.warning(f"Tornado job {job_id} has an unknown status: {last_status}")
|
||||
elif status_code is not None:
|
||||
last_status = f"HTTP {status_code}"
|
||||
await self._sleep(self._poll_interval_seconds)
|
||||
|
||||
await self._cancel(job_id)
|
||||
raise TornadoError(f"Tornado job {job_id} timed out after {self._deadline_seconds:.0f}s ({last_status})")
|
||||
|
||||
async def _get_job(self, job_id: str) -> "tuple[Optional[int], Optional[Dict[str, Any]]]":
|
||||
"""One status read. A failed read returns no job, so the loop tries again until the deadline."""
|
||||
try:
|
||||
async with self._http() as client:
|
||||
resp = await client.get(f"{self._base_url}/jobs/{job_id}", headers=self._headers())
|
||||
except httpx.HTTPError:
|
||||
return None, None
|
||||
self._raise_for_account(resp)
|
||||
if resp.status_code != 200:
|
||||
return resp.status_code, None
|
||||
return resp.status_code, self._json_object(resp)
|
||||
|
||||
async def _fetch_file(self, delivery_url: str, dest_path: str) -> None:
|
||||
"""Stream the delivered file to disk. Never sends the API key; a partial file is removed."""
|
||||
written = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=120.0, transport=self._transport, follow_redirects=True) as client:
|
||||
async with client.stream("GET", delivery_url) as resp:
|
||||
if resp.status_code != 200:
|
||||
raise TornadoError(f"Delivery URL returned {resp.status_code}")
|
||||
with open(dest_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
written += len(chunk)
|
||||
if written > _MAX_FILE_BYTES:
|
||||
raise TornadoError("Delivered file is larger than expected")
|
||||
f.write(chunk)
|
||||
if not written:
|
||||
raise TornadoError("Delivered file was empty")
|
||||
except httpx.HTTPError as e:
|
||||
self._remove_partial(dest_path)
|
||||
raise TornadoError(f"Fetching the delivered file failed: {type(e).__name__}") from e
|
||||
except OSError as e:
|
||||
self._remove_partial(dest_path)
|
||||
raise TornadoError(f"Writing the delivered file failed: {type(e).__name__}") from e
|
||||
except TornadoError:
|
||||
self._remove_partial(dest_path)
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _remove_partial(path: str) -> None:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
async def _delete_file(self, job_id: str) -> None:
|
||||
"""Best effort: it expires in 24 hours anyway."""
|
||||
try:
|
||||
async with self._http() as client:
|
||||
resp = await client.delete(f"{self._base_url}/jobs/{job_id}/file", headers=self._headers())
|
||||
if resp.status_code != 200:
|
||||
logger.info(f"Could not delete Tornado file for job {job_id}: HTTP {resp.status_code}")
|
||||
except httpx.HTTPError as e:
|
||||
logger.info(f"Could not delete Tornado file for job {job_id}: {type(e).__name__}")
|
||||
|
||||
async def _cancel(self, job_id: str) -> None:
|
||||
"""Best effort: stop paying for a job we've given up on."""
|
||||
try:
|
||||
async with self._http() as client:
|
||||
await client.post(f"{self._base_url}/jobs/{job_id}/cancel", headers=self._headers())
|
||||
except httpx.HTTPError as e:
|
||||
logger.info(f"Could not cancel Tornado job {job_id}: {type(e).__name__}")
|
||||
|
||||
@staticmethod
|
||||
def _extension(delivery_url: str) -> str:
|
||||
# We ask for Opus, which arrives in an Ogg container. Name it .ogg: that's accurate, and
|
||||
# gen's upload and review allow-lists accept .ogg but not .opus.
|
||||
ext = os.path.splitext(urlsplit(delivery_url).path)[1].lower()
|
||||
if ext == ".opus" or not re.fullmatch(r"\.[a-z0-9]{2,5}", ext):
|
||||
return ".ogg"
|
||||
return ext
|
||||
|
||||
|
||||
_client: Optional[TornadoClient] = None
|
||||
|
||||
|
||||
def get_tornado_client() -> Optional[TornadoClient]:
|
||||
"""The shared client, or None when Tornado isn't configured or is switched off."""
|
||||
global _client
|
||||
settings = get_settings()
|
||||
if not settings.tornado_api_key or settings.youtube_download_provider != "tornado":
|
||||
return None
|
||||
if _client is None:
|
||||
_client = TornadoClient(settings.tornado_api_key)
|
||||
return _client
|
||||
|
||||
|
||||
def reset_tornado_client() -> None:
|
||||
global _client
|
||||
_client = None
|
||||
@@ -1,25 +1,30 @@
|
||||
"""
|
||||
YouTube download service for cloud backend.
|
||||
|
||||
This service provides a single entry point for all YouTube downloads in the cloud.
|
||||
flacfetch is the SOLE downloader: downloads are always routed through the remote
|
||||
flacfetch service, which has YouTube cookies and avoids bot detection on Cloud Run IPs.
|
||||
This service provides a single entry point for all YouTube downloads in the cloud:
|
||||
|
||||
The flow is:
|
||||
1. Check if remote flacfetch is configured (FLACFETCH_API_URL)
|
||||
2. If yes: Use FlacfetchClient.download_by_id() - downloads on VM, uploads to GCS
|
||||
3. If no: raise YouTubeDownloadError — there is NO local yt-dlp fallback. We never
|
||||
run yt-dlp inside the Cloud Run container (it's blocked by bot detection and a
|
||||
silent fallback previously masked a real download bug — 2026-06-08 incident).
|
||||
1. TornadoAPI first, when configured (TORNADO_API_KEY). It doesn't depend on our cookies.
|
||||
2. flacfetch, the remote service with YouTube cookies, if Tornado isn't configured or fails. It is also
|
||||
the only route for non-YouTube URLs.
|
||||
3. Otherwise raise YouTubeDownloadError. There is NO local yt-dlp fallback. We never run yt-dlp inside
|
||||
the Cloud Run container (it's blocked by bot detection and a silent fallback previously masked a
|
||||
real download bug — 2026-06-08 incident).
|
||||
|
||||
Falling back from Tornado to flacfetch always logs, so it can't hide a Tornado outage: WARNING for one
|
||||
failed download, ERROR when the account is rejected and every download would fail.
|
||||
|
||||
All entry points (audio search selection, direct URL submission) should use this
|
||||
service for YouTube downloads to ensure consistent behavior.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
|
||||
from .flacfetch_client import get_flacfetch_client, FlacfetchServiceError
|
||||
from .tornado_client import TornadoAccountError, TornadoError, get_tornado_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -61,6 +66,7 @@ class YouTubeDownloadService:
|
||||
|
||||
def __init__(self):
|
||||
self._flacfetch_client = get_flacfetch_client()
|
||||
self._tornado_client = get_tornado_client()
|
||||
|
||||
if self._flacfetch_client:
|
||||
logger.info("YouTubeDownloadService using REMOTE flacfetch")
|
||||
@@ -71,8 +77,8 @@ class YouTubeDownloadService:
|
||||
)
|
||||
|
||||
def is_remote_enabled(self) -> bool:
|
||||
"""Check if remote flacfetch is configured."""
|
||||
return self._flacfetch_client is not None
|
||||
"""Check if a remote download route (flacfetch or Tornado) is configured."""
|
||||
return self._flacfetch_client is not None or self._tornado_client is not None
|
||||
|
||||
async def check_availability(self, url: str) -> Optional[dict]:
|
||||
"""
|
||||
@@ -86,13 +92,23 @@ class YouTubeDownloadService:
|
||||
performed (no remote configured, network error, etc.).
|
||||
Callers should treat None as "unknown, proceed anyway".
|
||||
"""
|
||||
if not self._flacfetch_client:
|
||||
return None
|
||||
|
||||
video_id = self._extract_video_id(url)
|
||||
if not video_id:
|
||||
return None
|
||||
|
||||
# flacfetch's check uses the same cookies as its downloads, so while they're bot-blocked it
|
||||
# would turn away a video Tornado can fetch. A failed lookup here proves nothing, so ask
|
||||
# flacfetch in that case: it is the one that can say why a video isn't available.
|
||||
if self._tornado_client:
|
||||
try:
|
||||
metadata = await self._tornado_client.get_metadata(video_id)
|
||||
return {"available": True, "video_id": video_id, "title": metadata.get("title")}
|
||||
except Exception as e:
|
||||
logger.info(f"Tornado metadata lookup failed for {video_id}: {type(e).__name__}: {e}")
|
||||
|
||||
if not self._flacfetch_client:
|
||||
return None
|
||||
|
||||
try:
|
||||
result = await self._flacfetch_client.check_youtube(url)
|
||||
logger.info(
|
||||
@@ -127,17 +143,33 @@ class YouTubeDownloadService:
|
||||
Raises:
|
||||
YouTubeDownloadError: If download fails
|
||||
"""
|
||||
# flacfetch is the SOLE downloader — never run yt-dlp on Cloud Run.
|
||||
# If it isn't configured, fail loudly rather than silently falling back
|
||||
# to a local download (which is blocked by bot detection on Cloud Run
|
||||
# IPs anyway).
|
||||
video_id = self._extract_video_id(url)
|
||||
tornado_failure = None
|
||||
if video_id and self._tornado_client:
|
||||
try:
|
||||
return await self._download_tornado(video_id, job_id, artist, title)
|
||||
except TornadoAccountError as e:
|
||||
# Needs a person (key or billing), even though the fallback may save this job.
|
||||
logger.error(f"Tornado account problem, falling back to flacfetch: {e}")
|
||||
tornado_failure = e
|
||||
except Exception as e: # Whatever goes wrong with Tornado must not block flacfetch.
|
||||
logger.warning(
|
||||
f"Tornado download failed for {video_id}, falling back to flacfetch: "
|
||||
f"{type(e).__name__}: {e}"
|
||||
)
|
||||
tornado_failure = e
|
||||
|
||||
# Never run yt-dlp on Cloud Run. If flacfetch isn't configured, fail loudly rather
|
||||
# than silently falling back to a local download (which is blocked by bot
|
||||
# detection on Cloud Run IPs anyway).
|
||||
if not self._flacfetch_client:
|
||||
if tornado_failure:
|
||||
raise YouTubeDownloadError(f"Tornado download failed: {tornado_failure}") from tornado_failure
|
||||
raise YouTubeDownloadError(
|
||||
"Remote flacfetch is not configured (FLACFETCH_API_URL). "
|
||||
"URL downloads require the flacfetch service."
|
||||
)
|
||||
|
||||
video_id = self._extract_video_id(url)
|
||||
if video_id:
|
||||
logger.info(f"URL download (YouTube): video_id={video_id}, job_id={job_id}")
|
||||
return await self._download_remote(video_id, job_id, artist, title)
|
||||
@@ -207,6 +239,35 @@ class YouTubeDownloadService:
|
||||
|
||||
return None
|
||||
|
||||
async def _download_tornado(
|
||||
self,
|
||||
video_id: str,
|
||||
job_id: str,
|
||||
artist: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Download through Tornado and upload to GCS. Raises TornadoError on any failure."""
|
||||
from karaoke_gen.utils import sanitize_filename
|
||||
|
||||
from .storage_service import StorageService
|
||||
|
||||
stem = f"{sanitize_filename(artist)} - {sanitize_filename(title)}" if artist and title else video_id
|
||||
with tempfile.TemporaryDirectory(prefix="tornado-") as tmp:
|
||||
local_path = os.path.join(tmp, "audio")
|
||||
download = await self._tornado_client.download_audio(video_id, local_path, submission=job_id)
|
||||
gcs_path = f"uploads/{job_id}/audio/{stem}{download.extension}"
|
||||
try:
|
||||
await asyncio.to_thread(StorageService().upload_file, local_path, gcs_path)
|
||||
except Exception as e:
|
||||
raise TornadoError(f"Could not store the Tornado download: {type(e).__name__}") from e
|
||||
|
||||
logger.info(
|
||||
f"Tornado YouTube download complete: video_id={video_id}, job_id={job_id}, "
|
||||
f"tornado_job={download.tornado_job_id}, file_size={download.file_size}, "
|
||||
f"billable_bytes={download.billable_bytes}, gcs_path={gcs_path}"
|
||||
)
|
||||
return gcs_path
|
||||
|
||||
async def _download_remote(
|
||||
self,
|
||||
video_id: str,
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Tests for TornadoClient. Network is mocked; nothing here submits real work."""
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from backend.services.tornado_client import (
|
||||
TornadoAccountError,
|
||||
TornadoClient,
|
||||
TornadoError,
|
||||
)
|
||||
|
||||
VIDEO_ID = "dQw4w9WgXcQ"
|
||||
DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.opus?verify=signed"
|
||||
|
||||
|
||||
class FakeTornado:
|
||||
"""Scripted Tornado API. Each list is consumed in order; the last entry repeats."""
|
||||
|
||||
def __init__(self, create=None, status=None, delivery=None, metadata=None):
|
||||
self.create = create or [httpx.Response(201, json={"job_id": "job-1"})]
|
||||
self.status = status or [
|
||||
httpx.Response(200, json={"status": "Completed", "s3_url": DELIVERY_URL, "file_size": 5, "billable_bytes": 9})
|
||||
]
|
||||
self.delivery = delivery or httpx.Response(200, content=b"audio")
|
||||
self.metadata = metadata or httpx.Response(200, json={"title": "A Song"})
|
||||
self.requests = []
|
||||
|
||||
@staticmethod
|
||||
def _next(items):
|
||||
item = items.pop(0) if len(items) > 1 else items[0]
|
||||
if isinstance(item, Exception):
|
||||
raise item
|
||||
return item
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.requests.append(request)
|
||||
path, method = request.url.path, request.method
|
||||
if request.url.host == "files.tornadoapi.io":
|
||||
return self.delivery
|
||||
if method == "POST" and path == "/jobs":
|
||||
return self._next(self.create)
|
||||
if method == "POST" and path == "/metadata":
|
||||
return self.metadata
|
||||
if method == "GET" and path.startswith("/jobs/"):
|
||||
return self._next(self.status)
|
||||
if method == "DELETE" or path.endswith("/cancel"):
|
||||
return httpx.Response(200, json={})
|
||||
raise AssertionError(f"Unexpected request: {method} {request.url}")
|
||||
|
||||
def of(self, method, suffix):
|
||||
return [r for r in self.requests if r.method == method and r.url.path.endswith(suffix)]
|
||||
|
||||
|
||||
def make_client(fake, deadline=30.0):
|
||||
now = {"t": 0.0}
|
||||
|
||||
async def sleep(seconds):
|
||||
now["t"] += seconds
|
||||
|
||||
return TornadoClient(
|
||||
"test-key",
|
||||
deadline_seconds=deadline,
|
||||
poll_interval_seconds=3.0,
|
||||
transport=httpx.MockTransport(fake),
|
||||
sleep=sleep,
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_writes_file_and_reports_size(tmp_path):
|
||||
fake = FakeTornado()
|
||||
dest = tmp_path / "audio"
|
||||
result = await make_client(fake).download_audio(VIDEO_ID, str(dest), submission="gen-1")
|
||||
assert dest.read_bytes() == b"audio"
|
||||
assert result.tornado_job_id == "job-1"
|
||||
assert result.extension == ".ogg"
|
||||
assert (result.file_size, result.billable_bytes) == (5, 9)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requests_the_source_opus_stream_for_the_canonical_url(tmp_path):
|
||||
fake = FakeTornado()
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
body = json.loads(fake.of("POST", "/jobs")[0].content)
|
||||
# audio_codec must stay out: with "copy" Tornado returned 128k AAC instead of the Opus stream.
|
||||
assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "audio_only": True, "format": "opus"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_anything_but_a_video_id_so_a_playlist_url_cant_be_sent(tmp_path):
|
||||
fake = FakeTornado()
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(fake).download_audio(f"{VIDEO_ID}&list=PLabc", str(tmp_path / "a"), submission="gen-1")
|
||||
assert fake.requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delivery_url_never_receives_the_api_key(tmp_path):
|
||||
fake = FakeTornado()
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
delivery = [r for r in fake.requests if r.url.host == "files.tornadoapi.io"]
|
||||
assert len(delivery) == 1
|
||||
assert "x-api-key" not in delivery[0].headers
|
||||
assert fake.of("POST", "/jobs")[0].headers["x-api-key"] == "test-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deletes_the_file_after_fetching_it(tmp_path):
|
||||
fake = FakeTornado()
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert len(fake.of("DELETE", "/jobs/job-1/file")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_delete_doesnt_fail_the_download(tmp_path):
|
||||
fake = FakeTornado()
|
||||
original = fake.__call__
|
||||
|
||||
def handler(request):
|
||||
if request.method == "DELETE":
|
||||
raise httpx.ConnectError("down")
|
||||
return original(request)
|
||||
|
||||
client = make_client(fake)
|
||||
client._transport = httpx.MockTransport(handler)
|
||||
result = await client.download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert result.tornado_job_id == "job-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_timeout_retries_with_the_same_idempotency_key(tmp_path):
|
||||
fake = FakeTornado(create=[httpx.ReadTimeout("slow"), httpx.Response(201, json={"job_id": "job-1"})])
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
posts = fake.of("POST", "/jobs")
|
||||
assert len(posts) == 2
|
||||
assert posts[0].headers["x-idempotency-key"] == posts[1].headers["x-idempotency-key"]
|
||||
assert "gen-1" in posts[0].headers["x-idempotency-key"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_download_call_gets_its_own_idempotency_key(tmp_path):
|
||||
fake = FakeTornado()
|
||||
client = make_client(fake)
|
||||
await client.download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
await client.download_audio(VIDEO_ID, str(tmp_path / "b"), submission="gen-1")
|
||||
keys = {r.headers["x-idempotency-key"] for r in fake.of("POST", "/jobs")}
|
||||
assert len(keys) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_429_honours_retry_after_then_succeeds(tmp_path):
|
||||
slept = []
|
||||
fake = FakeTornado(
|
||||
create=[httpx.Response(429, headers={"Retry-After": "7"}), httpx.Response(201, json={"job_id": "job-1"})]
|
||||
)
|
||||
client = make_client(fake)
|
||||
real_sleep = client._sleep
|
||||
|
||||
async def sleep(seconds):
|
||||
slept.append(seconds)
|
||||
await real_sleep(seconds)
|
||||
|
||||
client._sleep = sleep
|
||||
await client.download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert slept[0] == 7.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gives_up_creating_after_repeated_5xx(tmp_path):
|
||||
fake = FakeTornado(create=[httpx.Response(503)])
|
||||
with pytest.raises(TornadoError, match="503"):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert len(fake.of("POST", "/jobs")) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("code", [401, 402, 403])
|
||||
async def test_account_errors_are_distinct_and_not_retried(tmp_path, code):
|
||||
fake = FakeTornado(create=[httpx.Response(code)])
|
||||
with pytest.raises(TornadoAccountError):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert len(fake.of("POST", "/jobs")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validation_error_is_not_retried(tmp_path):
|
||||
fake = FakeTornado(create=[httpx.Response(422, json={"error": "bad"})])
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert len(fake.of("POST", "/jobs")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_response_fails_rather_than_being_treated_as_a_job(tmp_path):
|
||||
fake = FakeTornado(create=[httpx.Response(201, json={"batch_id": "b-1", "total_episodes": 40})])
|
||||
with pytest.raises(TornadoError, match="batch"):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response", [httpx.Response(201, content=b"not json"), httpx.Response(201, json={}), httpx.Response(201, json=[1])]
|
||||
)
|
||||
async def test_malformed_create_response(tmp_path, response):
|
||||
fake = FakeTornado(create=[response])
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["Failed", "Warning", "Skipped", "Cancelled", "CancelledByAdmin"])
|
||||
async def test_terminal_statuses_other_than_completed_fail_with_the_reason(tmp_path, status):
|
||||
fake = FakeTornado(
|
||||
status=[httpx.Response(200, json={"status": status, "error": "video unavailable", "error_type": "warning"})]
|
||||
)
|
||||
with pytest.raises(TornadoError, match="video unavailable"):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert not (tmp_path / "a").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polls_until_completed_surviving_failed_reads(tmp_path):
|
||||
fake = FakeTornado(
|
||||
status=[
|
||||
httpx.Response(200, json={"status": "Pending"}),
|
||||
httpx.ConnectError("blip"),
|
||||
httpx.Response(502),
|
||||
httpx.Response(200, content=b"not json"),
|
||||
httpx.Response(200, json=["not an object"]),
|
||||
httpx.Response(200, json={"status": "Processing"}),
|
||||
httpx.Response(200, json={"status": "Completed", "s3_url": DELIVERY_URL}),
|
||||
]
|
||||
)
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert (tmp_path / "a").read_bytes() == b"audio"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["Processing", "SomethingNew"])
|
||||
async def test_deadline_stops_polling_and_cancels_the_job(tmp_path, status):
|
||||
fake = FakeTornado(status=[httpx.Response(200, json={"status": status})])
|
||||
with pytest.raises(TornadoError, match="timed out"):
|
||||
await make_client(fake, deadline=9.0).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert len(fake.of("POST", "/jobs/job-1/cancel")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_account_error_while_polling_stops_immediately(tmp_path):
|
||||
fake = FakeTornado(status=[httpx.Response(401)])
|
||||
with pytest.raises(TornadoAccountError):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert len(fake.of("GET", "/jobs/job-1")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_without_a_delivery_url_fails(tmp_path):
|
||||
fake = FakeTornado(status=[httpx.Response(200, json={"status": "Completed"})])
|
||||
with pytest.raises(TornadoError, match="delivery URL"):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("delivery", [httpx.Response(403), httpx.Response(200, content=b"")])
|
||||
async def test_bad_delivery_leaves_no_partial_file(tmp_path, delivery):
|
||||
fake = FakeTornado(delivery=delivery)
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert not (tmp_path / "a").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_messages_dont_expose_the_signed_delivery_url(tmp_path):
|
||||
fake = FakeTornado(delivery=httpx.Response(403))
|
||||
with pytest.raises(TornadoError) as e:
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert "verify=" not in str(e.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_success_and_failure():
|
||||
assert (await make_client(FakeTornado()).get_metadata(VIDEO_ID))["title"] == "A Song"
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(FakeTornado(metadata=httpx.Response(400, json={"error": "x"}))).get_metadata(VIDEO_ID)
|
||||
with pytest.raises(TornadoAccountError):
|
||||
await make_client(FakeTornado(metadata=httpx.Response(401))).get_metadata(VIDEO_ID)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("https://files.tornadoapi.io/videos/abc.opus?verify=x", ".ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc.ogg", ".ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc", ".ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc.m4a", ".m4a"),
|
||||
],
|
||||
)
|
||||
def test_extension_comes_from_the_delivery_path(url, expected):
|
||||
assert TornadoClient._extension(url) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unwritable_destination_is_a_tornado_error(tmp_path):
|
||||
with pytest.raises(TornadoError, match="Writing"):
|
||||
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "missing" / "a"), submission="gen-1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_that_isnt_an_object_is_a_tornado_error():
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(FakeTornado(metadata=httpx.Response(200, json=[]))).get_metadata(VIDEO_ID)
|
||||
@@ -8,6 +8,13 @@ import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from backend.services.flacfetch_client import FlacfetchServiceError
|
||||
from backend.services.tornado_client import (
|
||||
TornadoAccountError,
|
||||
TornadoDownload,
|
||||
TornadoError,
|
||||
get_tornado_client,
|
||||
reset_tornado_client,
|
||||
)
|
||||
from backend.services.youtube_download_service import (
|
||||
YouTubeDownloadService,
|
||||
YouTubeDownloadError,
|
||||
@@ -459,3 +466,138 @@ class TestCheckAvailability:
|
||||
|
||||
assert result is None
|
||||
mock_client.check_youtube.assert_not_called()
|
||||
|
||||
|
||||
class TestTornadoRouting:
|
||||
"""Tornado first for YouTube, flacfetch as the logged fallback and for everything else."""
|
||||
|
||||
URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
|
||||
def make_service(self, tornado=True, flacfetch=True):
|
||||
reset_youtube_download_service()
|
||||
service = YouTubeDownloadService()
|
||||
service._tornado_client = MagicMock() if tornado else None
|
||||
if tornado:
|
||||
service._tornado_client.download_audio = AsyncMock(
|
||||
return_value=TornadoDownload("t-1", ".ogg", 5, 9)
|
||||
)
|
||||
service._tornado_client.get_metadata = AsyncMock(return_value={"title": "A Song"})
|
||||
service._flacfetch_client = MagicMock() if flacfetch else None
|
||||
service._download_remote = AsyncMock(return_value="uploads/j/audio/flacfetch.webm")
|
||||
service._download_remote_url = AsyncMock(return_value="uploads/j/audio/generic.mp3")
|
||||
if flacfetch:
|
||||
service._flacfetch_client.check_youtube = AsyncMock(return_value={"available": False, "is_bot_blocked": True})
|
||||
return service
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tornado_download_goes_to_gcs_named_like_flacfetch(self):
|
||||
service = self.make_service()
|
||||
with patch("backend.services.storage_service.StorageService") as storage:
|
||||
path = await service.download(self.URL, "job1", artist="An Artist", title="A Title")
|
||||
assert path == "uploads/job1/audio/An Artist - A Title.ogg"
|
||||
assert storage.return_value.upload_file.call_args.args[1] == path
|
||||
service._download_remote.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_artist_uses_the_video_id_as_filename(self):
|
||||
service = self.make_service()
|
||||
with patch("backend.services.storage_service.StorageService"):
|
||||
path = await service.download(self.URL, "job1")
|
||||
assert path == "uploads/job1/audio/dQw4w9WgXcQ.ogg"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_url_with_a_playlist_only_passes_the_video_id(self):
|
||||
service = self.make_service()
|
||||
with patch("backend.services.storage_service.StorageService"):
|
||||
await service.download(self.URL + "&list=PLabcdefghijk", "job1")
|
||||
assert service._tornado_client.download_audio.call_args.args[0] == "dQw4w9WgXcQ"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tornado_failure_falls_back_with_a_warning(self, caplog):
|
||||
service = self.make_service()
|
||||
service._tornado_client.download_audio = AsyncMock(side_effect=TornadoError("ended Failed"))
|
||||
with caplog.at_level("WARNING"):
|
||||
path = await service.download(self.URL, "job1")
|
||||
assert path == "uploads/j/audio/flacfetch.webm"
|
||||
records = [r for r in caplog.records if "falling back to flacfetch" in r.getMessage()]
|
||||
assert [r.levelname for r in records] == ["WARNING"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_account_problem_falls_back_with_an_error_that_alerts(self, caplog):
|
||||
service = self.make_service()
|
||||
service._tornado_client.download_audio = AsyncMock(side_effect=TornadoAccountError("HTTP 402"))
|
||||
with caplog.at_level("WARNING"):
|
||||
path = await service.download(self.URL, "job1")
|
||||
assert path == "uploads/j/audio/flacfetch.webm"
|
||||
assert [r.levelname for r in caplog.records if "falling back" in r.getMessage()] == ["ERROR"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unexpected_tornado_exception_still_falls_back(self):
|
||||
service = self.make_service()
|
||||
service._tornado_client.download_audio = AsyncMock(side_effect=AttributeError("surprise"))
|
||||
assert await service.download(self.URL, "job1") == "uploads/j/audio/flacfetch.webm"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unexpected_metadata_exception_still_asks_flacfetch(self):
|
||||
service = self.make_service()
|
||||
service._tornado_client.get_metadata = AsyncMock(side_effect=AttributeError("surprise"))
|
||||
assert (await service.check_availability(self.URL))["is_bot_blocked"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gcs_failure_falls_back(self):
|
||||
service = self.make_service()
|
||||
with patch("backend.services.storage_service.StorageService") as storage:
|
||||
storage.return_value.upload_file.side_effect = RuntimeError("gcs down")
|
||||
path = await service.download(self.URL, "job1")
|
||||
assert path == "uploads/j/audio/flacfetch.webm"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tornado_works_without_flacfetch(self):
|
||||
service = self.make_service(flacfetch=False)
|
||||
with patch("backend.services.storage_service.StorageService"):
|
||||
path = await service.download(self.URL, "job1")
|
||||
assert path.endswith(".ogg")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_both_failing_raises_a_clear_error_without_flacfetch(self):
|
||||
service = self.make_service(flacfetch=False)
|
||||
service._tornado_client.download_audio = AsyncMock(side_effect=TornadoError("boom"))
|
||||
with pytest.raises(YouTubeDownloadError, match="Tornado download failed: boom"):
|
||||
await service.download(self.URL, "job1")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_youtube_urls_never_go_to_tornado(self):
|
||||
service = self.make_service()
|
||||
path = await service.download("https://soundcloud.com/a/b", "job1")
|
||||
assert path == "uploads/j/audio/generic.mp3"
|
||||
service._tornado_client.download_audio.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_without_tornado_youtube_uses_flacfetch(self):
|
||||
service = self.make_service(tornado=False)
|
||||
assert await service.download(self.URL, "job1") == "uploads/j/audio/flacfetch.webm"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_availability_passes_on_tornado_metadata_despite_a_bot_blocked_flacfetch(self):
|
||||
service = self.make_service()
|
||||
result = await service.check_availability(self.URL)
|
||||
assert result["available"] is True
|
||||
service._flacfetch_client.check_youtube.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_availability_asks_flacfetch_when_tornado_lookup_fails(self):
|
||||
service = self.make_service()
|
||||
service._tornado_client.get_metadata = AsyncMock(side_effect=TornadoError("unavailable"))
|
||||
result = await service.check_availability(self.URL)
|
||||
assert result["is_bot_blocked"] is True
|
||||
|
||||
def test_settings_switch(self):
|
||||
reset_tornado_client()
|
||||
with patch("backend.services.tornado_client.get_settings") as settings:
|
||||
settings.return_value = MagicMock(tornado_api_key=None, youtube_download_provider="tornado")
|
||||
assert get_tornado_client() is None
|
||||
settings.return_value = MagicMock(tornado_api_key="k", youtube_download_provider="flacfetch")
|
||||
assert get_tornado_client() is None
|
||||
settings.return_value = MagicMock(tornado_api_key="k", youtube_download_provider="tornado")
|
||||
assert get_tornado_client() is not None
|
||||
reset_tornado_client()
|
||||
@@ -638,7 +638,8 @@ The Video Worker uses an orchestrator pattern to ensure all features work regard
|
||||
**Service Modules**:
|
||||
| Module | Purpose |
|
||||
|--------|---------|
|
||||
| `youtube_download_service.py` | YouTube audio downloads (via remote flacfetch) |
|
||||
| `youtube_download_service.py` | YouTube audio downloads (TornadoAPI first, then remote flacfetch) |
|
||||
| `tornado_client.py` | Client for TornadoAPI (hosted YouTube downloads) |
|
||||
| `flacfetch_client.py` | Client for remote flacfetch HTTP API |
|
||||
| `audio_search_service.py` | Audio source search (YouTube, RED, OPS, Spotify) |
|
||||
| `youtube_upload_service.py` | YouTube OAuth flow and upload |
|
||||
@@ -704,6 +705,8 @@ Audio can come from file upload or remote search (YouTube, torrents). All YouTub
|
||||
2026-09-26; the backend and `audio-download-job` / `bulk-search-job` have no VPC egress, so every
|
||||
dependency they call must be reachable on a public address.
|
||||
- `FLACFETCH_API_KEY` - API key for authentication
|
||||
- `TORNADO_API_KEY` - TornadoAPI key (secret `tornado-api-key`). When set, YouTube downloads and availability checks try Tornado first and fall back to flacfetch: WARNING per failed download, ERROR if the account is rejected. CI wires the secret once it has a version. It covers downloads only, so YouTube search still uses flacfetch's cookies.
|
||||
- `YOUTUBE_DOWNLOAD_PROVIDER` - set to `flacfetch` to switch Tornado off, e.g. `gcloud run services update karaoke-backend --update-env-vars YOUTUBE_DOWNLOAD_PROVIDER=flacfetch` (default `tornado`; the next CI deploy resets it, so also change the workflow to make it permanent).
|
||||
|
||||
## Worker Execution
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.290.4"
|
||||
version = "0.291.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user