mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(youtube): get source Opus from Tornado reliably; keep signed URLs out of logs (0.291.2)
audio_only returned Opus or 128k AAC unpredictably on our plan. A 360p WebM with its video stripped always carried the source Opus, packet-identical to flacfetch, at the same billable size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG
This commit is contained in:
1 parent
688ca162f8
commit
3a6cbdb901
3 files changed
+136
-31
No files matched your search
@@ -8,9 +8,10 @@ Three things that are easy to get wrong:
|
||||
|
||||
- Only ever send a canonical ``watch?v=<id>`` URL. A URL carrying a real playlist id downloads and
|
||||
bills the whole playlist, returning ``batch_id`` rather than ``job_id``.
|
||||
- ``format: "opus"`` without ``audio_codec`` returns the source's Opus stream, packet-identical to what
|
||||
flacfetch delivers (checked 2026-11-03). Adding ``audio_codec: "copy"`` or leaving ``format`` out
|
||||
returns YouTube's 128k AAC instead.
|
||||
- Request ``format: "webm", max_resolution: "360"`` and strip the video, because that returns the source's
|
||||
Opus stream, packet-identical to what flacfetch delivers. ``audio_only`` isn't reliable for this: on our
|
||||
plan the same request returned Opus or 128k AAC from one day to the next (checked 2026-11-03/05). It
|
||||
costs no more: audio-only and sub-720p video both bill the same full-source 720p reference.
|
||||
- Delivered files are only kept for 24 hours and count toward a storage limit, so copy them
|
||||
straight away and delete them. The delivery URL is signed: never send it the API key or log it.
|
||||
"""
|
||||
@@ -19,6 +20,7 @@ import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional
|
||||
@@ -30,6 +32,17 @@ from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _NoSignedUrls(logging.Filter):
|
||||
"""httpx logs every request's full URL at INFO, which would put delivery signatures in our logs."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
message = record.getMessage()
|
||||
return "X-Amz-Signature" not in message and "verify=" not in message
|
||||
|
||||
|
||||
logging.getLogger("httpx").addFilter(_NoSignedUrls())
|
||||
|
||||
BASE_URL = "https://api.tornadoapi.io"
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
@@ -38,7 +51,17 @@ _TERMINAL = {"Completed", "Failed", "Warning", "Skipped", "Cancelled", "Cancelle
|
||||
|
||||
_CREATE_ATTEMPTS = 3
|
||||
_MAX_RETRY_AFTER_SECONDS = 30.0
|
||||
_MAX_FILE_BYTES = 1024 * 1024 * 1024 # audio only; anything this big isn't a song
|
||||
_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024 # includes a 360p video track; anything this big isn't a song
|
||||
_DEMUX_TIMEOUT_SECONDS = 120
|
||||
# Delivered container -> (extension for the audio we keep, ffmpeg muxer). Audio is never re-encoded.
|
||||
_AUDIO_CONTAINERS = {
|
||||
".webm": (".webm", "webm"),
|
||||
".mkv": (".webm", "webm"),
|
||||
".opus": (".ogg", "ogg"), # .ogg: gen's upload and review allow-lists reject .opus
|
||||
".ogg": (".ogg", "ogg"),
|
||||
".mp4": (".m4a", "ipod"),
|
||||
".m4a": (".m4a", "ipod"),
|
||||
}
|
||||
|
||||
|
||||
class TornadoError(Exception):
|
||||
@@ -67,6 +90,7 @@ class TornadoClient:
|
||||
poll_interval_seconds: float = 3.0,
|
||||
transport: Optional[httpx.AsyncBaseTransport] = None,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
demux: Optional[Callable[[str, str, str], None]] = None,
|
||||
clock: Callable[[], float] = lambda: asyncio.get_running_loop().time(),
|
||||
):
|
||||
self._api_key = api_key
|
||||
@@ -75,6 +99,7 @@ class TornadoClient:
|
||||
self._poll_interval_seconds = poll_interval_seconds
|
||||
self._transport = transport
|
||||
self._sleep = sleep
|
||||
self._demux = demux or _demux_audio
|
||||
self._clock = clock
|
||||
|
||||
def _http(self, timeout: float = 30.0) -> httpx.AsyncClient:
|
||||
@@ -125,11 +150,17 @@ class TornadoClient:
|
||||
if not delivery_url:
|
||||
raise TornadoError(f"Tornado job {job_id} completed without a delivery URL")
|
||||
|
||||
await self._fetch_file(delivery_url, dest_path)
|
||||
extension, muxer = self._audio_container(delivery_url)
|
||||
source_path = f"{dest_path}.source"
|
||||
try:
|
||||
await self._fetch_file(delivery_url, source_path)
|
||||
await asyncio.to_thread(self._demux, source_path, dest_path, muxer)
|
||||
finally:
|
||||
self._remove_partial(source_path)
|
||||
await self._delete_file(job_id)
|
||||
return TornadoDownload(
|
||||
tornado_job_id=job_id,
|
||||
extension=self._extension(delivery_url),
|
||||
extension=extension,
|
||||
file_size=job.get("file_size"),
|
||||
billable_bytes=job.get("billable_bytes"),
|
||||
)
|
||||
@@ -148,7 +179,7 @@ class TornadoClient:
|
||||
raise TornadoAccountError(f"Tornado rejected the account: HTTP {resp.status_code}")
|
||||
|
||||
async def _create_job(self, url: str, idempotency_key: str) -> str:
|
||||
body = {"url": url, "audio_only": True, "format": "opus"}
|
||||
body = {"url": url, "format": "webm", "max_resolution": "360"}
|
||||
headers = {**self._headers(), "x-idempotency-key": idempotency_key}
|
||||
last_problem = "no attempt made"
|
||||
retry_after: Optional[str] = None
|
||||
@@ -280,13 +311,28 @@ class TornadoClient:
|
||||
logger.info(f"Could not cancel Tornado job {job_id}: {type(e).__name__}")
|
||||
|
||||
@staticmethod
|
||||
def _extension(delivery_url: str) -> str:
|
||||
# We ask for Opus, which arrives in an Ogg container. Name it .ogg: that's accurate, and
|
||||
# gen's upload and review allow-lists accept .ogg but not .opus.
|
||||
def _audio_container(delivery_url: str) -> "tuple[str, str]":
|
||||
ext = os.path.splitext(urlsplit(delivery_url).path)[1].lower()
|
||||
if ext == ".opus" or not re.fullmatch(r"\.[a-z0-9]{2,5}", ext):
|
||||
return ".ogg"
|
||||
return ext
|
||||
return _AUDIO_CONTAINERS.get(ext, _AUDIO_CONTAINERS[".webm"])
|
||||
|
||||
|
||||
def _demux_audio(source_path: str, dest_path: str, muxer: str) -> None:
|
||||
"""Keep only the audio track, without re-encoding it."""
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", source_path,
|
||||
"-map", "0:a:0", "-c:a", "copy", "-f", muxer, dest_path],
|
||||
check=True, capture_output=True, text=True, timeout=_DEMUX_TIMEOUT_SECONDS,
|
||||
)
|
||||
if not os.path.getsize(dest_path):
|
||||
raise TornadoError("ffmpeg produced an empty audio file")
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
try:
|
||||
os.remove(dest_path)
|
||||
except OSError:
|
||||
pass
|
||||
stderr = getattr(e, "stderr", None) or ""
|
||||
raise TornadoError(f"Could not extract the audio track: {type(e).__name__} {stderr[:200]}") from e
|
||||
|
||||
|
||||
_client: Optional[TornadoClient] = None
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
"""Tests for TornadoClient. Network is mocked; nothing here submits real work."""
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from backend.services.tornado_client import (
|
||||
_demux_audio,
|
||||
TornadoAccountError,
|
||||
TornadoClient,
|
||||
TornadoError,
|
||||
)
|
||||
|
||||
VIDEO_ID = "dQw4w9WgXcQ"
|
||||
DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.opus?verify=signed"
|
||||
DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.webm?verify=signed"
|
||||
|
||||
|
||||
class FakeTornado:
|
||||
@@ -64,6 +68,7 @@ def make_client(fake, deadline=30.0):
|
||||
poll_interval_seconds=3.0,
|
||||
transport=httpx.MockTransport(fake),
|
||||
sleep=sleep,
|
||||
demux=lambda source, dest, muxer: shutil.copy(source, dest),
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
|
||||
@@ -75,17 +80,17 @@ async def test_download_writes_file_and_reports_size(tmp_path):
|
||||
result = await make_client(fake).download_audio(VIDEO_ID, str(dest), submission="gen-1")
|
||||
assert dest.read_bytes() == b"audio"
|
||||
assert result.tornado_job_id == "job-1"
|
||||
assert result.extension == ".ogg"
|
||||
assert result.extension == ".webm"
|
||||
assert (result.file_size, result.billable_bytes) == (5, 9)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requests_the_source_opus_stream_for_the_canonical_url(tmp_path):
|
||||
async def test_requests_the_route_that_returns_source_opus_for_the_canonical_url(tmp_path):
|
||||
fake = FakeTornado()
|
||||
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
body = json.loads(fake.of("POST", "/jobs")[0].content)
|
||||
# audio_codec must stay out: with "copy" Tornado returned 128k AAC instead of the Opus stream.
|
||||
assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "audio_only": True, "format": "opus"}
|
||||
# Not audio_only: that returned 128k AAC or Opus unpredictably (2026-11-05).
|
||||
assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "format": "webm", "max_resolution": "360"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -288,25 +293,79 @@ async def test_metadata_success_and_failure():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
"url,extension,muxer",
|
||||
[
|
||||
("https://files.tornadoapi.io/videos/abc.opus?verify=x", ".ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc.ogg", ".ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc", ".ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc.m4a", ".m4a"),
|
||||
("https://files.tornadoapi.io/videos/abc.webm?verify=x", ".webm", "webm"),
|
||||
("https://fsn1.your-objectstorage.com/tornado/videos/abc.opus?X-Amz-Signature=x", ".ogg", "ogg"),
|
||||
("https://files.tornadoapi.io/videos/abc.mp4", ".m4a", "ipod"),
|
||||
("https://files.tornadoapi.io/videos/abc.m4a", ".m4a", "ipod"),
|
||||
("https://files.tornadoapi.io/videos/abc", ".webm", "webm"),
|
||||
],
|
||||
)
|
||||
def test_extension_comes_from_the_delivery_path(url, expected):
|
||||
assert TornadoClient._extension(url) == expected
|
||||
def test_audio_container_comes_from_the_delivery_path(url, extension, muxer):
|
||||
assert TornadoClient._audio_container(url) == (extension, muxer)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unwritable_destination_is_a_tornado_error(tmp_path):
|
||||
with pytest.raises(TornadoError, match="Writing"):
|
||||
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "missing" / "a"), submission="gen-1")
|
||||
async def test_source_file_is_removed_after_extraction(tmp_path):
|
||||
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert [p.name for p in tmp_path.iterdir()] == ["a"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_that_isnt_an_object_is_a_tornado_error():
|
||||
async def test_failed_extraction_fails_the_download_and_keeps_tornados_file(tmp_path):
|
||||
fake = FakeTornado()
|
||||
client = make_client(fake)
|
||||
|
||||
def demux(source, dest, muxer):
|
||||
raise TornadoError("no audio track")
|
||||
|
||||
client._demux = demux
|
||||
with pytest.raises(TornadoError, match="no audio track"):
|
||||
await client.download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert fake.of("DELETE", "/jobs/job-1/file") == []
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_signed_delivery_urls_stay_out_of_the_logs(tmp_path, caplog):
|
||||
with caplog.at_level(logging.INFO, logger="httpx"):
|
||||
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
|
||||
assert "HTTP Request" in caplog.text # other requests still log
|
||||
assert "verify=" not in caplog.text
|
||||
|
||||
|
||||
def _packet_md5(path):
|
||||
return subprocess.run(
|
||||
["ffmpeg", "-v", "error", "-i", str(path), "-map", "0:a", "-c", "copy", "-f", "md5", "-"],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
|
||||
def test_demux_keeps_the_audio_packets_and_drops_the_video(tmp_path):
|
||||
source = tmp_path / "source.mkv"
|
||||
made = subprocess.run(
|
||||
["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "sine=duration=1", "-f", "lavfi", "-i",
|
||||
"color=size=64x64:duration=1", "-c:a", "aac", "-c:v", "mpeg4", str(source)],
|
||||
capture_output=True,
|
||||
)
|
||||
if made.returncode:
|
||||
pytest.skip("this ffmpeg can't make the fixture")
|
||||
dest = tmp_path / "audio"
|
||||
_demux_audio(str(source), str(dest), "ipod")
|
||||
streams = subprocess.run(
|
||||
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type", "-of", "csv=p=0", str(dest)],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.split()
|
||||
assert streams == ["audio"]
|
||||
assert _packet_md5(dest) == _packet_md5(source)
|
||||
|
||||
|
||||
def test_demux_failure_is_a_tornado_error_and_leaves_nothing(tmp_path):
|
||||
source = tmp_path / "source"
|
||||
source.write_bytes(b"not media")
|
||||
dest = tmp_path / "audio"
|
||||
with pytest.raises(TornadoError):
|
||||
await make_client(FakeTornado(metadata=httpx.Response(200, json=[]))).get_metadata(VIDEO_ID)
|
||||
_demux_audio(str(source), str(dest), "webm")
|
||||
assert not dest.exists()
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.291.1"
|
||||
version = "0.291.2"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user