From 3a6cbdb90179d13fa4479a265128a9a554703ecc Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Thu, 5 Nov 2026 00:01:51 -0500 Subject: [PATCH] fix(youtube): get source Opus from Tornado reliably; keep signed URLs out of logs (0.291.2) audio_only returned Opus or 128k AAC unpredictably on our plan. A 360p WebM with its video stripped always carried the source Opus, packet-identical to flacfetch, at the same billable size. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG --- backend/services/tornado_client.py | 72 +++++++++++++++++---- backend/tests/test_tornado_client.py | 93 +++++++++++++++++++++++----- pyproject.toml | 2 +- 3 files changed, 136 insertions(+), 31 deletions(-) diff --git a/backend/services/tornado_client.py b/backend/services/tornado_client.py index 75921e86..df42d543 100644 --- a/backend/services/tornado_client.py +++ b/backend/services/tornado_client.py @@ -8,9 +8,10 @@ Three things that are easy to get wrong: - Only ever send a canonical ``watch?v=`` URL. A URL carrying a real playlist id downloads and bills the whole playlist, returning ``batch_id`` rather than ``job_id``. -- ``format: "opus"`` without ``audio_codec`` returns the source's Opus stream, packet-identical to what - flacfetch delivers (checked 2026-11-03). Adding ``audio_codec: "copy"`` or leaving ``format`` out - returns YouTube's 128k AAC instead. +- Request ``format: "webm", max_resolution: "360"`` and strip the video, because that returns the source's + Opus stream, packet-identical to what flacfetch delivers. ``audio_only`` isn't reliable for this: on our + plan the same request returned Opus or 128k AAC from one day to the next (checked 2026-11-03/05). It + costs no more: audio-only and sub-720p video both bill the same full-source 720p reference. - Delivered files are only kept for 24 hours and count toward a storage limit, so copy them straight away and delete them. The delivery URL is signed: never send it the API key or log it. """ @@ -19,6 +20,7 @@ import logging import os import random import re +import subprocess import uuid from dataclasses import dataclass from typing import Any, Awaitable, Callable, Dict, Optional @@ -30,6 +32,17 @@ from backend.config import get_settings logger = logging.getLogger(__name__) + +class _NoSignedUrls(logging.Filter): + """httpx logs every request's full URL at INFO, which would put delivery signatures in our logs.""" + + def filter(self, record: logging.LogRecord) -> bool: + message = record.getMessage() + return "X-Amz-Signature" not in message and "verify=" not in message + + +logging.getLogger("httpx").addFilter(_NoSignedUrls()) + BASE_URL = "https://api.tornadoapi.io" _VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") @@ -38,7 +51,17 @@ _TERMINAL = {"Completed", "Failed", "Warning", "Skipped", "Cancelled", "Cancelle _CREATE_ATTEMPTS = 3 _MAX_RETRY_AFTER_SECONDS = 30.0 -_MAX_FILE_BYTES = 1024 * 1024 * 1024 # audio only; anything this big isn't a song +_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024 # includes a 360p video track; anything this big isn't a song +_DEMUX_TIMEOUT_SECONDS = 120 +# Delivered container -> (extension for the audio we keep, ffmpeg muxer). Audio is never re-encoded. +_AUDIO_CONTAINERS = { + ".webm": (".webm", "webm"), + ".mkv": (".webm", "webm"), + ".opus": (".ogg", "ogg"), # .ogg: gen's upload and review allow-lists reject .opus + ".ogg": (".ogg", "ogg"), + ".mp4": (".m4a", "ipod"), + ".m4a": (".m4a", "ipod"), +} class TornadoError(Exception): @@ -67,6 +90,7 @@ class TornadoClient: poll_interval_seconds: float = 3.0, transport: Optional[httpx.AsyncBaseTransport] = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + demux: Optional[Callable[[str, str, str], None]] = None, clock: Callable[[], float] = lambda: asyncio.get_running_loop().time(), ): self._api_key = api_key @@ -75,6 +99,7 @@ class TornadoClient: self._poll_interval_seconds = poll_interval_seconds self._transport = transport self._sleep = sleep + self._demux = demux or _demux_audio self._clock = clock def _http(self, timeout: float = 30.0) -> httpx.AsyncClient: @@ -125,11 +150,17 @@ class TornadoClient: if not delivery_url: raise TornadoError(f"Tornado job {job_id} completed without a delivery URL") - await self._fetch_file(delivery_url, dest_path) + extension, muxer = self._audio_container(delivery_url) + source_path = f"{dest_path}.source" + try: + await self._fetch_file(delivery_url, source_path) + await asyncio.to_thread(self._demux, source_path, dest_path, muxer) + finally: + self._remove_partial(source_path) await self._delete_file(job_id) return TornadoDownload( tornado_job_id=job_id, - extension=self._extension(delivery_url), + extension=extension, file_size=job.get("file_size"), billable_bytes=job.get("billable_bytes"), ) @@ -148,7 +179,7 @@ class TornadoClient: raise TornadoAccountError(f"Tornado rejected the account: HTTP {resp.status_code}") async def _create_job(self, url: str, idempotency_key: str) -> str: - body = {"url": url, "audio_only": True, "format": "opus"} + body = {"url": url, "format": "webm", "max_resolution": "360"} headers = {**self._headers(), "x-idempotency-key": idempotency_key} last_problem = "no attempt made" retry_after: Optional[str] = None @@ -280,13 +311,28 @@ class TornadoClient: logger.info(f"Could not cancel Tornado job {job_id}: {type(e).__name__}") @staticmethod - def _extension(delivery_url: str) -> str: - # We ask for Opus, which arrives in an Ogg container. Name it .ogg: that's accurate, and - # gen's upload and review allow-lists accept .ogg but not .opus. + def _audio_container(delivery_url: str) -> "tuple[str, str]": ext = os.path.splitext(urlsplit(delivery_url).path)[1].lower() - if ext == ".opus" or not re.fullmatch(r"\.[a-z0-9]{2,5}", ext): - return ".ogg" - return ext + return _AUDIO_CONTAINERS.get(ext, _AUDIO_CONTAINERS[".webm"]) + + +def _demux_audio(source_path: str, dest_path: str, muxer: str) -> None: + """Keep only the audio track, without re-encoding it.""" + try: + subprocess.run( + ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", source_path, + "-map", "0:a:0", "-c:a", "copy", "-f", muxer, dest_path], + check=True, capture_output=True, text=True, timeout=_DEMUX_TIMEOUT_SECONDS, + ) + if not os.path.getsize(dest_path): + raise TornadoError("ffmpeg produced an empty audio file") + except (OSError, subprocess.SubprocessError) as e: + try: + os.remove(dest_path) + except OSError: + pass + stderr = getattr(e, "stderr", None) or "" + raise TornadoError(f"Could not extract the audio track: {type(e).__name__} {stderr[:200]}") from e _client: Optional[TornadoClient] = None diff --git a/backend/tests/test_tornado_client.py b/backend/tests/test_tornado_client.py index e961f405..980bb172 100644 --- a/backend/tests/test_tornado_client.py +++ b/backend/tests/test_tornado_client.py @@ -1,17 +1,21 @@ """Tests for TornadoClient. Network is mocked; nothing here submits real work.""" import json +import logging +import shutil +import subprocess import httpx import pytest from backend.services.tornado_client import ( + _demux_audio, TornadoAccountError, TornadoClient, TornadoError, ) VIDEO_ID = "dQw4w9WgXcQ" -DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.opus?verify=signed" +DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.webm?verify=signed" class FakeTornado: @@ -64,6 +68,7 @@ def make_client(fake, deadline=30.0): poll_interval_seconds=3.0, transport=httpx.MockTransport(fake), sleep=sleep, + demux=lambda source, dest, muxer: shutil.copy(source, dest), clock=lambda: now["t"], ) @@ -75,17 +80,17 @@ async def test_download_writes_file_and_reports_size(tmp_path): result = await make_client(fake).download_audio(VIDEO_ID, str(dest), submission="gen-1") assert dest.read_bytes() == b"audio" assert result.tornado_job_id == "job-1" - assert result.extension == ".ogg" + assert result.extension == ".webm" assert (result.file_size, result.billable_bytes) == (5, 9) @pytest.mark.asyncio -async def test_requests_the_source_opus_stream_for_the_canonical_url(tmp_path): +async def test_requests_the_route_that_returns_source_opus_for_the_canonical_url(tmp_path): fake = FakeTornado() await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1") body = json.loads(fake.of("POST", "/jobs")[0].content) - # audio_codec must stay out: with "copy" Tornado returned 128k AAC instead of the Opus stream. - assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "audio_only": True, "format": "opus"} + # Not audio_only: that returned 128k AAC or Opus unpredictably (2026-11-05). + assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "format": "webm", "max_resolution": "360"} @pytest.mark.asyncio @@ -288,25 +293,79 @@ async def test_metadata_success_and_failure(): @pytest.mark.parametrize( - "url,expected", + "url,extension,muxer", [ - ("https://files.tornadoapi.io/videos/abc.opus?verify=x", ".ogg"), - ("https://files.tornadoapi.io/videos/abc.ogg", ".ogg"), - ("https://files.tornadoapi.io/videos/abc", ".ogg"), - ("https://files.tornadoapi.io/videos/abc.m4a", ".m4a"), + ("https://files.tornadoapi.io/videos/abc.webm?verify=x", ".webm", "webm"), + ("https://fsn1.your-objectstorage.com/tornado/videos/abc.opus?X-Amz-Signature=x", ".ogg", "ogg"), + ("https://files.tornadoapi.io/videos/abc.mp4", ".m4a", "ipod"), + ("https://files.tornadoapi.io/videos/abc.m4a", ".m4a", "ipod"), + ("https://files.tornadoapi.io/videos/abc", ".webm", "webm"), ], ) -def test_extension_comes_from_the_delivery_path(url, expected): - assert TornadoClient._extension(url) == expected +def test_audio_container_comes_from_the_delivery_path(url, extension, muxer): + assert TornadoClient._audio_container(url) == (extension, muxer) @pytest.mark.asyncio -async def test_unwritable_destination_is_a_tornado_error(tmp_path): - with pytest.raises(TornadoError, match="Writing"): - await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "missing" / "a"), submission="gen-1") +async def test_source_file_is_removed_after_extraction(tmp_path): + await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1") + assert [p.name for p in tmp_path.iterdir()] == ["a"] @pytest.mark.asyncio -async def test_metadata_that_isnt_an_object_is_a_tornado_error(): +async def test_failed_extraction_fails_the_download_and_keeps_tornados_file(tmp_path): + fake = FakeTornado() + client = make_client(fake) + + def demux(source, dest, muxer): + raise TornadoError("no audio track") + + client._demux = demux + with pytest.raises(TornadoError, match="no audio track"): + await client.download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1") + assert fake.of("DELETE", "/jobs/job-1/file") == [] + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_signed_delivery_urls_stay_out_of_the_logs(tmp_path, caplog): + with caplog.at_level(logging.INFO, logger="httpx"): + await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1") + assert "HTTP Request" in caplog.text # other requests still log + assert "verify=" not in caplog.text + + +def _packet_md5(path): + return subprocess.run( + ["ffmpeg", "-v", "error", "-i", str(path), "-map", "0:a", "-c", "copy", "-f", "md5", "-"], + capture_output=True, text=True, check=True, + ).stdout + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg") +def test_demux_keeps_the_audio_packets_and_drops_the_video(tmp_path): + source = tmp_path / "source.mkv" + made = subprocess.run( + ["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "sine=duration=1", "-f", "lavfi", "-i", + "color=size=64x64:duration=1", "-c:a", "aac", "-c:v", "mpeg4", str(source)], + capture_output=True, + ) + if made.returncode: + pytest.skip("this ffmpeg can't make the fixture") + dest = tmp_path / "audio" + _demux_audio(str(source), str(dest), "ipod") + streams = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "stream=codec_type", "-of", "csv=p=0", str(dest)], + capture_output=True, text=True, check=True, + ).stdout.split() + assert streams == ["audio"] + assert _packet_md5(dest) == _packet_md5(source) + + +def test_demux_failure_is_a_tornado_error_and_leaves_nothing(tmp_path): + source = tmp_path / "source" + source.write_bytes(b"not media") + dest = tmp_path / "audio" with pytest.raises(TornadoError): - await make_client(FakeTornado(metadata=httpx.Response(200, json=[]))).get_metadata(VIDEO_ID) + _demux_audio(str(source), str(dest), "webm") + assert not dest.exists() diff --git a/pyproject.toml b/pyproject.toml index 9ed0c60a..d166768b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.291.1" +version = "0.291.2" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"