fix(youtube): get source Opus from Tornado reliably; keep signed URLs out of logs (0.291.2) (#1253)

The first full pipeline run with Tornado (job 586644dd) delivered 128k
AAC rather than Opus. `audio_only` + `format: opus`, which #1247 relied
on, is unreliable on our plan: `Tk3ty5ansqs` returned Opus on 11-03 and
AAC on 11-05, and 4 of 5 videos returned AAC with a `reencoding_notice`.

Alert:
https://discord.com/channels/1198691101943541770/1265517528608669707/1567241547739570298

## Fix
Request `format: webm, max_resolution: 360` and strip the video with
`ffmpeg -c:a copy`. All 3 videos tried this way carried Opus, and
billing was identical to audio-only on the videos compared (e.g.
21,026,831 bytes for `dQw4w9WgXcQ`).

Checked against a real flacfetch job input (`Tk3ty5ansqs`): same audio
stream MD5, same `.webm` container, 5,531,092 vs 5,529,792 bytes.

Also stops httpx logging signed delivery URLs at INFO. They were
appearing in job logs for 586644dd.

## Testing
- 83 mocked tests pass, including a real ffmpeg demux test (audio
packets unchanged, video dropped) and a failure test.
- Not yet run through the deployed pipeline with this format; that test
follows this merge.

## Left as is
Local CodeRabbit flagged that cancelling during demux doesn't stop the
ffmpeg thread. Skipped: the copy takes under a second, both files are in
a temp directory that gets discarded, and nothing else reads them.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_012RYLVTvqXyL4QyA2EjzQMG

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 5.5 authored and GitHub committed 2026-11-05 05:23:33 +00:00
1 parent 688ca162f8
commit 394def0b53
3 files changed
+136 -31

No files matched your search

+59 -13
View File
@@ -8,9 +8,10 @@ Three things that are easy to get wrong:
- Only ever send a canonical ``watch?v=<id>`` URL. A URL carrying a real playlist id downloads and
bills the whole playlist, returning ``batch_id`` rather than ``job_id``.
- ``format: "opus"`` without ``audio_codec`` returns the source's Opus stream, packet-identical to what
flacfetch delivers (checked 2026-11-03). Adding ``audio_codec: "copy"`` or leaving ``format`` out
returns YouTube's 128k AAC instead.
- Request ``format: "webm", max_resolution: "360"`` and strip the video, because that returns the source's
Opus stream, packet-identical to what flacfetch delivers. ``audio_only`` isn't reliable for this: on our
plan the same request returned Opus or 128k AAC from one day to the next (checked 2026-11-03/05). It
costs no more: audio-only and sub-720p video both bill the same full-source 720p reference.
- Delivered files are only kept for 24 hours and count toward a storage limit, so copy them
straight away and delete them. The delivery URL is signed: never send it the API key or log it.
"""
@@ -19,6 +20,7 @@ import logging
import os
import random
import re
import subprocess
import uuid
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Dict, Optional
@@ -30,6 +32,17 @@ from backend.config import get_settings
logger = logging.getLogger(__name__)
class _NoSignedUrls(logging.Filter):
"""httpx logs every request's full URL at INFO, which would put delivery signatures in our logs."""
def filter(self, record: logging.LogRecord) -> bool:
message = record.getMessage()
return "X-Amz-Signature" not in message and "verify=" not in message
logging.getLogger("httpx").addFilter(_NoSignedUrls())
BASE_URL = "https://api.tornadoapi.io"
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
@@ -38,7 +51,17 @@ _TERMINAL = {"Completed", "Failed", "Warning", "Skipped", "Cancelled", "Cancelle
_CREATE_ATTEMPTS = 3
_MAX_RETRY_AFTER_SECONDS = 30.0
_MAX_FILE_BYTES = 1024 * 1024 * 1024 # audio only; anything this big isn't a song
_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024 # includes a 360p video track; anything this big isn't a song
_DEMUX_TIMEOUT_SECONDS = 120
# Delivered container -> (extension for the audio we keep, ffmpeg muxer). Audio is never re-encoded.
_AUDIO_CONTAINERS = {
".webm": (".webm", "webm"),
".mkv": (".webm", "webm"),
".opus": (".ogg", "ogg"), # .ogg: gen's upload and review allow-lists reject .opus
".ogg": (".ogg", "ogg"),
".mp4": (".m4a", "ipod"),
".m4a": (".m4a", "ipod"),
}
class TornadoError(Exception):
@@ -67,6 +90,7 @@ class TornadoClient:
poll_interval_seconds: float = 3.0,
transport: Optional[httpx.AsyncBaseTransport] = None,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
demux: Optional[Callable[[str, str, str], None]] = None,
clock: Callable[[], float] = lambda: asyncio.get_running_loop().time(),
):
self._api_key = api_key
@@ -75,6 +99,7 @@ class TornadoClient:
self._poll_interval_seconds = poll_interval_seconds
self._transport = transport
self._sleep = sleep
self._demux = demux or _demux_audio
self._clock = clock
def _http(self, timeout: float = 30.0) -> httpx.AsyncClient:
@@ -125,11 +150,17 @@ class TornadoClient:
if not delivery_url:
raise TornadoError(f"Tornado job {job_id} completed without a delivery URL")
await self._fetch_file(delivery_url, dest_path)
extension, muxer = self._audio_container(delivery_url)
source_path = f"{dest_path}.source"
try:
await self._fetch_file(delivery_url, source_path)
await asyncio.to_thread(self._demux, source_path, dest_path, muxer)
finally:
self._remove_partial(source_path)
await self._delete_file(job_id)
return TornadoDownload(
tornado_job_id=job_id,
extension=self._extension(delivery_url),
extension=extension,
file_size=job.get("file_size"),
billable_bytes=job.get("billable_bytes"),
)
@@ -148,7 +179,7 @@ class TornadoClient:
raise TornadoAccountError(f"Tornado rejected the account: HTTP {resp.status_code}")
async def _create_job(self, url: str, idempotency_key: str) -> str:
body = {"url": url, "audio_only": True, "format": "opus"}
body = {"url": url, "format": "webm", "max_resolution": "360"}
headers = {**self._headers(), "x-idempotency-key": idempotency_key}
last_problem = "no attempt made"
retry_after: Optional[str] = None
@@ -280,13 +311,28 @@ class TornadoClient:
logger.info(f"Could not cancel Tornado job {job_id}: {type(e).__name__}")
@staticmethod
def _extension(delivery_url: str) -> str:
# We ask for Opus, which arrives in an Ogg container. Name it .ogg: that's accurate, and
# gen's upload and review allow-lists accept .ogg but not .opus.
def _audio_container(delivery_url: str) -> "tuple[str, str]":
ext = os.path.splitext(urlsplit(delivery_url).path)[1].lower()
if ext == ".opus" or not re.fullmatch(r"\.[a-z0-9]{2,5}", ext):
return ".ogg"
return ext
return _AUDIO_CONTAINERS.get(ext, _AUDIO_CONTAINERS[".webm"])
def _demux_audio(source_path: str, dest_path: str, muxer: str) -> None:
"""Keep only the audio track, without re-encoding it."""
try:
subprocess.run(
["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", source_path,
"-map", "0:a:0", "-c:a", "copy", "-f", muxer, dest_path],
check=True, capture_output=True, text=True, timeout=_DEMUX_TIMEOUT_SECONDS,
)
if not os.path.getsize(dest_path):
raise TornadoError("ffmpeg produced an empty audio file")
except (OSError, subprocess.SubprocessError) as e:
try:
os.remove(dest_path)
except OSError:
pass
stderr = getattr(e, "stderr", None) or ""
raise TornadoError(f"Could not extract the audio track: {type(e).__name__} {stderr[:200]}") from e
_client: Optional[TornadoClient] = None
+76 -17
View File
@@ -1,17 +1,21 @@
"""Tests for TornadoClient. Network is mocked; nothing here submits real work."""
import json
import logging
import shutil
import subprocess
import httpx
import pytest
from backend.services.tornado_client import (
_demux_audio,
TornadoAccountError,
TornadoClient,
TornadoError,
)
VIDEO_ID = "dQw4w9WgXcQ"
DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.opus?verify=signed"
DELIVERY_URL = "https://files.tornadoapi.io/videos/abc.webm?verify=signed"
class FakeTornado:
@@ -64,6 +68,7 @@ def make_client(fake, deadline=30.0):
poll_interval_seconds=3.0,
transport=httpx.MockTransport(fake),
sleep=sleep,
demux=lambda source, dest, muxer: shutil.copy(source, dest),
clock=lambda: now["t"],
)
@@ -75,17 +80,17 @@ async def test_download_writes_file_and_reports_size(tmp_path):
result = await make_client(fake).download_audio(VIDEO_ID, str(dest), submission="gen-1")
assert dest.read_bytes() == b"audio"
assert result.tornado_job_id == "job-1"
assert result.extension == ".ogg"
assert result.extension == ".webm"
assert (result.file_size, result.billable_bytes) == (5, 9)
@pytest.mark.asyncio
async def test_requests_the_source_opus_stream_for_the_canonical_url(tmp_path):
async def test_requests_the_route_that_returns_source_opus_for_the_canonical_url(tmp_path):
fake = FakeTornado()
await make_client(fake).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
body = json.loads(fake.of("POST", "/jobs")[0].content)
# audio_codec must stay out: with "copy" Tornado returned 128k AAC instead of the Opus stream.
assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "audio_only": True, "format": "opus"}
# Not audio_only: that returned 128k AAC or Opus unpredictably (2026-11-05).
assert body == {"url": f"https://www.youtube.com/watch?v={VIDEO_ID}", "format": "webm", "max_resolution": "360"}
@pytest.mark.asyncio
@@ -288,25 +293,79 @@ async def test_metadata_success_and_failure():
@pytest.mark.parametrize(
"url,expected",
"url,extension,muxer",
[
("https://files.tornadoapi.io/videos/abc.opus?verify=x", ".ogg"),
("https://files.tornadoapi.io/videos/abc.ogg", ".ogg"),
("https://files.tornadoapi.io/videos/abc", ".ogg"),
("https://files.tornadoapi.io/videos/abc.m4a", ".m4a"),
("https://files.tornadoapi.io/videos/abc.webm?verify=x", ".webm", "webm"),
("https://fsn1.your-objectstorage.com/tornado/videos/abc.opus?X-Amz-Signature=x", ".ogg", "ogg"),
("https://files.tornadoapi.io/videos/abc.mp4", ".m4a", "ipod"),
("https://files.tornadoapi.io/videos/abc.m4a", ".m4a", "ipod"),
("https://files.tornadoapi.io/videos/abc", ".webm", "webm"),
],
)
def test_extension_comes_from_the_delivery_path(url, expected):
assert TornadoClient._extension(url) == expected
def test_audio_container_comes_from_the_delivery_path(url, extension, muxer):
assert TornadoClient._audio_container(url) == (extension, muxer)
@pytest.mark.asyncio
async def test_unwritable_destination_is_a_tornado_error(tmp_path):
with pytest.raises(TornadoError, match="Writing"):
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "missing" / "a"), submission="gen-1")
async def test_source_file_is_removed_after_extraction(tmp_path):
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
assert [p.name for p in tmp_path.iterdir()] == ["a"]
@pytest.mark.asyncio
async def test_metadata_that_isnt_an_object_is_a_tornado_error():
async def test_failed_extraction_fails_the_download_and_keeps_tornados_file(tmp_path):
fake = FakeTornado()
client = make_client(fake)
def demux(source, dest, muxer):
raise TornadoError("no audio track")
client._demux = demux
with pytest.raises(TornadoError, match="no audio track"):
await client.download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
assert fake.of("DELETE", "/jobs/job-1/file") == []
assert list(tmp_path.iterdir()) == []
@pytest.mark.asyncio
async def test_signed_delivery_urls_stay_out_of_the_logs(tmp_path, caplog):
with caplog.at_level(logging.INFO, logger="httpx"):
await make_client(FakeTornado()).download_audio(VIDEO_ID, str(tmp_path / "a"), submission="gen-1")
assert "HTTP Request" in caplog.text # other requests still log
assert "verify=" not in caplog.text
def _packet_md5(path):
return subprocess.run(
["ffmpeg", "-v", "error", "-i", str(path), "-map", "0:a", "-c", "copy", "-f", "md5", "-"],
capture_output=True, text=True, check=True,
).stdout
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
def test_demux_keeps_the_audio_packets_and_drops_the_video(tmp_path):
source = tmp_path / "source.mkv"
made = subprocess.run(
["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "sine=duration=1", "-f", "lavfi", "-i",
"color=size=64x64:duration=1", "-c:a", "aac", "-c:v", "mpeg4", str(source)],
capture_output=True,
)
if made.returncode:
pytest.skip("this ffmpeg can't make the fixture")
dest = tmp_path / "audio"
_demux_audio(str(source), str(dest), "ipod")
streams = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type", "-of", "csv=p=0", str(dest)],
capture_output=True, text=True, check=True,
).stdout.split()
assert streams == ["audio"]
assert _packet_md5(dest) == _packet_md5(source)
def test_demux_failure_is_a_tornado_error_and_leaves_nothing(tmp_path):
source = tmp_path / "source"
source.write_bytes(b"not media")
dest = tmp_path / "audio"
with pytest.raises(TornadoError):
await make_client(FakeTornado(metadata=httpx.Response(200, json=[]))).get_metadata(VIDEO_ID)
_demux_audio(str(source), str(dest), "webm")
assert not dest.exists()
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.291.1"
version = "0.291.2"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"