fix(youtube): comment bot survives long runs (Broken pipe on first prod run) (v0.280.3) (#1208)

## Summary

The first prod run of the YouTube comment bot (#1205/#1207) posted 2
correct replies and then died with `BrokenPipeError`.

**Cause:** `commentThreads.list` pages were fetched lazily between slow
per-thread work. Background tasks on Cloud Run are CPU-throttled, so
Gemini classification of a 100-thread page took about 10 minutes. The
shared httplib2 connection idled out and the next page fetch broke.

**Fix:**
- **Reads:** all comment pages are fetched up front, and reads retry
(`num_retries=3`, which reconnects). If a page still fails, the threads
already fetched are processed, the run is marked incomplete (high-water
not advanced), and #alerts gets a throttled message.
- **Replies:** each post uses a freshly built client, built **before**
the comment is claimed, so a client failure leaves it unclaimed. Posts
are never retried, because a lost response could mean a double reply.

Re-running is safe: the 2 posted replies have action docs, and the
failed run saved no state, so the one-time backfill start still applies.

## Testing

- `backend/tests/test_youtube_comment_bot.py`: reads use `num_retries`,
posts don't.
- `backend/tests/test_youtube_comment_bot_worker.py`: a later page
failure still processes the fetched threads and alerts; a posting-client
failure leaves the comment unclaimed.
- 55 bot tests pass. Local CodeRabbit: 2 cycles, all findings fixed.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_016E3ibN2EcKxnGQvEf6T4V8

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 5.5 authored and GitHub committed 2026-10-22 03:41:20 -04:00
1 parent 055a5d3f73
commit 032ac11a45
6 files changed
+82 -12

No files matched your search

+10 -3
View File
@@ -469,6 +469,11 @@ def decide(
# ---------------------------------------------------------------------------
# YouTube API helpers (shared by the replay script and the worker)
# ---------------------------------------------------------------------------
# Reads are idempotent, so retry them: a long run idles the shared HTTP connection
# between calls (Gemini work in between) and Google drops it ("Broken pipe").
# Writes (post_reply) are NOT retried: a lost response could mean a double reply.
READ_RETRIES = 3
def iter_comment_threads(youtube, channel_id: str = NOMAD_CHANNEL_ID,
stop_before: Optional[datetime] = None, max_pages: int = 500):
"""Yield ``commentThreads.list`` items for the whole channel, newest first.
@@ -482,7 +487,7 @@ def iter_comment_threads(youtube, channel_id: str = NOMAD_CHANNEL_ID,
youtube.commentThreads()
.list(part="snippet,replies", allThreadsRelatedToChannelId=channel_id,
order="time", maxResults=100, textFormat="plainText", pageToken=token)
.execute()
.execute(num_retries=READ_RETRIES)
)
items = resp.get("items", [])
reached_end = False
@@ -507,7 +512,7 @@ def fetch_owner_replies(youtube, parent_id: str, channel_id: str = NOMAD_CHANNEL
youtube.comments()
.list(part="snippet", parentId=parent_id, maxResults=100,
textFormat="plainText", pageToken=token)
.execute()
.execute(num_retries=READ_RETRIES)
)
for c in resp.get("items", []):
s = c.get("snippet", {})
@@ -528,7 +533,9 @@ def load_thread(youtube, item: Dict, channel_id: str = NOMAD_CHANNEL_ID) -> Comm
def post_reply(youtube, parent_comment_id: str, text: str) -> str:
"""Post a reply as the channel (needs the youtube.force-ssl scope; 50 units)."""
"""Post a reply as the channel (needs the youtube.force-ssl scope; 50 units).
Pass a freshly built client in long runs (see READ_RETRIES): never retried."""
resp = (
youtube.comments()
.insert(part="snippet", body={"snippet": {"parentId": parent_comment_id, "textOriginal": text}})
+11 -1
View File
@@ -261,6 +261,7 @@ class FakeYouTube:
self.pages = pages
self.reply_pages = reply_pages or []
self.inserted = []
self.retries = []
def commentThreads(self):
return self
@@ -277,7 +278,8 @@ class FakeYouTube:
self._kw = {"insert": True}
return self
def execute(self):
def execute(self, num_retries=0):
self.retries.append((("insert" if self._kw.get("insert") else "list"), num_retries))
if self._kw.get("insert"):
return {"id": "reply1"}
if "parentId" in self._kw:
@@ -305,3 +307,11 @@ class TestApiHelpers:
yt = FakeYouTube([])
assert bot.post_reply(yt, "c1", "hi") == "reply1"
assert yt.inserted[0]["body"]["snippet"] == {"parentId": "c1", "textOriginal": "hi"}
def test_reads_retry_but_posts_never_do(self):
yt = FakeYouTube([{"items": [_item()]}], reply_pages=[{"items": []}])
list(bot.iter_comment_threads(yt))
bot.load_thread(yt, _item(replies=[("UCa", "x")], total_replies=7))
bot.post_reply(yt, "c1", "hi")
assert yt.retries == [("list", bot.READ_RETRIES), ("list", bot.READ_RETRIES), ("insert", 0)]
@@ -71,7 +71,7 @@ class FakeYouTube:
self._kw = {"insert": True}
return self
def execute(self):
def execute(self, num_retries=0):
if self._kw.get("insert"):
return {"id": f"reply{len(self.inserted)}"}
return {"items": self.items}
@@ -236,3 +236,28 @@ def test_post_noise_disables_discord_mentions(monkeypatch):
monkeypatch.setattr(w.requests, "post", lambda url, json, timeout: sent.append(json) or SimpleNamespace(ok=True))
w._post_noise(SimpleNamespace(get_secret=lambda name: "https://discord.test/hook"), "@everyone hi")
assert sent[0]["allowed_mentions"] == {"parse": []}
def test_failed_later_page_still_processes_fetched_threads(env, monkeypatch):
def pages(youtube, stop_before=None):
yield _item("c1")
raise BrokenPipeError(32, "Broken pipe")
monkeypatch.setattr(w.bot, "iter_comment_threads", pages)
summary = w.run_comment_bot_sync()
assert summary["replied"] == 1
assert env.db.store[(w.STATE_COLLECTION, w.STATE_DOC)]["incomplete"] is True
w._alert.assert_called_once()
def test_poster_build_failure_leaves_comment_unclaimed(env, monkeypatch):
calls = {"n": 0}
yt = env.yt
def build(creds):
calls["n"] += 1
if calls["n"] > 1: # first call is the reading client
raise RuntimeError("token refresh failed")
return yt
monkeypatch.setattr(w.ytb, "build_youtube", build)
summary = w.run_comment_bot_sync()
assert summary["errors"] == 1 and "c1" not in _actions(env.db) and yt.inserted == []
+30 -6
View File
@@ -174,14 +174,28 @@ def run_comment_bot_sync(dry_run: bool = False, max_replies: Optional[int] = Non
generate = bot.gemini_generate()
quota = get_youtube_quota_service()
# A run that stops early (reply cap, quota, a transient classifier error, a failed
# page fetch) must not advance the high-water mark, or older unprocessed threads
# would fall out of the next run's window. Re-reading is cheap: handled comments
# have an action doc.
incomplete = False
summary = {"status": "ok", "dry_run": dry_run, "threads": 0, "replied": 0, "skipped": {},
"errors": 0, "actions": []}
newest = high_water
# A run that stops early (reply cap, quota, a transient classifier error) must not
# advance the high-water mark, or older unprocessed threads would fall out of the
# next run's window. Re-reading is cheap: handled comments have an action doc.
incomplete = False
for item in bot.iter_comment_threads(youtube, stop_before=stop_before):
# Fetch every page up front (a few quick reads) rather than lazily between slow
# per-thread work, which idles the connection until Google drops it. If a page
# still fails after retries, process what we have and retry the rest next run.
items = []
try:
for item in bot.iter_comment_threads(youtube, stop_before=stop_before):
items.append(item)
except Exception as exc: # noqa: BLE001
incomplete = True
logger.warning("YOUTUBE_COMMENT_BOT: comment fetch failed after %d threads: %s", len(items), exc)
_alert("fetch", f"YouTube comment bot: reading channel comments failed after {len(items)} threads "
f"({exc}). {'Processing those; older' if items else 'No replies this run;'} "
"comments are retried next run (high-water not advanced).")
for item in items:
thread = bot.load_thread(youtube, item)
summary["threads"] += 1
if thread.published and thread.published > newest:
@@ -219,11 +233,21 @@ def run_comment_bot_sync(dry_run: bool = False, max_replies: Optional[int] = Non
logger.warning("YOUTUBE_COMMENT_BOT: quota low (%s left); stopping", remaining)
incomplete = True
break
# Fresh client per post: the shared connection may have idled out, and a post
# must never be retried (double-reply risk). Built before claiming, so a
# failure here leaves the comment unclaimed for the next run.
try:
poster = ytb.build_youtube(creds)
except Exception as exc: # noqa: BLE001
incomplete = True
summary["errors"] += 1
logger.warning("YOUTUBE_COMMENT_BOT: couldn't build posting client: %s", exc)
break
# Claim the comment BEFORE posting so a concurrent run can't double-reply.
if not _create(action_ref, {**record, "action": "replying"}):
continue
try:
reply_id = bot.post_reply(youtube, thread.comment_id, decision.reply_text)
reply_id = bot.post_reply(poster, thread.comment_id, decision.reply_text)
quota.record_upload(f"comment-reply:{thread.comment_id}", units=REPLY_COST)
action_ref.update({"action": "replied", "reply_id": reply_id, "replied_at": datetime.now(timezone.utc)})
summary["replied"] += 1
+4
View File
@@ -171,6 +171,10 @@ anonymous YouTuber to sign in with an email; follow-up emails do the gen convers
- `..._MAX_REPLIES` (20 per run) and `..._MIN_CONFIDENCE` (0.8).
- Notifications: each reply posts a one-liner to Discord **#noise**. Missing credentials,
missing scope, YouTube errors and failed replies go to **#alerts**, throttled to 6h.
- Long runs: all comment pages are fetched up front and reads retry (`num_retries`).
Each post uses a freshly built client and is never retried. Without this, the slow
Gemini work between calls (background tasks are CPU-throttled) idled the shared
connection until Google dropped it (`BrokenPipeError`, first prod run 2026-10-22).
- Quota: `commentThreads.list` costs 1 unit per page and each reply costs 50, recorded through
`YouTubeQuotaService` as `comment-reply:<comment_id>`.
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.280.2"
version = "0.280.3"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"