mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(youtube): comment bot survives long runs (Broken pipe on first prod run) (v0.280.3) (#1208)
## Summary The first prod run of the YouTube comment bot (#1205/#1207) posted 2 correct replies and then died with `BrokenPipeError`. **Cause:** `commentThreads.list` pages were fetched lazily between slow per-thread work. Background tasks on Cloud Run are CPU-throttled, so Gemini classification of a 100-thread page took about 10 minutes. The shared httplib2 connection idled out and the next page fetch broke. **Fix:** - **Reads:** all comment pages are fetched up front, and reads retry (`num_retries=3`, which reconnects). If a page still fails, the threads already fetched are processed, the run is marked incomplete (high-water not advanced), and #alerts gets a throttled message. - **Replies:** each post uses a freshly built client, built **before** the comment is claimed, so a client failure leaves it unclaimed. Posts are never retried, because a lost response could mean a double reply. Re-running is safe: the 2 posted replies have action docs, and the failed run saved no state, so the one-time backfill start still applies. ## Testing - `backend/tests/test_youtube_comment_bot.py`: reads use `num_retries`, posts don't. - `backend/tests/test_youtube_comment_bot_worker.py`: a later page failure still processes the fetched threads and alerts; a posting-client failure leaves the comment unclaimed. - 55 bot tests pass. Local CodeRabbit: 2 cycles, all findings fixed. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_016E3ibN2EcKxnGQvEf6T4V8 --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
055a5d3f73
commit
032ac11a45
6 files changed
+82
-12
No files matched your search
@@ -469,6 +469,11 @@ def decide(
|
||||
# ---------------------------------------------------------------------------
|
||||
# YouTube API helpers (shared by the replay script and the worker)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reads are idempotent, so retry them: a long run idles the shared HTTP connection
|
||||
# between calls (Gemini work in between) and Google drops it ("Broken pipe").
|
||||
# Writes (post_reply) are NOT retried: a lost response could mean a double reply.
|
||||
READ_RETRIES = 3
|
||||
|
||||
def iter_comment_threads(youtube, channel_id: str = NOMAD_CHANNEL_ID,
|
||||
stop_before: Optional[datetime] = None, max_pages: int = 500):
|
||||
"""Yield ``commentThreads.list`` items for the whole channel, newest first.
|
||||
@@ -482,7 +487,7 @@ def iter_comment_threads(youtube, channel_id: str = NOMAD_CHANNEL_ID,
|
||||
youtube.commentThreads()
|
||||
.list(part="snippet,replies", allThreadsRelatedToChannelId=channel_id,
|
||||
order="time", maxResults=100, textFormat="plainText", pageToken=token)
|
||||
.execute()
|
||||
.execute(num_retries=READ_RETRIES)
|
||||
)
|
||||
items = resp.get("items", [])
|
||||
reached_end = False
|
||||
@@ -507,7 +512,7 @@ def fetch_owner_replies(youtube, parent_id: str, channel_id: str = NOMAD_CHANNEL
|
||||
youtube.comments()
|
||||
.list(part="snippet", parentId=parent_id, maxResults=100,
|
||||
textFormat="plainText", pageToken=token)
|
||||
.execute()
|
||||
.execute(num_retries=READ_RETRIES)
|
||||
)
|
||||
for c in resp.get("items", []):
|
||||
s = c.get("snippet", {})
|
||||
@@ -528,7 +533,9 @@ def load_thread(youtube, item: Dict, channel_id: str = NOMAD_CHANNEL_ID) -> Comm
|
||||
|
||||
|
||||
def post_reply(youtube, parent_comment_id: str, text: str) -> str:
|
||||
"""Post a reply as the channel (needs the youtube.force-ssl scope; 50 units)."""
|
||||
"""Post a reply as the channel (needs the youtube.force-ssl scope; 50 units).
|
||||
|
||||
Pass a freshly built client in long runs (see READ_RETRIES): never retried."""
|
||||
resp = (
|
||||
youtube.comments()
|
||||
.insert(part="snippet", body={"snippet": {"parentId": parent_comment_id, "textOriginal": text}})
|
||||
|
||||
@@ -261,6 +261,7 @@ class FakeYouTube:
|
||||
self.pages = pages
|
||||
self.reply_pages = reply_pages or []
|
||||
self.inserted = []
|
||||
self.retries = []
|
||||
|
||||
def commentThreads(self):
|
||||
return self
|
||||
@@ -277,7 +278,8 @@ class FakeYouTube:
|
||||
self._kw = {"insert": True}
|
||||
return self
|
||||
|
||||
def execute(self):
|
||||
def execute(self, num_retries=0):
|
||||
self.retries.append((("insert" if self._kw.get("insert") else "list"), num_retries))
|
||||
if self._kw.get("insert"):
|
||||
return {"id": "reply1"}
|
||||
if "parentId" in self._kw:
|
||||
@@ -305,3 +307,11 @@ class TestApiHelpers:
|
||||
yt = FakeYouTube([])
|
||||
assert bot.post_reply(yt, "c1", "hi") == "reply1"
|
||||
assert yt.inserted[0]["body"]["snippet"] == {"parentId": "c1", "textOriginal": "hi"}
|
||||
|
||||
|
||||
def test_reads_retry_but_posts_never_do(self):
|
||||
yt = FakeYouTube([{"items": [_item()]}], reply_pages=[{"items": []}])
|
||||
list(bot.iter_comment_threads(yt))
|
||||
bot.load_thread(yt, _item(replies=[("UCa", "x")], total_replies=7))
|
||||
bot.post_reply(yt, "c1", "hi")
|
||||
assert yt.retries == [("list", bot.READ_RETRIES), ("list", bot.READ_RETRIES), ("insert", 0)]
|
||||
@@ -71,7 +71,7 @@ class FakeYouTube:
|
||||
self._kw = {"insert": True}
|
||||
return self
|
||||
|
||||
def execute(self):
|
||||
def execute(self, num_retries=0):
|
||||
if self._kw.get("insert"):
|
||||
return {"id": f"reply{len(self.inserted)}"}
|
||||
return {"items": self.items}
|
||||
@@ -236,3 +236,28 @@ def test_post_noise_disables_discord_mentions(monkeypatch):
|
||||
monkeypatch.setattr(w.requests, "post", lambda url, json, timeout: sent.append(json) or SimpleNamespace(ok=True))
|
||||
w._post_noise(SimpleNamespace(get_secret=lambda name: "https://discord.test/hook"), "@everyone hi")
|
||||
assert sent[0]["allowed_mentions"] == {"parse": []}
|
||||
|
||||
|
||||
def test_failed_later_page_still_processes_fetched_threads(env, monkeypatch):
|
||||
def pages(youtube, stop_before=None):
|
||||
yield _item("c1")
|
||||
raise BrokenPipeError(32, "Broken pipe")
|
||||
monkeypatch.setattr(w.bot, "iter_comment_threads", pages)
|
||||
summary = w.run_comment_bot_sync()
|
||||
assert summary["replied"] == 1
|
||||
assert env.db.store[(w.STATE_COLLECTION, w.STATE_DOC)]["incomplete"] is True
|
||||
w._alert.assert_called_once()
|
||||
|
||||
|
||||
def test_poster_build_failure_leaves_comment_unclaimed(env, monkeypatch):
|
||||
calls = {"n": 0}
|
||||
yt = env.yt
|
||||
|
||||
def build(creds):
|
||||
calls["n"] += 1
|
||||
if calls["n"] > 1: # first call is the reading client
|
||||
raise RuntimeError("token refresh failed")
|
||||
return yt
|
||||
monkeypatch.setattr(w.ytb, "build_youtube", build)
|
||||
summary = w.run_comment_bot_sync()
|
||||
assert summary["errors"] == 1 and "c1" not in _actions(env.db) and yt.inserted == []
|
||||
@@ -174,14 +174,28 @@ def run_comment_bot_sync(dry_run: bool = False, max_replies: Optional[int] = Non
|
||||
generate = bot.gemini_generate()
|
||||
quota = get_youtube_quota_service()
|
||||
|
||||
# A run that stops early (reply cap, quota, a transient classifier error, a failed
|
||||
# page fetch) must not advance the high-water mark, or older unprocessed threads
|
||||
# would fall out of the next run's window. Re-reading is cheap: handled comments
|
||||
# have an action doc.
|
||||
incomplete = False
|
||||
summary = {"status": "ok", "dry_run": dry_run, "threads": 0, "replied": 0, "skipped": {},
|
||||
"errors": 0, "actions": []}
|
||||
newest = high_water
|
||||
# A run that stops early (reply cap, quota, a transient classifier error) must not
|
||||
# advance the high-water mark, or older unprocessed threads would fall out of the
|
||||
# next run's window. Re-reading is cheap: handled comments have an action doc.
|
||||
incomplete = False
|
||||
for item in bot.iter_comment_threads(youtube, stop_before=stop_before):
|
||||
# Fetch every page up front (a few quick reads) rather than lazily between slow
|
||||
# per-thread work, which idles the connection until Google drops it. If a page
|
||||
# still fails after retries, process what we have and retry the rest next run.
|
||||
items = []
|
||||
try:
|
||||
for item in bot.iter_comment_threads(youtube, stop_before=stop_before):
|
||||
items.append(item)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
incomplete = True
|
||||
logger.warning("YOUTUBE_COMMENT_BOT: comment fetch failed after %d threads: %s", len(items), exc)
|
||||
_alert("fetch", f"YouTube comment bot: reading channel comments failed after {len(items)} threads "
|
||||
f"({exc}). {'Processing those; older' if items else 'No replies this run;'} "
|
||||
"comments are retried next run (high-water not advanced).")
|
||||
for item in items:
|
||||
thread = bot.load_thread(youtube, item)
|
||||
summary["threads"] += 1
|
||||
if thread.published and thread.published > newest:
|
||||
@@ -219,11 +233,21 @@ def run_comment_bot_sync(dry_run: bool = False, max_replies: Optional[int] = Non
|
||||
logger.warning("YOUTUBE_COMMENT_BOT: quota low (%s left); stopping", remaining)
|
||||
incomplete = True
|
||||
break
|
||||
# Fresh client per post: the shared connection may have idled out, and a post
|
||||
# must never be retried (double-reply risk). Built before claiming, so a
|
||||
# failure here leaves the comment unclaimed for the next run.
|
||||
try:
|
||||
poster = ytb.build_youtube(creds)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
incomplete = True
|
||||
summary["errors"] += 1
|
||||
logger.warning("YOUTUBE_COMMENT_BOT: couldn't build posting client: %s", exc)
|
||||
break
|
||||
# Claim the comment BEFORE posting so a concurrent run can't double-reply.
|
||||
if not _create(action_ref, {**record, "action": "replying"}):
|
||||
continue
|
||||
try:
|
||||
reply_id = bot.post_reply(youtube, thread.comment_id, decision.reply_text)
|
||||
reply_id = bot.post_reply(poster, thread.comment_id, decision.reply_text)
|
||||
quota.record_upload(f"comment-reply:{thread.comment_id}", units=REPLY_COST)
|
||||
action_ref.update({"action": "replied", "reply_id": reply_id, "replied_at": datetime.now(timezone.utc)})
|
||||
summary["replied"] += 1
|
||||
|
||||
@@ -171,6 +171,10 @@ anonymous YouTuber to sign in with an email; follow-up emails do the gen convers
|
||||
- `..._MAX_REPLIES` (20 per run) and `..._MIN_CONFIDENCE` (0.8).
|
||||
- Notifications: each reply posts a one-liner to Discord **#noise**. Missing credentials,
|
||||
missing scope, YouTube errors and failed replies go to **#alerts**, throttled to 6h.
|
||||
- Long runs: all comment pages are fetched up front and reads retry (`num_retries`).
|
||||
Each post uses a freshly built client and is never retried. Without this, the slow
|
||||
Gemini work between calls (background tasks are CPU-throttled) idled the shared
|
||||
connection until Google dropped it (`BrokenPipeError`, first prod run 2026-10-22).
|
||||
- Quota: `commentThreads.list` costs 1 unit per page and each reply costs 50, recorded through
|
||||
`YouTubeQuotaService` as `comment-reply:<comment_id>`.
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.280.2"
|
||||
version = "0.280.3"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user