diff --git a/backend/services/youtube_comment_bot.py b/backend/services/youtube_comment_bot.py index e3734bd2..586aaa18 100644 --- a/backend/services/youtube_comment_bot.py +++ b/backend/services/youtube_comment_bot.py @@ -469,6 +469,11 @@ def decide( # --------------------------------------------------------------------------- # YouTube API helpers (shared by the replay script and the worker) # --------------------------------------------------------------------------- +# Reads are idempotent, so retry them: a long run idles the shared HTTP connection +# between calls (Gemini work in between) and Google drops it ("Broken pipe"). +# Writes (post_reply) are NOT retried: a lost response could mean a double reply. +READ_RETRIES = 3 + def iter_comment_threads(youtube, channel_id: str = NOMAD_CHANNEL_ID, stop_before: Optional[datetime] = None, max_pages: int = 500): """Yield ``commentThreads.list`` items for the whole channel, newest first. @@ -482,7 +487,7 @@ def iter_comment_threads(youtube, channel_id: str = NOMAD_CHANNEL_ID, youtube.commentThreads() .list(part="snippet,replies", allThreadsRelatedToChannelId=channel_id, order="time", maxResults=100, textFormat="plainText", pageToken=token) - .execute() + .execute(num_retries=READ_RETRIES) ) items = resp.get("items", []) reached_end = False @@ -507,7 +512,7 @@ def fetch_owner_replies(youtube, parent_id: str, channel_id: str = NOMAD_CHANNEL youtube.comments() .list(part="snippet", parentId=parent_id, maxResults=100, textFormat="plainText", pageToken=token) - .execute() + .execute(num_retries=READ_RETRIES) ) for c in resp.get("items", []): s = c.get("snippet", {}) @@ -528,7 +533,9 @@ def load_thread(youtube, item: Dict, channel_id: str = NOMAD_CHANNEL_ID) -> Comm def post_reply(youtube, parent_comment_id: str, text: str) -> str: - """Post a reply as the channel (needs the youtube.force-ssl scope; 50 units).""" + """Post a reply as the channel (needs the youtube.force-ssl scope; 50 units). + + Pass a freshly built client in long runs (see READ_RETRIES): never retried.""" resp = ( youtube.comments() .insert(part="snippet", body={"snippet": {"parentId": parent_comment_id, "textOriginal": text}}) diff --git a/backend/tests/test_youtube_comment_bot.py b/backend/tests/test_youtube_comment_bot.py index 4b43c1c4..f88d7e4c 100644 --- a/backend/tests/test_youtube_comment_bot.py +++ b/backend/tests/test_youtube_comment_bot.py @@ -261,6 +261,7 @@ class FakeYouTube: self.pages = pages self.reply_pages = reply_pages or [] self.inserted = [] + self.retries = [] def commentThreads(self): return self @@ -277,7 +278,8 @@ class FakeYouTube: self._kw = {"insert": True} return self - def execute(self): + def execute(self, num_retries=0): + self.retries.append((("insert" if self._kw.get("insert") else "list"), num_retries)) if self._kw.get("insert"): return {"id": "reply1"} if "parentId" in self._kw: @@ -305,3 +307,11 @@ class TestApiHelpers: yt = FakeYouTube([]) assert bot.post_reply(yt, "c1", "hi") == "reply1" assert yt.inserted[0]["body"]["snippet"] == {"parentId": "c1", "textOriginal": "hi"} + + + def test_reads_retry_but_posts_never_do(self): + yt = FakeYouTube([{"items": [_item()]}], reply_pages=[{"items": []}]) + list(bot.iter_comment_threads(yt)) + bot.load_thread(yt, _item(replies=[("UCa", "x")], total_replies=7)) + bot.post_reply(yt, "c1", "hi") + assert yt.retries == [("list", bot.READ_RETRIES), ("list", bot.READ_RETRIES), ("insert", 0)] diff --git a/backend/tests/test_youtube_comment_bot_worker.py b/backend/tests/test_youtube_comment_bot_worker.py index 9cbabdc9..0e730dc4 100644 --- a/backend/tests/test_youtube_comment_bot_worker.py +++ b/backend/tests/test_youtube_comment_bot_worker.py @@ -71,7 +71,7 @@ class FakeYouTube: self._kw = {"insert": True} return self - def execute(self): + def execute(self, num_retries=0): if self._kw.get("insert"): return {"id": f"reply{len(self.inserted)}"} return {"items": self.items} @@ -236,3 +236,28 @@ def test_post_noise_disables_discord_mentions(monkeypatch): monkeypatch.setattr(w.requests, "post", lambda url, json, timeout: sent.append(json) or SimpleNamespace(ok=True)) w._post_noise(SimpleNamespace(get_secret=lambda name: "https://discord.test/hook"), "@everyone hi") assert sent[0]["allowed_mentions"] == {"parse": []} + + +def test_failed_later_page_still_processes_fetched_threads(env, monkeypatch): + def pages(youtube, stop_before=None): + yield _item("c1") + raise BrokenPipeError(32, "Broken pipe") + monkeypatch.setattr(w.bot, "iter_comment_threads", pages) + summary = w.run_comment_bot_sync() + assert summary["replied"] == 1 + assert env.db.store[(w.STATE_COLLECTION, w.STATE_DOC)]["incomplete"] is True + w._alert.assert_called_once() + + +def test_poster_build_failure_leaves_comment_unclaimed(env, monkeypatch): + calls = {"n": 0} + yt = env.yt + + def build(creds): + calls["n"] += 1 + if calls["n"] > 1: # first call is the reading client + raise RuntimeError("token refresh failed") + return yt + monkeypatch.setattr(w.ytb, "build_youtube", build) + summary = w.run_comment_bot_sync() + assert summary["errors"] == 1 and "c1" not in _actions(env.db) and yt.inserted == [] diff --git a/backend/workers/youtube_comment_bot_worker.py b/backend/workers/youtube_comment_bot_worker.py index f344eb61..3bd99343 100644 --- a/backend/workers/youtube_comment_bot_worker.py +++ b/backend/workers/youtube_comment_bot_worker.py @@ -174,14 +174,28 @@ def run_comment_bot_sync(dry_run: bool = False, max_replies: Optional[int] = Non generate = bot.gemini_generate() quota = get_youtube_quota_service() + # A run that stops early (reply cap, quota, a transient classifier error, a failed + # page fetch) must not advance the high-water mark, or older unprocessed threads + # would fall out of the next run's window. Re-reading is cheap: handled comments + # have an action doc. + incomplete = False summary = {"status": "ok", "dry_run": dry_run, "threads": 0, "replied": 0, "skipped": {}, "errors": 0, "actions": []} newest = high_water - # A run that stops early (reply cap, quota, a transient classifier error) must not - # advance the high-water mark, or older unprocessed threads would fall out of the - # next run's window. Re-reading is cheap: handled comments have an action doc. - incomplete = False - for item in bot.iter_comment_threads(youtube, stop_before=stop_before): + # Fetch every page up front (a few quick reads) rather than lazily between slow + # per-thread work, which idles the connection until Google drops it. If a page + # still fails after retries, process what we have and retry the rest next run. + items = [] + try: + for item in bot.iter_comment_threads(youtube, stop_before=stop_before): + items.append(item) + except Exception as exc: # noqa: BLE001 + incomplete = True + logger.warning("YOUTUBE_COMMENT_BOT: comment fetch failed after %d threads: %s", len(items), exc) + _alert("fetch", f"YouTube comment bot: reading channel comments failed after {len(items)} threads " + f"({exc}). {'Processing those; older' if items else 'No replies this run;'} " + "comments are retried next run (high-water not advanced).") + for item in items: thread = bot.load_thread(youtube, item) summary["threads"] += 1 if thread.published and thread.published > newest: @@ -219,11 +233,21 @@ def run_comment_bot_sync(dry_run: bool = False, max_replies: Optional[int] = Non logger.warning("YOUTUBE_COMMENT_BOT: quota low (%s left); stopping", remaining) incomplete = True break + # Fresh client per post: the shared connection may have idled out, and a post + # must never be retried (double-reply risk). Built before claiming, so a + # failure here leaves the comment unclaimed for the next run. + try: + poster = ytb.build_youtube(creds) + except Exception as exc: # noqa: BLE001 + incomplete = True + summary["errors"] += 1 + logger.warning("YOUTUBE_COMMENT_BOT: couldn't build posting client: %s", exc) + break # Claim the comment BEFORE posting so a concurrent run can't double-reply. if not _create(action_ref, {**record, "action": "replying"}): continue try: - reply_id = bot.post_reply(youtube, thread.comment_id, decision.reply_text) + reply_id = bot.post_reply(poster, thread.comment_id, decision.reply_text) quota.record_upload(f"comment-reply:{thread.comment_id}", units=REPLY_COST) action_ref.update({"action": "replied", "reply_id": reply_id, "replied_at": datetime.now(timezone.utc)}) summary["replied"] += 1 diff --git a/docs/REQUESTS-BOARD.md b/docs/REQUESTS-BOARD.md index dd541937..051ed1fe 100644 --- a/docs/REQUESTS-BOARD.md +++ b/docs/REQUESTS-BOARD.md @@ -171,6 +171,10 @@ anonymous YouTuber to sign in with an email; follow-up emails do the gen convers - `..._MAX_REPLIES` (20 per run) and `..._MIN_CONFIDENCE` (0.8). - Notifications: each reply posts a one-liner to Discord **#noise**. Missing credentials, missing scope, YouTube errors and failed replies go to **#alerts**, throttled to 6h. +- Long runs: all comment pages are fetched up front and reads retry (`num_retries`). + Each post uses a freshly built client and is never retried. Without this, the slow + Gemini work between calls (background tasks are CPU-throttled) idled the shared + connection until Google dropped it (`BrokenPipeError`, first prod run 2026-10-22). - Quota: `commentThreads.list` costs 1 unit per page and each reply costs 50, recorded through `YouTubeQuotaService` as `comment-reply:`. diff --git a/pyproject.toml b/pyproject.toml index b7a88211..110f1ce3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.280.2" +version = "0.280.3" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"