diff --git a/backend/services/youtube_backfill.py b/backend/services/youtube_backfill.py index c7d3947b..c6c15bef 100644 --- a/backend/services/youtube_backfill.py +++ b/backend/services/youtube_backfill.py @@ -329,7 +329,9 @@ def _chunks(seq: List, n: int): def fetch_snippets(youtube, ids: List[str]) -> Dict[str, Dict]: out: Dict[str, Dict] = {} for batch in _chunks(ids, 50): - resp = youtube.videos().list(part="snippet,status", id=",".join(batch)).execute() + resp = youtube.videos().list( + part="snippet,status,contentDetails", id=",".join(batch) + ).execute() for it in resp.get("items", []): out[it["id"]] = it return out diff --git a/backend/services/youtube_video_catalog.py b/backend/services/youtube_video_catalog.py new file mode 100644 index 00000000..08661a4a --- /dev/null +++ b/backend/services/youtube_video_catalog.py @@ -0,0 +1,244 @@ +""" +Firestore catalogue of every video on the Nomad Karaoke YouTube channel. + +Kept up to date by the daily description backfill worker, which already fetches +every video's snippet/status, so this costs no extra YouTube quota. + +Layout: +- `youtube_videos/{video_id}`: latest metadata plus bookkeeping + (`first_seen_at`, `last_seen_at`, `last_changed_at`, `in_uploads_playlist`, + `deleted`, `deleted_at`). +- `youtube_videos/{video_id}/history/{auto_id}`: one entry per observed change, + `{at, event, changes: {field: {old, new}}, source}`. `event` is `first_seen`, + `changed`, `deleted` or `restored`. `source` is `backfill` when this job's own + description rewrite caused the change, otherwise `observed` (someone else, + e.g. YouTube Studio, or a new upload). + +Videos are NEVER removed: when YouTube stops returning a known video it is +soft-deleted (`deleted=True`) and keeps being checked, so a reappearance is +recorded as `restored`. + +Writes only happen when something changed (plus `last_seen_at`, refreshed at +most once a day per video), so a normal day costs ~1 read per video and a +handful of writes. +""" +import logging +from datetime import datetime, timedelta +from typing import Dict, Iterable, List, Mapping, Optional, Set + +logger = logging.getLogger(__name__) + +COLLECTION = "youtube_videos" +HISTORY_SUBCOLLECTION = "history" +_BATCH_LIMIT = 400 # Firestore allows 500 writes per batch +_READ_CHUNK = 300 +_LAST_SEEN_REFRESH = timedelta(hours=20) + +# Fields whose changes are recorded in history. +TRACKED_FIELDS = ( + "title", + "description", + "tags", + "published_at", + "privacy_status", + "upload_status", + "category_id", + "duration", + "default_language", + "made_for_kids", + "license", + "embeddable", +) + + +def metadata_from_video(video: Dict) -> Dict: + """Flatten a videos.list item into the stored (tracked) metadata.""" + snip = video.get("snippet") or {} + status = video.get("status") or {} + details = video.get("contentDetails") or {} + return { + "title": snip.get("title") or "", + "description": snip.get("description") or "", + "tags": list(snip.get("tags") or []), + "published_at": snip.get("publishedAt"), + "privacy_status": status.get("privacyStatus"), + "upload_status": status.get("uploadStatus"), + "category_id": snip.get("categoryId"), + "duration": details.get("duration"), + "default_language": snip.get("defaultLanguage"), + "made_for_kids": status.get("madeForKids"), + "license": status.get("license"), + "embeddable": status.get("embeddable"), + } + + +def _parsed_fields(entry: Optional[Dict]) -> Dict: + """Backfill classification of the video (informational, not tracked).""" + entry = entry or {} + return { + "artist": entry.get("artist"), + "song_title": entry.get("song_title"), + "brand_code": entry.get("brand_code"), + "kind": entry.get("kind"), + "backfill_target": bool(entry.get("target")), + } + + +def _diff(old: Dict, new: Dict) -> Dict: + return { + f: {"old": old.get(f), "new": new.get(f)} + for f in TRACKED_FIELDS + if old.get(f) != new.get(f) + } + + +def _chunks(seq: List, n: int): + for i in range(0, len(seq), n): + yield seq[i : i + n] + + +class _Writer: + """Batches writes; a group (doc + its history) never spans two commits.""" + + def __init__(self, db): + self.db = db + self.batch = db.batch() + self.pending = 0 + + def group(self, ops): + """ops: list of (ref, data, merge). Committed together, atomically.""" + if self.pending and self.pending + len(ops) > _BATCH_LIMIT: + self._commit() + for ref, data, merge in ops: + self.batch.set(ref, data, merge=merge) + self.pending += len(ops) + + def _commit(self): + self.batch.commit() + self.batch = self.db.batch() + self.pending = 0 + + def flush(self): + if self.pending: + self._commit() + + +def sync_catalog( + db, + videos: Dict[str, Dict], + entries: Iterable[Dict], + listed_ids: Iterable[str], + gone_ids: Iterable[str], + now: datetime, + backfill_pre_update: Optional[Mapping[str, Dict]] = None, +) -> Dict[str, int]: + """Upsert every returned video and soft-delete the gone ones. + + `videos` must reflect the state AFTER this run's own description updates. + `backfill_pre_update` maps each video this run rewrote to its videos.list + item from BEFORE the rewrite: changes up to that point are logged as + `observed`, and the rewrite itself as `backfill`. + """ + coll = db.collection(COLLECTION) + entries_by_id = {e["video_id"]: e for e in entries} + listed: Set[str] = set(listed_ids) + pre_update: Mapping[str, Dict] = backfill_pre_update or {} + gone = [vid for vid in dict.fromkeys(gone_ids) if vid not in videos] + + all_ids = list(videos.keys()) + gone + existing: Dict[str, Dict] = {} + for chunk in _chunks(all_ids, _READ_CHUNK): + for snap in db.get_all([coll.document(vid) for vid in chunk]): + if snap.exists: + existing[snap.id] = snap.to_dict() or {} + + stats = {"new": 0, "changed": 0, "deleted": 0, "restored": 0, "touched": 0} + writer = _Writer(db) + + def _history(vid, event, changes, source): + return ( + coll.document(vid).collection(HISTORY_SUBCOLLECTION).document(), + {"at": now, "event": event, "changes": changes, "source": source}, + False, + ) + + for vid, video in videos.items(): + meta = metadata_from_video(video) + doc = { + "video_id": vid, + **meta, + **_parsed_fields(entries_by_id.get(vid)), + "in_uploads_playlist": vid in listed, + "last_seen_at": now, + "deleted": False, + "deleted_at": None, + } + prev = existing.get(vid) + if prev is None: + doc.update(first_seen_at=now, last_changed_at=now) + ops = [(coll.document(vid), doc, False), _history(vid, "first_seen", {}, "observed")] + if vid in pre_update: + # Rewritten in the same run it was first seen: keep the original. + ours = _diff(metadata_from_video(pre_update[vid]), meta) + if ours: + ops.append(_history(vid, "changed", ours, "backfill")) + writer.group(ops) + stats["new"] += 1 + continue + + if vid in pre_update: + before = metadata_from_video(pre_update[vid]) + observed = _diff(prev, before) + ours = _diff(before, meta) + else: + observed, ours = _diff(prev, meta), {} + changes = {**observed, **ours} + restored = bool(prev.get("deleted")) + bookkeeping_changed = any( + prev.get(k) != doc[k] + for k in ("in_uploads_playlist", "artist", "song_title", "brand_code", "kind", "backfill_target") + ) + last_seen = prev.get("last_seen_at") + stale_seen = not isinstance(last_seen, datetime) or now - last_seen >= _LAST_SEEN_REFRESH + + if changes or restored: + doc["last_changed_at"] = now + ops = [] + if changes or restored or bookkeeping_changed or stale_seen: + ops.append((coll.document(vid), doc, True)) + stats["touched"] += 1 + if restored: + ops.append(_history(vid, "restored", observed, "observed")) + stats["restored"] += 1 + elif observed: + ops.append(_history(vid, "changed", observed, "observed")) + if ours: + ops.append(_history(vid, "changed", ours, "backfill")) + if observed or ours: + stats["changed"] += 1 + if ops: + writer.group(ops) + + for vid in gone: + prev = existing.get(vid) + if prev is None: + # Known ID we never catalogued (e.g. deleted before the catalogue + # existed): keep a stub so the record survives. + writer.group([ + (coll.document(vid), + {"video_id": vid, "deleted": True, "deleted_at": now, + "first_seen_at": now, "last_changed_at": now}, False), + _history(vid, "deleted", {}, "observed"), + ]) + stats["deleted"] += 1 + elif not prev.get("deleted"): + writer.group([ + (coll.document(vid), + {"deleted": True, "deleted_at": now, "last_changed_at": now, + "in_uploads_playlist": False}, True), + _history(vid, "deleted", {}, "observed"), + ]) + stats["deleted"] += 1 + + writer.flush() + return stats diff --git a/backend/tests/test_youtube_description_backfill_worker.py b/backend/tests/test_youtube_description_backfill_worker.py index a60cdf02..d995b80c 100644 --- a/backend/tests/test_youtube_description_backfill_worker.py +++ b/backend/tests/test_youtube_description_backfill_worker.py @@ -125,10 +125,12 @@ class _Ctx: return_value=FakeDB(self.ref, self.known_ref, self.ledger_docs)), patch.object(worker, "get_youtube_quota_service", return_value=self.quota), patch.object(worker, "_email_service", return_value=self.email), + patch.object(worker.catalog, "sync_catalog", return_value={"new": 0}), ] for p in self._patches: p.start() self.enumerate = worker.bf.enumerate_channel + self.sync_catalog = worker.catalog.sync_catalog return self def __exit__(self, *a): @@ -280,13 +282,13 @@ def test_existing_known_ids_are_passed_and_not_reseeded(): assert known_passed == ["k1", "k2"] -def test_known_ids_saved_from_returned_videos(): +def test_known_ids_keep_gone_videos_for_soft_delete(): vids = ["a", "b"] ctx = _Ctx([_entry(v, will_change=False) for v in vids], _videos(vids), gone_ids=["deleted1"]) ctx.known_ref = FakeDocRef({"ids": ["a", "deleted1"]}) with ctx: worker.run_backfill_sync() - assert ctx.known_ref.data["ids"] == ["a", "b"] + assert ctx.known_ref.data["ids"] == ["a", "b", "deleted1"] def test_dry_run_does_not_save_known_ids(): @@ -335,3 +337,39 @@ def test_failed_seed_is_retried_next_run(): worker.run_backfill_sync() assert ctx2.enumerate.call_args.args[1] == ["hidden"] assert ctx2.known_ref.set_calls + + +# ---------------------------------------------------------------------------- +# Video catalogue +# ---------------------------------------------------------------------------- +def test_catalogue_gets_post_update_descriptions_and_backfill_ids(): + vids = ["a", "b"] + entries = [_entry("a"), _entry("b", will_change=False)] + ctx = _Ctx(entries, _videos(vids), gone_ids=["gone1"], listed_ids=["a"]) + with ctx: + result = worker.run_backfill_sync() + kwargs = ctx.sync_catalog.call_args.kwargs + videos_arg = ctx.sync_catalog.call_args.args[1] + assert list(kwargs["backfill_pre_update"]) == ["a"] + assert kwargs["backfill_pre_update"]["a"]["snippet"]["description"] == "old description" + assert kwargs["gone_ids"] == ["gone1"] + assert kwargs["listed_ids"] == ["a"] + assert videos_arg["a"]["snippet"]["description"] != "old description" + assert videos_arg["b"]["snippet"]["description"] == "old description" + assert result["catalog"] == {"new": 0} + + +def test_catalogue_failure_does_not_fail_run(): + ctx = _Ctx([_entry("a")], _videos(["a"])) + with ctx: + ctx.sync_catalog.side_effect = RuntimeError("firestore down") + result = worker.run_backfill_sync() + assert result["status"] == "ok" and result["updated"] == 1 + assert ctx.ref.data["last_run_updated"] == 1 + + +def test_dry_run_skips_catalogue(): + ctx = _Ctx([_entry("a")], _videos(["a"])) + with ctx: + worker.run_backfill_sync(dry_run=True) + ctx.sync_catalog.assert_not_called() diff --git a/backend/tests/test_youtube_video_catalog.py b/backend/tests/test_youtube_video_catalog.py new file mode 100644 index 00000000..7976b6fe --- /dev/null +++ b/backend/tests/test_youtube_video_catalog.py @@ -0,0 +1,284 @@ +"""Tests for the Firestore YouTube video catalogue (soft-delete + change history).""" +import itertools +from datetime import datetime, timedelta, timezone + +from backend.services import youtube_video_catalog as catalog + + +# ---------------------------------------------------------------------------- +# Minimal in-memory Firestore +# ---------------------------------------------------------------------------- +class _Snap: + def __init__(self, doc_id, data): + self.id = doc_id + self.exists = data is not None + self._data = data + + def to_dict(self): + return dict(self._data) if self._data is not None else None + + +class MemDoc: + def __init__(self, store, path): + self.store, self.path = store, path + self.id = path.split("/")[-1] + + def collection(self, name): + return MemColl(self.store, f"{self.path}/{name}") + + def get(self): + return _Snap(self.id, self.store.docs.get(self.path)) + + +class MemColl: + _auto = itertools.count() + + def __init__(self, store, path): + self.store, self.path = store, path + + def document(self, doc_id=None): + doc_id = doc_id or f"auto{next(self._auto):06d}" + return MemDoc(self.store, f"{self.path}/{doc_id}") + + def docs(self): + prefix = self.path + "/" + return { + p[len(prefix):]: d + for p, d in self.store.docs.items() + if p.startswith(prefix) and "/" not in p[len(prefix):] + } + + +class MemBatch: + def __init__(self, store): + self.store, self.ops = store, [] + + def set(self, ref, data, merge=False): + self.ops.append((ref.path, dict(data), merge)) + + def commit(self): + self.store.commits += 1 + for path, data, merge in self.ops: + if merge and path in self.store.docs: + self.store.docs[path] = {**self.store.docs[path], **data} + else: + self.store.docs[path] = data + self.ops = [] + + +class MemDB: + def __init__(self): + self.docs = {} + self.commits = 0 + + def collection(self, name): + return MemColl(self, name) + + def batch(self): + return MemBatch(self) + + def get_all(self, refs): + return [r.get() for r in refs] + + +# ---------------------------------------------------------------------------- +# Helpers +# ---------------------------------------------------------------------------- +T0 = datetime(2026, 11, 17, 9, 0, tzinfo=timezone.utc) + + +def _video(vid, title=None, desc="desc", privacy="public", tags=("karaoke",)): + return { + "id": vid, + "snippet": { + "title": title or f"Artist - {vid} (Karaoke)", + "description": desc, + "tags": list(tags), + "publishedAt": "2023-06-12T02:49:47Z", + "categoryId": "10", + }, + "status": {"privacyStatus": privacy, "uploadStatus": "processed", "license": "youtube", + "embeddable": True, "madeForKids": False}, + "contentDetails": {"duration": "PT3M30S"}, + } + + +def _entry(vid): + return {"video_id": vid, "artist": "Artist", "song_title": vid, "brand_code": "NOMAD-0001", + "kind": "nomad_recent", "target": True} + + +def _sync(db, videos, now, listed=None, gone=(), pre=None): + return catalog.sync_catalog( + db, videos, [_entry(v) for v in videos], + listed_ids=list(videos) if listed is None else listed, + gone_ids=list(gone), now=now, backfill_pre_update=pre, + ) + + +def _doc(db, vid): + return db.docs[f"{catalog.COLLECTION}/{vid}"] + + +def _history(db, vid): + return list(db.collection(catalog.COLLECTION).document(vid).collection("history").docs().values()) + + +# ---------------------------------------------------------------------------- +# Tests +# ---------------------------------------------------------------------------- +def test_first_sync_creates_docs_with_metadata_and_first_seen_history(): + db = MemDB() + stats = _sync(db, {"a": _video("a"), "b": _video("b")}, T0, listed=["a"]) + assert stats["new"] == 2 + a = _doc(db, "a") + assert a["title"] == "Artist - a (Karaoke)" + assert a["privacy_status"] == "public" + assert a["duration"] == "PT3M30S" + assert a["tags"] == ["karaoke"] + assert a["brand_code"] == "NOMAD-0001" and a["song_title"] == "a" + assert a["first_seen_at"] == T0 and a["last_seen_at"] == T0 + assert a["deleted"] is False + assert a["in_uploads_playlist"] is True + assert _doc(db, "b")["in_uploads_playlist"] is False + assert [h["event"] for h in _history(db, "a")] == ["first_seen"] + + +def test_unchanged_video_same_day_is_not_rewritten(): + db = MemDB() + _sync(db, {"a": _video("a")}, T0) + before = dict(_doc(db, "a")) + stats = _sync(db, {"a": _video("a")}, T0 + timedelta(hours=2)) + assert stats["touched"] == 0 and stats["changed"] == 0 + assert _doc(db, "a") == before + assert len(_history(db, "a")) == 1 + + +def test_unchanged_video_next_day_refreshes_last_seen_only(): + db = MemDB() + _sync(db, {"a": _video("a")}, T0) + t1 = T0 + timedelta(days=1) + stats = _sync(db, {"a": _video("a")}, t1) + assert stats["touched"] == 1 and stats["changed"] == 0 + assert _doc(db, "a")["last_seen_at"] == t1 + assert _doc(db, "a")["last_changed_at"] == T0 + assert len(_history(db, "a")) == 1 + + +def test_title_and_privacy_change_recorded_as_observed(): + db = MemDB() + _sync(db, {"a": _video("a", title="Old (Karaoke)", privacy="private")}, T0) + t1 = T0 + timedelta(days=1) + stats = _sync(db, {"a": _video("a", title="New (Karaoke)")}, t1) + assert stats["changed"] == 1 + hist = [h for h in _history(db, "a") if h["event"] == "changed"] + assert len(hist) == 1 + assert hist[0]["source"] == "observed" + assert hist[0]["changes"]["title"] == {"old": "Old (Karaoke)", "new": "New (Karaoke)"} + assert hist[0]["changes"]["privacy_status"] == {"old": "private", "new": "public"} + assert "description" not in hist[0]["changes"] + assert _doc(db, "a")["title"] == "New (Karaoke)" + assert _doc(db, "a")["last_changed_at"] == t1 + + +def test_backfill_rewrite_attributed_to_backfill(): + db = MemDB() + _sync(db, {"a": _video("a", desc="old template")}, T0) + _sync(db, {"a": _video("a", desc="new template")}, T0 + timedelta(days=1), + pre={"a": _video("a", desc="old template")}) + hist = [h for h in _history(db, "a") if h["event"] == "changed"] + assert len(hist) == 1 + assert hist[0]["source"] == "backfill" + assert hist[0]["changes"]["description"]["old"] == "old template" + + +def test_studio_edit_and_backfill_rewrite_in_one_run_are_logged_separately(): + db = MemDB() + _sync(db, {"a": _video("a", title="Old (Karaoke)", desc="old template")}, T0) + # Someone renamed it in Studio; then this run rewrote the description. + before = _video("a", title="New (Karaoke)", desc="old template") + after = _video("a", title="New (Karaoke)", desc="new template") + _sync(db, {"a": after}, T0 + timedelta(days=1), pre={"a": before}) + hist = {h["source"]: h for h in _history(db, "a") if h["event"] == "changed"} + assert set(hist["observed"]["changes"]) == {"title"} + assert set(hist["backfill"]["changes"]) == {"description"} + + +def test_gone_video_is_soft_deleted_once_and_keeps_metadata(): + db = MemDB() + _sync(db, {"a": _video("a"), "b": _video("b")}, T0) + t1 = T0 + timedelta(days=1) + stats = _sync(db, {"a": _video("a")}, t1, gone=["b"]) + assert stats["deleted"] == 1 + b = _doc(db, "b") + assert b["deleted"] is True and b["deleted_at"] == t1 + assert b["title"] == "Artist - b (Karaoke)" # record retained + # Still gone the next day: no second delete event. + stats = _sync(db, {"a": _video("a")}, t1 + timedelta(days=1), gone=["b"]) + assert stats["deleted"] == 0 + assert [h["event"] for h in _history(db, "b")].count("deleted") == 1 + + +def test_gone_video_never_catalogued_gets_a_stub(): + db = MemDB() + _sync(db, {}, T0, gone=["ghost"]) + ghost = _doc(db, "ghost") + assert ghost["deleted"] is True and ghost["video_id"] == "ghost" + assert [h["event"] for h in _history(db, "ghost")] == ["deleted"] + + +def test_deleted_video_that_reappears_is_restored(): + db = MemDB() + _sync(db, {"a": _video("a")}, T0) + _sync(db, {}, T0 + timedelta(days=1), gone=["a"]) + t2 = T0 + timedelta(days=2) + stats = _sync(db, {"a": _video("a", privacy="unlisted")}, t2) + assert stats["restored"] == 1 + a = _doc(db, "a") + assert a["deleted"] is False and a["deleted_at"] is None + restored = [h for h in _history(db, "a") if h["event"] == "restored"] + assert restored and restored[0]["changes"]["privacy_status"]["new"] == "unlisted" + + +def test_playlist_membership_change_updates_doc_without_history(): + db = MemDB() + _sync(db, {"a": _video("a")}, T0, listed=["a"]) + stats = _sync(db, {"a": _video("a")}, T0 + timedelta(hours=1), listed=[]) + assert stats["touched"] == 1 and stats["changed"] == 0 + assert _doc(db, "a")["in_uploads_playlist"] is False + assert len(_history(db, "a")) == 1 + + +def test_large_sync_commits_in_multiple_batches_without_splitting_pairs(): + db = MemDB() + batches = [] + orig_commit = MemBatch.commit + + def _spy(self): + batches.append([p for p, _, _ in self.ops]) + orig_commit(self) + + MemBatch.commit = _spy + try: + videos = {f"v{i}": _video(f"v{i}") for i in range(450)} + stats = _sync(db, videos, T0) + finally: + MemBatch.commit = orig_commit + assert stats["new"] == 450 + assert len(batches) == 3 and all(len(b) <= catalog._BATCH_LIMIT for b in batches) + # every video's doc and its first_seen history land in the same batch + for b in batches: + docs = {p.split("/")[1] for p in b if p.count("/") == 1} + hist = {p.split("/")[1] for p in b if "/history/" in p} + assert docs == hist + assert len(db.collection(catalog.COLLECTION).docs()) == 450 + + +def test_new_video_rewritten_in_same_run_keeps_original_in_history(): + db = MemDB() + _sync(db, {"a": _video("a", desc="new template")}, T0, pre={"a": _video("a", desc="upload description")}) + events = sorted((h["event"], h["source"]) for h in _history(db, "a")) + assert events == [("changed", "backfill"), ("first_seen", "observed")] + changed = [h for h in _history(db, "a") if h["event"] == "changed"][0] + assert changed["changes"]["description"] == {"old": "upload description", "new": "new template"} + assert _doc(db, "a")["description"] == "new template" diff --git a/backend/workers/youtube_description_backfill_worker.py b/backend/workers/youtube_description_backfill_worker.py index 36ccf7c7..2c2f73a4 100644 --- a/backend/workers/youtube_description_backfill_worker.py +++ b/backend/workers/youtube_description_backfill_worker.py @@ -19,6 +19,9 @@ Design (built to be re-usable, not a one-shot): video ID ever seen is kept in Firestore (`youtube_backfill/known_video_ids`) and looked up directly each run. First run seeds it from the quota ledger's `desc-backfill:` entries (every video this job has ever rewritten). +- Every video's metadata is mirrored to the `youtube_videos` catalogue (with a + per-video change history; vanished videos are soft-deleted, never removed). + See `backend/services/youtube_video_catalog.py`. - A Postmark email is sent after each run (progress) and once when the channel is fully drained (completion). @@ -31,6 +34,7 @@ from typing import Dict, List, Optional from backend.config import get_settings from backend.services import youtube_backfill as bf +from backend.services import youtube_video_catalog as catalog from backend.services.firestore_service import get_firestore_client from backend.services.youtube_quota_service import get_youtube_quota_service @@ -148,6 +152,7 @@ def run_backfill_sync(max_updates: Optional[int] = None, dry_run: bool = False) budget = min(budget, max_updates) updated = 0 + pre_update: Dict[str, Dict] = {} errors: Dict[str, str] = {} if not dry_run and pending_before > 0 and budget > 0: for entry in pending[:budget]: @@ -169,6 +174,11 @@ def run_backfill_sync(max_updates: Optional[int] = None, dry_run: bool = False) logger.error(f"YOUTUBE_BACKFILL: update failed for {vid}: {msg}") continue updated += 1 + # Reflect our own rewrite so the catalogue stores the new + # description and attributes the change to the backfill. + if vid in videos: + pre_update[vid] = videos[vid] + videos[vid] = {**videos[vid], "snippet": {**current_snippet, **body}} try: quota.record_upload(f"desc-backfill:{vid}", units=bf.UPDATE_COST) except Exception: # noqa: BLE001 @@ -227,12 +237,28 @@ def run_backfill_sync(max_updates: Optional[int] = None, dry_run: bool = False) logger.error(f"YOUTUBE_BACKFILL: failed to send report email: {exc}") state_ref.set(state) - # Remember every video videos.list returned (drops deleted ones). Skipped - # when the first-time seed failed, so the next run retries it. + # Remember every video ever seen. Gone ones are kept (soft-deleted in the + # catalogue) and re-checked, so a reappearance is noticed. Skipped when the + # first-time seed failed, so the next run retries it. if known_ids is not None: db.collection(STATE_COLLECTION).document(KNOWN_IDS_DOC).set( - {"ids": sorted(videos.keys()), "updated_at": now} + {"ids": sorted(set(videos) | set(known_ids)), "updated_at": now} ) + # Best-effort mirror of every video's metadata; never fails the run. + try: + catalog_stats = catalog.sync_catalog( + db, + videos, + entries, + listed_ids=channel.listed_ids, + gone_ids=channel.gone_ids, + now=now, + backfill_pre_update=pre_update, + ) + summary["catalog"] = catalog_stats + logger.info(f"YOUTUBE_BACKFILL: catalogue synced: {catalog_stats}") + except Exception as exc: # noqa: BLE001 + logger.warning(f"YOUTUBE_BACKFILL: catalogue sync failed: {exc}") logger.info(f"YOUTUBE_BACKFILL: run complete: {summary}") return summary diff --git a/pyproject.toml b/pyproject.toml index 313b117d..ce9c0c26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.297.2" +version = "0.297.3" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/youtube-descriptions/README.md b/scripts/youtube-descriptions/README.md index cc1755e6..ca64fafe 100644 --- a/scripts/youtube-descriptions/README.md +++ b/scripts/youtube-descriptions/README.md @@ -25,6 +25,18 @@ and `backend/services/youtube_description.py` for the template itself): The progress email and the run summary (`recovered_from_known`) say how many videos the playlist left out. + **Video catalogue.** Each run also mirrors every video's metadata to Firestore + `youtube_videos/{video_id}` (title, description, tags, publish date, privacy, + duration, parsed artist/song/brand code, `first_seen_at`, `last_seen_at`, + `last_changed_at`, `in_uploads_playlist`). Every change is logged in + `youtube_videos/{id}/history` as `{at, event, changes: {field: {old, new}}, source}` + (`event` = first_seen / changed / deleted / restored; `source` = `backfill` for + this job's own rewrites, else `observed`). Videos YouTube stops returning are + **soft-deleted** (`deleted=True`, `deleted_at`), never removed, and keep being + checked. No extra YouTube quota; writes only on change. Code: + `backend/services/youtube_video_catalog.py`. Example: what changed on a video? + `db.collection("youtube_videos").document(vid).collection("history").order_by("at").stream()`. + 2. **This CLI (manual / local one-offs).** For analysis, dry-runs, reviewing diffs, or updating specific videos by hand.