From 0449d0c1070824aacb5284f6b9a82cf7099aaa37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 19 Nov 2026 15:16:34 -0500 Subject: [PATCH] fix(infra): build Cloud Function source archives without __pycache__ (v0.297.4) (#1272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary A full `pulumi preview` from main showed runner-manager drift ("replace source, update function"), last applied from the `karaoke-gen-runner-scan-retry` worktree. I checked it: - #1262 (that worktree's branch) is fully on main. The squash merge is byte-identical to the branch tip. The live scheduler job (`retryCount 2`, `maxRetryDuration 60s`, `attemptDeadline 300s`) also matches the code exactly. - The drift comes from the **deployed zip containing `__pycache__/*.pyc`** (including a pytest pyc) from the worktree's test run. `pulumi.FileArchive` hashes every file, so every clean checkout saw a phantom diff. Separately, the 7-day lifecycle rule had already deleted the bucket object. ## Changes - New `infrastructure/modules/source_archive.clean_source_archive()` builds an `AssetArchive` of source files only. It skips `__pycache__`, `*.pyc`, `.pytest_cache`, other tool caches and `.DS_Store`. - Used for runner-manager, divebar-lookup and divebar-mirror (the three `FileArchive` users). - Unit tests, added to the "Infra - Cloud Function tests" CI job. - LESSONS-LEARNED entry. ## Applied (before merge, per repo convention) - I ran a targeted preview with `__pycache__` present in the worktree. Runner-manager showed source replace + function update (the one-time correction). Divebar-lookup and divebar-mirror showed **no change**. - Targeted `pulumi up` ran as claude-admin (Andrew's grant): 2 changes, 289 unchanged. - The new revision is `github-runner-manager-00031-sam`, and the deployed zip now holds only the 4 source files. - The re-preview shows 291 unchanged. ## Testing - `pytest infrastructure/test_source_archive.py`: 4 passed - `pytest infrastructure/functions/runner_manager`: 75 passed - Local CodeRabbit review: no findings @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVSfwoV9WyncjqMPHrq --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 + docs/LESSONS-LEARNED.md | 9 +++++ infrastructure/modules/divebar_lookup.py | 3 +- infrastructure/modules/divebar_mirror.py | 4 +- infrastructure/modules/runner_manager.py | 3 +- infrastructure/modules/source_archive.py | 44 +++++++++++++++++++++ infrastructure/test_source_archive.py | 50 ++++++++++++++++++++++++ pyproject.toml | 2 +- 8 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 infrastructure/modules/source_archive.py create mode 100644 infrastructure/test_source_archive.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87e14ebe..a9a849fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,8 @@ jobs: - run: pip install -r infrastructure/functions/encoding_worker_idle/requirements.txt # Separate invocation: both functions have a top-level `main` module. - run: pytest infrastructure/functions/encoding_worker_idle -q -p no:cacheprovider -o addopts="" + - run: pip install "pulumi>=3.0.0,<4.0.0" + - run: pytest infrastructure/test_source_archive.py -q -p no:cacheprovider -o addopts="" frontend-lint: name: "Frontend - Lint (Rules of Hooks)" diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index eb4ac6c1..c03c52a8 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -7,6 +7,15 @@ Key insights for future AI agents working on this codebase. --- +## `pulumi.FileArchive` packed `__pycache__`, so every checkout saw phantom drift (Nov 2026, v0.297.4) + +The runner-manager function was applied from a worktree that had just run its tests, so the deployed zip held +`__pycache__/*.pyc`. `FileArchive` hashes every file in the directory, so a clean checkout of main then previewed +"replace source + update function" with byte-identical code, which looked like unmerged work. Cloud Function +sources are now built with `modules/source_archive.clean_source_archive()`, which packs source files only. Use it +for any new function. To tell real drift from this kind, compare the live zip +(`gcloud functions describe ... buildConfig.source` → `gcloud storage cp` → `unzip -l`) against the checkout. + ## An AI "correction" nothing can confirm is a guess: ground it in real data (Nov 2026, v0.294.0) At a live kjbox night a singer typed "Cole Phillips – Roll of Dice", a 2025 release missing from the diff --git a/infrastructure/modules/divebar_lookup.py b/infrastructure/modules/divebar_lookup.py index 11090fea..ce7daf69 100644 --- a/infrastructure/modules/divebar_lookup.py +++ b/infrastructure/modules/divebar_lookup.py @@ -20,6 +20,7 @@ from pulumi_gcp import ( bigquery, ) +from modules.source_archive import clean_source_archive from config import PROJECT_ID, REGION, get_project_number BIGQUERY_DATASET = "karaoke_decide" @@ -81,7 +82,7 @@ def create_divebar_lookup_resources(all_secrets: dict) -> dict: "divebar-lookup-source", bucket=source_bucket.name, name="divebar-lookup-source.zip", - source=pulumi.FileArchive(str(source_dir)), + source=clean_source_archive(source_dir), ) resources["source_archive"] = source_archive diff --git a/infrastructure/modules/divebar_mirror.py b/infrastructure/modules/divebar_mirror.py index dd33ee68..77ee828c 100644 --- a/infrastructure/modules/divebar_mirror.py +++ b/infrastructure/modules/divebar_mirror.py @@ -13,7 +13,6 @@ Creates: import base64 from pathlib import Path -import pulumi import pulumi_gcp as gcp from pulumi_gcp import ( cloudfunctionsv2, @@ -23,6 +22,7 @@ from pulumi_gcp import ( bigquery, ) +from modules.source_archive import clean_source_archive from config import PROJECT_ID, REGION, get_project_number # Divebar shared folder ID (public Google Drive folder) @@ -124,7 +124,7 @@ def create_divebar_mirror_resources(all_secrets: dict) -> dict: "divebar-mirror-source", bucket=source_bucket.name, name="divebar-mirror-source.zip", - source=pulumi.FileArchive(str(source_dir)), + source=clean_source_archive(source_dir), ) resources["source_archive"] = source_archive diff --git a/infrastructure/modules/runner_manager.py b/infrastructure/modules/runner_manager.py index ce645804..d7fb54a0 100644 --- a/infrastructure/modules/runner_manager.py +++ b/infrastructure/modules/runner_manager.py @@ -29,6 +29,7 @@ from pathlib import Path import pulumi import pulumi_gcp as gcp +from modules.source_archive import clean_source_archive from config import ( PROJECT_ID, REGION, @@ -133,7 +134,7 @@ def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.Bu "runner-manager-source", bucket=bucket.name, name="runner-manager-source.zip", - source=pulumi.FileArchive(str(source_dir)), + source=clean_source_archive(source_dir), ) diff --git a/infrastructure/modules/source_archive.py b/infrastructure/modules/source_archive.py new file mode 100644 index 00000000..f2fed7aa --- /dev/null +++ b/infrastructure/modules/source_archive.py @@ -0,0 +1,44 @@ +"""Deterministic Cloud Function source archives. + +``pulumi.FileArchive(dir)`` zips and hashes *every* file in the directory. If +someone runs the function's tests first, ``__pycache__/*.pyc`` and +``.pytest_cache`` get packed into the deployed zip. The archive hash then +depends on which checkout last ran ``pulumi up``, so a clean checkout of main +shows phantom drift: the source object gets replaced and the function updated +with identical code. This happened to runner-manager on 2026-11-09. + +``clean_source_archive`` packs only the real source files, so the hash depends +only on the code. +""" + +from pathlib import Path + +import pulumi + +EXCLUDED_DIRS = frozenset({"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "venv", ".venv"}) +EXCLUDED_SUFFIXES = (".pyc", ".pyo") +EXCLUDED_NAMES = frozenset({".DS_Store"}) + + +def source_files(source_dir: Path) -> dict[str, Path]: + """Map archive-relative path -> file, skipping caches and OS junk.""" + files: dict[str, Path] = {} + for path in sorted(source_dir.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(source_dir) + if any(part in EXCLUDED_DIRS for part in rel.parts[:-1]): + continue + if path.name in EXCLUDED_NAMES or path.suffix in EXCLUDED_SUFFIXES: + continue + files[rel.as_posix()] = path + if not files: + raise ValueError(f"No source files found in {source_dir}") + return files + + +def clean_source_archive(source_dir: Path) -> pulumi.AssetArchive: + """An archive of ``source_dir``'s source files only (see module docstring).""" + return pulumi.AssetArchive( + {rel: pulumi.FileAsset(str(path)) for rel, path in source_files(source_dir).items()} + ) diff --git a/infrastructure/test_source_archive.py b/infrastructure/test_source_archive.py new file mode 100644 index 00000000..05e987a1 --- /dev/null +++ b/infrastructure/test_source_archive.py @@ -0,0 +1,50 @@ +"""Tests for modules/source_archive.py (run: pytest infrastructure/test_source_archive.py).""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent / "modules")) + +from source_archive import clean_source_archive, source_files # noqa: E402 + + +def _write(path: Path, text: str = "x") -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +def test_skips_caches_and_os_junk(tmp_path): + _write(tmp_path / "main.py") + _write(tmp_path / "requirements.txt") + _write(tmp_path / "pkg" / "helper.py") + _write(tmp_path / "__pycache__" / "main.cpython-313.pyc") + _write(tmp_path / "pkg" / "__pycache__" / "helper.cpython-313.pyc") + _write(tmp_path / ".pytest_cache" / "v" / "cache" / "lastfailed") + _write(tmp_path / "stray.pyc") + _write(tmp_path / ".DS_Store") + + assert list(source_files(tmp_path)) == ["main.py", "pkg/helper.py", "requirements.txt"] + + +def test_same_files_regardless_of_test_run_leftovers(tmp_path): + clean = tmp_path / "clean" + dirty = tmp_path / "dirty" + for root in (clean, dirty): + _write(root / "main.py", "print('hi')") + _write(dirty / "__pycache__" / "main.cpython-313.pyc", "bytecode") + + assert list(source_files(clean)) == list(source_files(dirty)) + + +def test_archive_holds_file_assets(tmp_path): + _write(tmp_path / "main.py") + archive = clean_source_archive(tmp_path) + assert set(archive.assets) == {"main.py"} + + +def test_empty_dir_raises(tmp_path): + _write(tmp_path / "__pycache__" / "only.pyc") + with pytest.raises(ValueError): + source_files(tmp_path) diff --git a/pyproject.toml b/pyproject.toml index ce9c0c26..a02554af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.297.3" +version = "0.297.4" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"