fix(infra): build Cloud Function source archives without __pycache__ (v0.297.4) (#1272)

## Summary
A full `pulumi preview` from main showed runner-manager drift ("replace
source, update function"), last applied from the
`karaoke-gen-runner-scan-retry` worktree. I checked it:

- #1262 (that worktree's branch) is fully on main. The squash merge is
byte-identical to the branch tip. The live scheduler job (`retryCount
2`, `maxRetryDuration 60s`, `attemptDeadline 300s`) also matches the
code exactly.
- The drift comes from the **deployed zip containing
`__pycache__/*.pyc`** (including a pytest pyc) from the worktree's test
run. `pulumi.FileArchive` hashes every file, so every clean checkout saw
a phantom diff. Separately, the 7-day lifecycle rule had already deleted
the bucket object.

## Changes
- New `infrastructure/modules/source_archive.clean_source_archive()`
builds an `AssetArchive` of source files only. It skips `__pycache__`,
`*.pyc`, `.pytest_cache`, other tool caches and `.DS_Store`.
- Used for runner-manager, divebar-lookup and divebar-mirror (the three
`FileArchive` users).
- Unit tests, added to the "Infra - Cloud Function tests" CI job.
- LESSONS-LEARNED entry.

## Applied (before merge, per repo convention)
- I ran a targeted preview with `__pycache__` present in the worktree.
Runner-manager showed source replace + function update (the one-time
correction). Divebar-lookup and divebar-mirror showed **no change**.
- Targeted `pulumi up` ran as claude-admin (Andrew's grant): 2 changes,
289 unchanged.
- The new revision is `github-runner-manager-00031-sam`, and the
deployed zip now holds only the 4 source files.
- The re-preview shows 291 unchanged.

## Testing
- `pytest infrastructure/test_source_archive.py`: 4 passed
- `pytest infrastructure/functions/runner_manager`: 75 passed
- Local CodeRabbit review: no findings

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_012RYLVSfwoV9WyncjqMPHrq

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 5.5 authored and GitHub committed 2026-11-19 15:16:34 -05:00
1 parent 2971145c85
commit 0449d0c107
8 files changed
+112 -5

No files matched your search

+2
View File
@@ -103,6 +103,8 @@ jobs:
- run: pip install -r infrastructure/functions/encoding_worker_idle/requirements.txt
# Separate invocation: both functions have a top-level `main` module.
- run: pytest infrastructure/functions/encoding_worker_idle -q -p no:cacheprovider -o addopts=""
- run: pip install "pulumi>=3.0.0,<4.0.0"
- run: pytest infrastructure/test_source_archive.py -q -p no:cacheprovider -o addopts=""
frontend-lint:
name: "Frontend - Lint (Rules of Hooks)"
+9
View File
@@ -7,6 +7,15 @@ Key insights for future AI agents working on this codebase.
---
## `pulumi.FileArchive` packed `__pycache__`, so every checkout saw phantom drift (Nov 2026, v0.297.4)
The runner-manager function was applied from a worktree that had just run its tests, so the deployed zip held
`__pycache__/*.pyc`. `FileArchive` hashes every file in the directory, so a clean checkout of main then previewed
"replace source + update function" with byte-identical code, which looked like unmerged work. Cloud Function
sources are now built with `modules/source_archive.clean_source_archive()`, which packs source files only. Use it
for any new function. To tell real drift from this kind, compare the live zip
(`gcloud functions describe ... buildConfig.source` → `gcloud storage cp` → `unzip -l`) against the checkout.
## An AI "correction" nothing can confirm is a guess: ground it in real data (Nov 2026, v0.294.0)
At a live kjbox night a singer typed "Cole Phillips – Roll of Dice", a 2025 release missing from the
+2 -1
View File
@@ -20,6 +20,7 @@ from pulumi_gcp import (
bigquery,
)
from modules.source_archive import clean_source_archive
from config import PROJECT_ID, REGION, get_project_number
BIGQUERY_DATASET = "karaoke_decide"
@@ -81,7 +82,7 @@ def create_divebar_lookup_resources(all_secrets: dict) -> dict:
"divebar-lookup-source",
bucket=source_bucket.name,
name="divebar-lookup-source.zip",
source=pulumi.FileArchive(str(source_dir)),
source=clean_source_archive(source_dir),
)
resources["source_archive"] = source_archive
+2 -2
View File
@@ -13,7 +13,6 @@ Creates:
import base64
from pathlib import Path
import pulumi
import pulumi_gcp as gcp
from pulumi_gcp import (
cloudfunctionsv2,
@@ -23,6 +22,7 @@ from pulumi_gcp import (
bigquery,
)
from modules.source_archive import clean_source_archive
from config import PROJECT_ID, REGION, get_project_number
# Divebar shared folder ID (public Google Drive folder)
@@ -124,7 +124,7 @@ def create_divebar_mirror_resources(all_secrets: dict) -> dict:
"divebar-mirror-source",
bucket=source_bucket.name,
name="divebar-mirror-source.zip",
source=pulumi.FileArchive(str(source_dir)),
source=clean_source_archive(source_dir),
)
resources["source_archive"] = source_archive
+2 -1
View File
@@ -29,6 +29,7 @@ from pathlib import Path
import pulumi
import pulumi_gcp as gcp
from modules.source_archive import clean_source_archive
from config import (
PROJECT_ID,
REGION,
@@ -133,7 +134,7 @@ def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.Bu
"runner-manager-source",
bucket=bucket.name,
name="runner-manager-source.zip",
source=pulumi.FileArchive(str(source_dir)),
source=clean_source_archive(source_dir),
)
+44
View File
@@ -0,0 +1,44 @@
"""Deterministic Cloud Function source archives.
``pulumi.FileArchive(dir)`` zips and hashes *every* file in the directory. If
someone runs the function's tests first, ``__pycache__/*.pyc`` and
``.pytest_cache`` get packed into the deployed zip. The archive hash then
depends on which checkout last ran ``pulumi up``, so a clean checkout of main
shows phantom drift: the source object gets replaced and the function updated
with identical code. This happened to runner-manager on 2026-11-09.
``clean_source_archive`` packs only the real source files, so the hash depends
only on the code.
"""
from pathlib import Path
import pulumi
EXCLUDED_DIRS = frozenset({"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "venv", ".venv"})
EXCLUDED_SUFFIXES = (".pyc", ".pyo")
EXCLUDED_NAMES = frozenset({".DS_Store"})
def source_files(source_dir: Path) -> dict[str, Path]:
"""Map archive-relative path -> file, skipping caches and OS junk."""
files: dict[str, Path] = {}
for path in sorted(source_dir.rglob("*")):
if not path.is_file():
continue
rel = path.relative_to(source_dir)
if any(part in EXCLUDED_DIRS for part in rel.parts[:-1]):
continue
if path.name in EXCLUDED_NAMES or path.suffix in EXCLUDED_SUFFIXES:
continue
files[rel.as_posix()] = path
if not files:
raise ValueError(f"No source files found in {source_dir}")
return files
def clean_source_archive(source_dir: Path) -> pulumi.AssetArchive:
"""An archive of ``source_dir``'s source files only (see module docstring)."""
return pulumi.AssetArchive(
{rel: pulumi.FileAsset(str(path)) for rel, path in source_files(source_dir).items()}
)
+50
View File
@@ -0,0 +1,50 @@
"""Tests for modules/source_archive.py (run: pytest infrastructure/test_source_archive.py)."""
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parent / "modules"))
from source_archive import clean_source_archive, source_files # noqa: E402
def _write(path: Path, text: str = "x") -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text)
def test_skips_caches_and_os_junk(tmp_path):
_write(tmp_path / "main.py")
_write(tmp_path / "requirements.txt")
_write(tmp_path / "pkg" / "helper.py")
_write(tmp_path / "__pycache__" / "main.cpython-313.pyc")
_write(tmp_path / "pkg" / "__pycache__" / "helper.cpython-313.pyc")
_write(tmp_path / ".pytest_cache" / "v" / "cache" / "lastfailed")
_write(tmp_path / "stray.pyc")
_write(tmp_path / ".DS_Store")
assert list(source_files(tmp_path)) == ["main.py", "pkg/helper.py", "requirements.txt"]
def test_same_files_regardless_of_test_run_leftovers(tmp_path):
clean = tmp_path / "clean"
dirty = tmp_path / "dirty"
for root in (clean, dirty):
_write(root / "main.py", "print('hi')")
_write(dirty / "__pycache__" / "main.cpython-313.pyc", "bytecode")
assert list(source_files(clean)) == list(source_files(dirty))
def test_archive_holds_file_assets(tmp_path):
_write(tmp_path / "main.py")
archive = clean_source_archive(tmp_path)
assert set(archive.assets) == {"main.py"}
def test_empty_dir_raises(tmp_path):
_write(tmp_path / "__pycache__" / "only.pyc")
with pytest.raises(ValueError):
source_files(tmp_path)
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.297.3"
version = "0.297.4"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"