mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(infra): build Cloud Function source archives without __pycache__ (v0.297.4) (#1272)
## Summary
A full `pulumi preview` from main showed runner-manager drift ("replace
source, update function"), last applied from the
`karaoke-gen-runner-scan-retry` worktree. I checked it:
- #1262 (that worktree's branch) is fully on main. The squash merge is
byte-identical to the branch tip. The live scheduler job (`retryCount
2`, `maxRetryDuration 60s`, `attemptDeadline 300s`) also matches the
code exactly.
- The drift comes from the **deployed zip containing
`__pycache__/*.pyc`** (including a pytest pyc) from the worktree's test
run. `pulumi.FileArchive` hashes every file, so every clean checkout saw
a phantom diff. Separately, the 7-day lifecycle rule had already deleted
the bucket object.
## Changes
- New `infrastructure/modules/source_archive.clean_source_archive()`
builds an `AssetArchive` of source files only. It skips `__pycache__`,
`*.pyc`, `.pytest_cache`, other tool caches and `.DS_Store`.
- Used for runner-manager, divebar-lookup and divebar-mirror (the three
`FileArchive` users).
- Unit tests, added to the "Infra - Cloud Function tests" CI job.
- LESSONS-LEARNED entry.
## Applied (before merge, per repo convention)
- I ran a targeted preview with `__pycache__` present in the worktree.
Runner-manager showed source replace + function update (the one-time
correction). Divebar-lookup and divebar-mirror showed **no change**.
- Targeted `pulumi up` ran as claude-admin (Andrew's grant): 2 changes,
289 unchanged.
- The new revision is `github-runner-manager-00031-sam`, and the
deployed zip now holds only the 4 source files.
- The re-preview shows 291 unchanged.
## Testing
- `pytest infrastructure/test_source_archive.py`: 4 passed
- `pytest infrastructure/functions/runner_manager`: 75 passed
- Local CodeRabbit review: no findings
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_012RYLVSfwoV9WyncjqMPHrq
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2971145c85
commit
0449d0c107
8 files changed
+112
-5
No files matched your search
@@ -103,6 +103,8 @@ jobs:
|
||||
- run: pip install -r infrastructure/functions/encoding_worker_idle/requirements.txt
|
||||
# Separate invocation: both functions have a top-level `main` module.
|
||||
- run: pytest infrastructure/functions/encoding_worker_idle -q -p no:cacheprovider -o addopts=""
|
||||
- run: pip install "pulumi>=3.0.0,<4.0.0"
|
||||
- run: pytest infrastructure/test_source_archive.py -q -p no:cacheprovider -o addopts=""
|
||||
|
||||
frontend-lint:
|
||||
name: "Frontend - Lint (Rules of Hooks)"
|
||||
|
||||
@@ -7,6 +7,15 @@ Key insights for future AI agents working on this codebase.
|
||||
---
|
||||
|
||||
|
||||
## `pulumi.FileArchive` packed `__pycache__`, so every checkout saw phantom drift (Nov 2026, v0.297.4)
|
||||
|
||||
The runner-manager function was applied from a worktree that had just run its tests, so the deployed zip held
|
||||
`__pycache__/*.pyc`. `FileArchive` hashes every file in the directory, so a clean checkout of main then previewed
|
||||
"replace source + update function" with byte-identical code, which looked like unmerged work. Cloud Function
|
||||
sources are now built with `modules/source_archive.clean_source_archive()`, which packs source files only. Use it
|
||||
for any new function. To tell real drift from this kind, compare the live zip
|
||||
(`gcloud functions describe ... buildConfig.source` → `gcloud storage cp` → `unzip -l`) against the checkout.
|
||||
|
||||
## An AI "correction" nothing can confirm is a guess: ground it in real data (Nov 2026, v0.294.0)
|
||||
|
||||
At a live kjbox night a singer typed "Cole Phillips – Roll of Dice", a 2025 release missing from the
|
||||
|
||||
@@ -20,6 +20,7 @@ from pulumi_gcp import (
|
||||
bigquery,
|
||||
)
|
||||
|
||||
from modules.source_archive import clean_source_archive
|
||||
from config import PROJECT_ID, REGION, get_project_number
|
||||
|
||||
BIGQUERY_DATASET = "karaoke_decide"
|
||||
@@ -81,7 +82,7 @@ def create_divebar_lookup_resources(all_secrets: dict) -> dict:
|
||||
"divebar-lookup-source",
|
||||
bucket=source_bucket.name,
|
||||
name="divebar-lookup-source.zip",
|
||||
source=pulumi.FileArchive(str(source_dir)),
|
||||
source=clean_source_archive(source_dir),
|
||||
)
|
||||
resources["source_archive"] = source_archive
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ Creates:
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
import pulumi
|
||||
import pulumi_gcp as gcp
|
||||
from pulumi_gcp import (
|
||||
cloudfunctionsv2,
|
||||
@@ -23,6 +22,7 @@ from pulumi_gcp import (
|
||||
bigquery,
|
||||
)
|
||||
|
||||
from modules.source_archive import clean_source_archive
|
||||
from config import PROJECT_ID, REGION, get_project_number
|
||||
|
||||
# Divebar shared folder ID (public Google Drive folder)
|
||||
@@ -124,7 +124,7 @@ def create_divebar_mirror_resources(all_secrets: dict) -> dict:
|
||||
"divebar-mirror-source",
|
||||
bucket=source_bucket.name,
|
||||
name="divebar-mirror-source.zip",
|
||||
source=pulumi.FileArchive(str(source_dir)),
|
||||
source=clean_source_archive(source_dir),
|
||||
)
|
||||
resources["source_archive"] = source_archive
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from pathlib import Path
|
||||
|
||||
import pulumi
|
||||
import pulumi_gcp as gcp
|
||||
from modules.source_archive import clean_source_archive
|
||||
from config import (
|
||||
PROJECT_ID,
|
||||
REGION,
|
||||
@@ -133,7 +134,7 @@ def create_function_source_archive(bucket: gcp.storage.Bucket) -> gcp.storage.Bu
|
||||
"runner-manager-source",
|
||||
bucket=bucket.name,
|
||||
name="runner-manager-source.zip",
|
||||
source=pulumi.FileArchive(str(source_dir)),
|
||||
source=clean_source_archive(source_dir),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Deterministic Cloud Function source archives.
|
||||
|
||||
``pulumi.FileArchive(dir)`` zips and hashes *every* file in the directory. If
|
||||
someone runs the function's tests first, ``__pycache__/*.pyc`` and
|
||||
``.pytest_cache`` get packed into the deployed zip. The archive hash then
|
||||
depends on which checkout last ran ``pulumi up``, so a clean checkout of main
|
||||
shows phantom drift: the source object gets replaced and the function updated
|
||||
with identical code. This happened to runner-manager on 2026-11-09.
|
||||
|
||||
``clean_source_archive`` packs only the real source files, so the hash depends
|
||||
only on the code.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pulumi
|
||||
|
||||
EXCLUDED_DIRS = frozenset({"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "venv", ".venv"})
|
||||
EXCLUDED_SUFFIXES = (".pyc", ".pyo")
|
||||
EXCLUDED_NAMES = frozenset({".DS_Store"})
|
||||
|
||||
|
||||
def source_files(source_dir: Path) -> dict[str, Path]:
|
||||
"""Map archive-relative path -> file, skipping caches and OS junk."""
|
||||
files: dict[str, Path] = {}
|
||||
for path in sorted(source_dir.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
rel = path.relative_to(source_dir)
|
||||
if any(part in EXCLUDED_DIRS for part in rel.parts[:-1]):
|
||||
continue
|
||||
if path.name in EXCLUDED_NAMES or path.suffix in EXCLUDED_SUFFIXES:
|
||||
continue
|
||||
files[rel.as_posix()] = path
|
||||
if not files:
|
||||
raise ValueError(f"No source files found in {source_dir}")
|
||||
return files
|
||||
|
||||
|
||||
def clean_source_archive(source_dir: Path) -> pulumi.AssetArchive:
|
||||
"""An archive of ``source_dir``'s source files only (see module docstring)."""
|
||||
return pulumi.AssetArchive(
|
||||
{rel: pulumi.FileAsset(str(path)) for rel, path in source_files(source_dir).items()}
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Tests for modules/source_archive.py (run: pytest infrastructure/test_source_archive.py)."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent / "modules"))
|
||||
|
||||
from source_archive import clean_source_archive, source_files # noqa: E402
|
||||
|
||||
|
||||
def _write(path: Path, text: str = "x") -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text)
|
||||
|
||||
|
||||
def test_skips_caches_and_os_junk(tmp_path):
|
||||
_write(tmp_path / "main.py")
|
||||
_write(tmp_path / "requirements.txt")
|
||||
_write(tmp_path / "pkg" / "helper.py")
|
||||
_write(tmp_path / "__pycache__" / "main.cpython-313.pyc")
|
||||
_write(tmp_path / "pkg" / "__pycache__" / "helper.cpython-313.pyc")
|
||||
_write(tmp_path / ".pytest_cache" / "v" / "cache" / "lastfailed")
|
||||
_write(tmp_path / "stray.pyc")
|
||||
_write(tmp_path / ".DS_Store")
|
||||
|
||||
assert list(source_files(tmp_path)) == ["main.py", "pkg/helper.py", "requirements.txt"]
|
||||
|
||||
|
||||
def test_same_files_regardless_of_test_run_leftovers(tmp_path):
|
||||
clean = tmp_path / "clean"
|
||||
dirty = tmp_path / "dirty"
|
||||
for root in (clean, dirty):
|
||||
_write(root / "main.py", "print('hi')")
|
||||
_write(dirty / "__pycache__" / "main.cpython-313.pyc", "bytecode")
|
||||
|
||||
assert list(source_files(clean)) == list(source_files(dirty))
|
||||
|
||||
|
||||
def test_archive_holds_file_assets(tmp_path):
|
||||
_write(tmp_path / "main.py")
|
||||
archive = clean_source_archive(tmp_path)
|
||||
assert set(archive.assets) == {"main.py"}
|
||||
|
||||
|
||||
def test_empty_dir_raises(tmp_path):
|
||||
_write(tmp_path / "__pycache__" / "only.pyc")
|
||||
with pytest.raises(ValueError):
|
||||
source_files(tmp_path)
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.297.3"
|
||||
version = "0.297.4"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user