fix: trim baseline GCP cost (secret retention + divebar shutdown) (#764)

## Summary

Two cost-reduction fixes bundled together:

1. **Secret version retention**
(`backend/services/credential_manager.py`): `dropbox-oauth-credentials`
had accumulated 1191 enabled versions costing ~$65/mo in replica
storage. Each refresh calls `add_secret_version` but never destroys the
prior version. Same pattern for the YouTube/GDrive secrets refreshed via
`_update_google_credentials`. Add a small
`_destroy_old_secret_versions(keep=5)` helper called after every
`add_secret_version` in this module. Errors swallowed so a prune failure
cannot break credential refresh.

2. **Divebar-sync auto-shutdown**
(`infrastructure/compute/startup_scripts/divebar_sync.sh`): VM was
running 24/7 for 45 days since 2026-03-31 — burning ~$15/mo of idle
compute. Root cause: the sync Python process crashed with a memory
error, and `set -euo pipefail` aborted the script before `shutdown -h
now` could execute. Switched to a `trap` on EXIT so shutdown runs no
matter how the script terminates. The scheduler `divebar-sync-vm-daily`
is currently **paused** — resume it after CI deploys this fix.

Estimated total savings: ~$80/mo.

The historical version backlog has already been manually cleaned:
`dropbox-oauth-credentials`, `youtube-cookies`, `spotify-oauth-token`
are all at 5 versions. This PR prevents regrowth.

@coderabbitai ignore

## Test plan

- [x] Unit tests added for `_destroy_old_secret_versions`
(destroy-beyond-keep, swallow-list-errors,
swallow-per-version-destroy-errors, no-op-when-below-keep)
- [x] Full `test_credential_manager.py` passes (31 tests)
- [x] Full backend suite passes (3187 passed, 36 skipped)
- [ ] After deploy: verify the three secrets stay at ~5 versions over
the next refresh cycle
- [ ] After deploy: resume `divebar-sync-vm-daily` scheduler and confirm
next run auto-shuts-down VM

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.7 authored and GitHub committed 2026-05-16 16:43:21 -04:00
1 parent e41d6a6860
commit 53be57fe60
4 files changed
+141 -14

No files matched your search

+33 -5
View File
@@ -22,6 +22,30 @@ from backend.config import get_settings
logger = logging.getLogger(__name__)
def _destroy_old_secret_versions(client, secret_path: str, keep: int = 5) -> None:
"""Destroy ENABLED secret versions beyond the newest `keep`.
OAuth credential secrets are rewritten on every refresh, which would
otherwise accumulate hundreds of versions and cost ~$0.06/version/month
in replica storage. Only the newest version is read (`versions/latest`),
so older versions are dead weight.
"""
try:
versions = list(client.list_secret_versions(request={"parent": secret_path}))
enabled = [
v for v in versions
if v.state == v.State.ENABLED # type: ignore[attr-defined]
]
enabled.sort(key=lambda v: v.create_time, reverse=True)
for v in enabled[keep:]:
try:
client.destroy_secret_version(request={"name": v.name})
except Exception as e:
logger.warning(f"Failed to destroy {v.name}: {e}")
except Exception as e:
logger.warning(f"Failed to prune old secret versions for {secret_path}: {e}")
class CredentialStatus(str, Enum):
"""Status of OAuth credentials."""
VALID = "valid"
@@ -329,7 +353,9 @@ class CredentialManager:
parent=secret_path,
payload={"data": json.dumps(updated_data).encode("utf-8")}
)
_destroy_old_secret_versions(client, secret_path, keep=5)
logger.info(f"Updated {secret_name} with refreshed token")
return True
@@ -421,19 +447,21 @@ class CredentialManager:
"""Update stored Dropbox credentials."""
try:
from google.cloud import secretmanager
client = secretmanager.SecretManagerServiceClient()
project_id = self.settings.google_cloud_project
secret_path = f"projects/{project_id}/secrets/{self.DROPBOX_SECRET}"
client.add_secret_version(
parent=secret_path,
payload={"data": json.dumps(creds).encode("utf-8")}
)
_destroy_old_secret_versions(client, secret_path, keep=5)
logger.info("Updated Dropbox credentials with refreshed token")
return True
except Exception as e:
logger.error(f"Failed to update Dropbox credentials: {e}")
return False
+95 -1
View File
@@ -11,6 +11,7 @@ from backend.services.credential_manager import (
CredentialStatus,
CredentialCheckResult,
DeviceAuthInfo,
_destroy_old_secret_versions,
get_credential_manager,
)
@@ -369,9 +370,102 @@ class TestAuthRoutes:
class TestFileUploadCredentialValidation:
"""Tests for credential validation in file upload."""
def test_credential_manager_imported(self):
"""Test that credential manager is imported in file upload."""
from backend.api.routes.file_upload import get_credential_manager, CredentialStatus
assert get_credential_manager is not None
assert CredentialStatus is not None
class TestDestroyOldSecretVersions:
"""Tests for _destroy_old_secret_versions helper."""
def _make_version(self, name, state, create_time):
v = MagicMock()
v.name = name
v.state = state
v.create_time = create_time
return v
def test_destroys_versions_beyond_keep_count(self):
"""Keep newest N, destroy the rest."""
client = MagicMock()
enabled = MagicMock()
# State enum lookup on the version (v.State.ENABLED) — make .State.ENABLED a sentinel
# and match it on each version's `.state` field.
State = MagicMock()
State.ENABLED = "ENABLED"
def mk(name, ts):
v = MagicMock()
v.name = name
v.state = State.ENABLED
v.State = State # so v.state == v.State.ENABLED resolves correctly
v.create_time = ts
return v
# 7 versions, oldest → newest by create_time
versions = [mk(f"v{i}", i) for i in range(7)]
client.list_secret_versions.return_value = versions
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=3)
# Should destroy 4 oldest (v0..v3); keep v4, v5, v6
destroyed_names = [
c.kwargs["request"]["name"]
for c in client.destroy_secret_version.call_args_list
]
assert sorted(destroyed_names) == ["v0", "v1", "v2", "v3"]
def test_swallows_list_errors(self):
"""List failure must not raise — refresh path must keep working."""
client = MagicMock()
client.list_secret_versions.side_effect = RuntimeError("boom")
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=5)
client.destroy_secret_version.assert_not_called()
def test_swallows_per_version_destroy_errors(self):
"""One bad destroy doesn't abort the rest."""
client = MagicMock()
State = MagicMock()
State.ENABLED = "ENABLED"
def mk(name, ts):
v = MagicMock()
v.name = name
v.state = State.ENABLED
v.State = State
v.create_time = ts
return v
versions = [mk(f"v{i}", i) for i in range(7)]
client.list_secret_versions.return_value = versions
# First destroy fails, rest succeed
client.destroy_secret_version.side_effect = [
RuntimeError("nope"), None, None, None,
]
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=3)
# All 4 destroy calls attempted despite first failing
assert client.destroy_secret_version.call_count == 4
def test_does_nothing_when_below_keep_count(self):
"""If fewer than `keep` enabled versions exist, destroy nothing."""
client = MagicMock()
State = MagicMock()
State.ENABLED = "ENABLED"
def mk(name, ts):
v = MagicMock()
v.name = name
v.state = State.ENABLED
v.State = State
v.create_time = ts
return v
versions = [mk(f"v{i}", i) for i in range(3)]
client.list_secret_versions.return_value = versions
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=5)
client.destroy_secret_version.assert_not_called()
@@ -8,7 +8,7 @@
# Logs: journalctl -u divebar-sync -f
# OR: /var/log/divebar-sync.log
set -euo pipefail
set -uo pipefail
LOG_FILE="/var/log/divebar-sync.log"
REPO_DIR="/opt/nomad/divebar-sync"
@@ -18,6 +18,12 @@ log() {
echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG_FILE"
}
# Guarantee shutdown on ANY exit path (success, error, signal).
# Without this, a crashed sync (e.g. Python OOM, segfault) leaves the VM
# running 24/7 — previously cost ~$15/mo of idle compute. shutdown -h +1
# gives systemd 1min to flush logs before halt.
trap 'log "=== Exit code $?; shutting down VM ==="; shutdown -h +1' EXIT
log "=== Divebar file sync starting ==="
# Install dependencies (idempotent)
@@ -48,11 +54,10 @@ pip3 install -q --break-system-packages 'google-auth<2.28' google-api-python-cli
export GCE_METADATA_HOST="169.254.169.254"
export GOOGLE_CLOUD_PROJECT="nomadkaraoke"
# Run the sync
# Run the sync. `|| true` ensures the pipeline never blocks the EXIT trap
# (which is what guarantees shutdown). The exit code from python is logged
# by the trap.
log "Starting file sync (workers=4)..."
python3 "$SCRIPT" --workers 4 2>&1 | tee -a "$LOG_FILE"
python3 "$SCRIPT" --workers 4 2>&1 | tee -a "$LOG_FILE" || true
log "=== Divebar file sync script exited ==="
log "Shutting down VM after sync completion..."
shutdown -h now
log "=== Divebar file sync script finished ==="
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.174.9"
version = "0.174.10"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"