mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: trim baseline GCP cost (secret retention + divebar shutdown) (#764)
## Summary Two cost-reduction fixes bundled together: 1. **Secret version retention** (`backend/services/credential_manager.py`): `dropbox-oauth-credentials` had accumulated 1191 enabled versions costing ~$65/mo in replica storage. Each refresh calls `add_secret_version` but never destroys the prior version. Same pattern for the YouTube/GDrive secrets refreshed via `_update_google_credentials`. Add a small `_destroy_old_secret_versions(keep=5)` helper called after every `add_secret_version` in this module. Errors swallowed so a prune failure cannot break credential refresh. 2. **Divebar-sync auto-shutdown** (`infrastructure/compute/startup_scripts/divebar_sync.sh`): VM was running 24/7 for 45 days since 2026-03-31 — burning ~$15/mo of idle compute. Root cause: the sync Python process crashed with a memory error, and `set -euo pipefail` aborted the script before `shutdown -h now` could execute. Switched to a `trap` on EXIT so shutdown runs no matter how the script terminates. The scheduler `divebar-sync-vm-daily` is currently **paused** — resume it after CI deploys this fix. Estimated total savings: ~$80/mo. The historical version backlog has already been manually cleaned: `dropbox-oauth-credentials`, `youtube-cookies`, `spotify-oauth-token` are all at 5 versions. This PR prevents regrowth. @coderabbitai ignore ## Test plan - [x] Unit tests added for `_destroy_old_secret_versions` (destroy-beyond-keep, swallow-list-errors, swallow-per-version-destroy-errors, no-op-when-below-keep) - [x] Full `test_credential_manager.py` passes (31 tests) - [x] Full backend suite passes (3187 passed, 36 skipped) - [ ] After deploy: verify the three secrets stay at ~5 versions over the next refresh cycle - [ ] After deploy: resume `divebar-sync-vm-daily` scheduler and confirm next run auto-shuts-down VM --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e41d6a6860
commit
53be57fe60
4 files changed
+141
-14
No files matched your search
@@ -22,6 +22,30 @@ from backend.config import get_settings
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _destroy_old_secret_versions(client, secret_path: str, keep: int = 5) -> None:
|
||||
"""Destroy ENABLED secret versions beyond the newest `keep`.
|
||||
|
||||
OAuth credential secrets are rewritten on every refresh, which would
|
||||
otherwise accumulate hundreds of versions and cost ~$0.06/version/month
|
||||
in replica storage. Only the newest version is read (`versions/latest`),
|
||||
so older versions are dead weight.
|
||||
"""
|
||||
try:
|
||||
versions = list(client.list_secret_versions(request={"parent": secret_path}))
|
||||
enabled = [
|
||||
v for v in versions
|
||||
if v.state == v.State.ENABLED # type: ignore[attr-defined]
|
||||
]
|
||||
enabled.sort(key=lambda v: v.create_time, reverse=True)
|
||||
for v in enabled[keep:]:
|
||||
try:
|
||||
client.destroy_secret_version(request={"name": v.name})
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to destroy {v.name}: {e}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to prune old secret versions for {secret_path}: {e}")
|
||||
|
||||
|
||||
class CredentialStatus(str, Enum):
|
||||
"""Status of OAuth credentials."""
|
||||
VALID = "valid"
|
||||
@@ -329,7 +353,9 @@ class CredentialManager:
|
||||
parent=secret_path,
|
||||
payload={"data": json.dumps(updated_data).encode("utf-8")}
|
||||
)
|
||||
|
||||
|
||||
_destroy_old_secret_versions(client, secret_path, keep=5)
|
||||
|
||||
logger.info(f"Updated {secret_name} with refreshed token")
|
||||
return True
|
||||
|
||||
@@ -421,19 +447,21 @@ class CredentialManager:
|
||||
"""Update stored Dropbox credentials."""
|
||||
try:
|
||||
from google.cloud import secretmanager
|
||||
|
||||
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
project_id = self.settings.google_cloud_project
|
||||
secret_path = f"projects/{project_id}/secrets/{self.DROPBOX_SECRET}"
|
||||
|
||||
|
||||
client.add_secret_version(
|
||||
parent=secret_path,
|
||||
payload={"data": json.dumps(creds).encode("utf-8")}
|
||||
)
|
||||
|
||||
|
||||
_destroy_old_secret_versions(client, secret_path, keep=5)
|
||||
|
||||
logger.info("Updated Dropbox credentials with refreshed token")
|
||||
return True
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update Dropbox credentials: {e}")
|
||||
return False
|
||||
|
||||
@@ -11,6 +11,7 @@ from backend.services.credential_manager import (
|
||||
CredentialStatus,
|
||||
CredentialCheckResult,
|
||||
DeviceAuthInfo,
|
||||
_destroy_old_secret_versions,
|
||||
get_credential_manager,
|
||||
)
|
||||
|
||||
@@ -369,9 +370,102 @@ class TestAuthRoutes:
|
||||
|
||||
class TestFileUploadCredentialValidation:
|
||||
"""Tests for credential validation in file upload."""
|
||||
|
||||
|
||||
def test_credential_manager_imported(self):
|
||||
"""Test that credential manager is imported in file upload."""
|
||||
from backend.api.routes.file_upload import get_credential_manager, CredentialStatus
|
||||
assert get_credential_manager is not None
|
||||
assert CredentialStatus is not None
|
||||
|
||||
|
||||
class TestDestroyOldSecretVersions:
|
||||
"""Tests for _destroy_old_secret_versions helper."""
|
||||
|
||||
def _make_version(self, name, state, create_time):
|
||||
v = MagicMock()
|
||||
v.name = name
|
||||
v.state = state
|
||||
v.create_time = create_time
|
||||
return v
|
||||
|
||||
def test_destroys_versions_beyond_keep_count(self):
|
||||
"""Keep newest N, destroy the rest."""
|
||||
client = MagicMock()
|
||||
enabled = MagicMock()
|
||||
# State enum lookup on the version (v.State.ENABLED) — make .State.ENABLED a sentinel
|
||||
# and match it on each version's `.state` field.
|
||||
State = MagicMock()
|
||||
State.ENABLED = "ENABLED"
|
||||
|
||||
def mk(name, ts):
|
||||
v = MagicMock()
|
||||
v.name = name
|
||||
v.state = State.ENABLED
|
||||
v.State = State # so v.state == v.State.ENABLED resolves correctly
|
||||
v.create_time = ts
|
||||
return v
|
||||
|
||||
# 7 versions, oldest → newest by create_time
|
||||
versions = [mk(f"v{i}", i) for i in range(7)]
|
||||
client.list_secret_versions.return_value = versions
|
||||
|
||||
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=3)
|
||||
|
||||
# Should destroy 4 oldest (v0..v3); keep v4, v5, v6
|
||||
destroyed_names = [
|
||||
c.kwargs["request"]["name"]
|
||||
for c in client.destroy_secret_version.call_args_list
|
||||
]
|
||||
assert sorted(destroyed_names) == ["v0", "v1", "v2", "v3"]
|
||||
|
||||
def test_swallows_list_errors(self):
|
||||
"""List failure must not raise — refresh path must keep working."""
|
||||
client = MagicMock()
|
||||
client.list_secret_versions.side_effect = RuntimeError("boom")
|
||||
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=5)
|
||||
client.destroy_secret_version.assert_not_called()
|
||||
|
||||
def test_swallows_per_version_destroy_errors(self):
|
||||
"""One bad destroy doesn't abort the rest."""
|
||||
client = MagicMock()
|
||||
State = MagicMock()
|
||||
State.ENABLED = "ENABLED"
|
||||
|
||||
def mk(name, ts):
|
||||
v = MagicMock()
|
||||
v.name = name
|
||||
v.state = State.ENABLED
|
||||
v.State = State
|
||||
v.create_time = ts
|
||||
return v
|
||||
|
||||
versions = [mk(f"v{i}", i) for i in range(7)]
|
||||
client.list_secret_versions.return_value = versions
|
||||
# First destroy fails, rest succeed
|
||||
client.destroy_secret_version.side_effect = [
|
||||
RuntimeError("nope"), None, None, None,
|
||||
]
|
||||
|
||||
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=3)
|
||||
# All 4 destroy calls attempted despite first failing
|
||||
assert client.destroy_secret_version.call_count == 4
|
||||
|
||||
def test_does_nothing_when_below_keep_count(self):
|
||||
"""If fewer than `keep` enabled versions exist, destroy nothing."""
|
||||
client = MagicMock()
|
||||
State = MagicMock()
|
||||
State.ENABLED = "ENABLED"
|
||||
|
||||
def mk(name, ts):
|
||||
v = MagicMock()
|
||||
v.name = name
|
||||
v.state = State.ENABLED
|
||||
v.State = State
|
||||
v.create_time = ts
|
||||
return v
|
||||
|
||||
versions = [mk(f"v{i}", i) for i in range(3)]
|
||||
client.list_secret_versions.return_value = versions
|
||||
|
||||
_destroy_old_secret_versions(client, "projects/p/secrets/s", keep=5)
|
||||
client.destroy_secret_version.assert_not_called()
|
||||
@@ -8,7 +8,7 @@
|
||||
# Logs: journalctl -u divebar-sync -f
|
||||
# OR: /var/log/divebar-sync.log
|
||||
|
||||
set -euo pipefail
|
||||
set -uo pipefail
|
||||
|
||||
LOG_FILE="/var/log/divebar-sync.log"
|
||||
REPO_DIR="/opt/nomad/divebar-sync"
|
||||
@@ -18,6 +18,12 @@ log() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Guarantee shutdown on ANY exit path (success, error, signal).
|
||||
# Without this, a crashed sync (e.g. Python OOM, segfault) leaves the VM
|
||||
# running 24/7 — previously cost ~$15/mo of idle compute. shutdown -h +1
|
||||
# gives systemd 1min to flush logs before halt.
|
||||
trap 'log "=== Exit code $?; shutting down VM ==="; shutdown -h +1' EXIT
|
||||
|
||||
log "=== Divebar file sync starting ==="
|
||||
|
||||
# Install dependencies (idempotent)
|
||||
@@ -48,11 +54,10 @@ pip3 install -q --break-system-packages 'google-auth<2.28' google-api-python-cli
|
||||
export GCE_METADATA_HOST="169.254.169.254"
|
||||
export GOOGLE_CLOUD_PROJECT="nomadkaraoke"
|
||||
|
||||
# Run the sync
|
||||
# Run the sync. `|| true` ensures the pipeline never blocks the EXIT trap
|
||||
# (which is what guarantees shutdown). The exit code from python is logged
|
||||
# by the trap.
|
||||
log "Starting file sync (workers=4)..."
|
||||
python3 "$SCRIPT" --workers 4 2>&1 | tee -a "$LOG_FILE"
|
||||
python3 "$SCRIPT" --workers 4 2>&1 | tee -a "$LOG_FILE" || true
|
||||
|
||||
log "=== Divebar file sync script exited ==="
|
||||
|
||||
log "Shutting down VM after sync completion..."
|
||||
shutdown -h now
|
||||
log "=== Divebar file sync script finished ==="
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.174.9"
|
||||
version = "0.174.10"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user