From 1878ad3c1d2b07a3f95c43469ab60b1ecbdc2dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 29 Oct 2026 03:25:11 -0400 Subject: [PATCH] feat(retention): keep lead-vocal stems as word-timing training data (v0.287.1) (#1234) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Storage retention purged `stems/lead_vocals.*` 30 days after completion. As a result, 660 of 865 completed jobs have no audio left to pair with their human-reviewed word timings (`lyrics/corrections_updated.json`, which is kept). The full-auto v2 timing work (an aligner fine-tuned on our data, plus a per-word timing verifier as the auto-ship gate) needs exactly that pair. Today's first bake-off had to make do with the 41 reviewed jobs that still had stems, and data volume was the limiting factor. The cost is about 15 MB per job; every other separated stem is still purged. ## Changes - `storage_retention.classify_file`: `stems/lead_vocals.*` (exact basename) is now KEEP with category `training_lead_vocals`. - Tests: the stem moves from the purge list to the keep list; a near-match (`lead_vocals_backup`) is still purged. - Docs: ARCHITECTURE retention list. ## Testing `backend/tests/test_storage_retention.py`: 133 passed. CodeRabbit ran locally; its one finding (exact-match the basename) is fixed. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVVVPsvPbXFhRYY5VSJ --------- Co-authored-by: Claude Opus 5.5 --- backend/services/storage_retention.py | 6 ++++++ backend/tests/test_storage_retention.py | 10 +++++++++- docs/ARCHITECTURE.md | 2 +- pyproject.toml | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/backend/services/storage_retention.py b/backend/services/storage_retention.py index 8a3bbb05..b64b30ed 100644 --- a/backend/services/storage_retention.py +++ b/backend/services/storage_retention.py @@ -15,6 +15,10 @@ KEEP forever — they hold user mute edits / uploads, which aren't stored anywhere else - ``stems/vocals_derived.*`` (cheap waveform helper of bring-your-own- instrumental jobs; separation doesn't recreate it) + - ``stems/lead_vocals.*`` (~15 MB): paired with the human-reviewed + ``lyrics/corrections_updated.json`` it is the training/eval data for word + timing (full-auto v2, 2026-10-29). Re-separating to recover it costs GPU + time per job, keeping it costs pennies. - anything not explicitly on the purge list (unknown files are kept) PURGE (regenerated on demand by backend/services/regenerate_service.py) @@ -127,6 +131,8 @@ def classify_file(rel_path: str, finalise_only: bool = False, stems_purgeable: b return (KEEP, "source_screen") if finalise_only else (PURGE, "screens_mov") return KEEP, "screens_image" if top == "stems": + if stem == "lead_vocals": + return KEEP, "training_lead_vocals" if stem.startswith(_KEPT_STEM_STEMS): return KEEP, "user_instrumental" if stem.startswith("custom_instrumental") else "stems_kept" if finalise_only: diff --git a/backend/tests/test_storage_retention.py b/backend/tests/test_storage_retention.py index 2e72184b..1ee7d331 100644 --- a/backend/tests/test_storage_retention.py +++ b/backend/tests/test_storage_retention.py @@ -54,6 +54,7 @@ class TestClassifyFile: "stems/custom_instrumental.flac", "stems/custom_instrumental.mp3", "stems/vocals_derived.flac", + "stems/lead_vocals.flac", "custom_instrumental.mp3", "custom_instrumental.wav", "existing_instrumental.flac", @@ -88,7 +89,6 @@ class TestClassifyFile: ("stems/instrumental_clean.flac", "stems"), ("stems/instrumental_with_backing.flac", "stems"), ("stems/vocals_clean.flac", "stems"), - ("stems/lead_vocals.flac", "stems"), ("stems/backing_vocals.flac", "stems"), ]) def test_purged(self, rel, category): @@ -890,3 +890,11 @@ class TestDeletionLogFlushFailure: report = svc.purge_orphans(dry_run=False, now=NOW) assert storage.deleted == ["jobs/g1/finals/a.mp4"] assert "deletion log not persisted" in report["errors"][0]["error"] + + +def test_lead_vocals_kept_as_training_data(): + # Word-timing training/eval data (full-auto v2): never purged, even when + # the other separated stems are. + assert classify_file("stems/lead_vocals.flac") == (KEEP, "training_lead_vocals") + assert classify_file("stems/instrumental_clean.flac") == (PURGE, "stems") + assert classify_file("stems/lead_vocals_backup.flac") == (PURGE, "stems") diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c4d8f006..5a169fb8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -310,7 +310,7 @@ jobs completed (newest `complete` timeline entry) more than `STORAGE_RETENTION_M - **Kept forever:** `input/*` (incl. `edited.flac`), `lyrics/`, `style/`, `review_sessions/`, `audio_edit*/`, `uploads/`, `packages/`, `analysis/`, `screens/*.png|jpg`, the 720p final (both the - descriptive and `lossy_720p_mp4.mp4` copies), `stems/custom_instrumental.*`, `stems/vocals_derived.*` + descriptive and `lossy_720p_mp4.mp4` copies), `stems/custom_instrumental.*`, `stems/vocals_derived.*`, `stems/lead_vocals.*` (word-timing training data, since v0.287.1) and the job-root `custom_instrumental.*` / `existing_instrumental.*`. Unknown files are kept. - **Purged:** other `finals/*`, `videos/*`, `previews/*`, `encoded/*`, `quick/*.mp4`, `screens/*.mov`, `review-audio/*`, and separated `stems/*` (only when the input audio is in `jobs/{id}/input/`). diff --git a/pyproject.toml b/pyproject.toml index 3886ddf4..ba6d60d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.287.0" +version = "0.287.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"