mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(retention): keep lead-vocal stems as word-timing training data (v0.287.1) (#1234)
## Summary Storage retention purged `stems/lead_vocals.*` 30 days after completion. As a result, 660 of 865 completed jobs have no audio left to pair with their human-reviewed word timings (`lyrics/corrections_updated.json`, which is kept). The full-auto v2 timing work (an aligner fine-tuned on our data, plus a per-word timing verifier as the auto-ship gate) needs exactly that pair. Today's first bake-off had to make do with the 41 reviewed jobs that still had stems, and data volume was the limiting factor. The cost is about 15 MB per job; every other separated stem is still purged. ## Changes - `storage_retention.classify_file`: `stems/lead_vocals.*` (exact basename) is now KEEP with category `training_lead_vocals`. - Tests: the stem moves from the purge list to the keep list; a near-match (`lead_vocals_backup`) is still purged. - Docs: ARCHITECTURE retention list. ## Testing `backend/tests/test_storage_retention.py`: 133 passed. CodeRabbit ran locally; its one finding (exact-match the basename) is fixed. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVVVPsvPbXFhRYY5VSJ --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bfcf1fd951
commit
1878ad3c1d
4 files changed
+17
-3
No files matched your search
@@ -15,6 +15,10 @@ KEEP forever
|
||||
— they hold user mute edits / uploads, which aren't stored anywhere else
|
||||
- ``stems/vocals_derived.*`` (cheap waveform helper of bring-your-own-
|
||||
instrumental jobs; separation doesn't recreate it)
|
||||
- ``stems/lead_vocals.*`` (~15 MB): paired with the human-reviewed
|
||||
``lyrics/corrections_updated.json`` it is the training/eval data for word
|
||||
timing (full-auto v2, 2026-10-29). Re-separating to recover it costs GPU
|
||||
time per job, keeping it costs pennies.
|
||||
- anything not explicitly on the purge list (unknown files are kept)
|
||||
|
||||
PURGE (regenerated on demand by backend/services/regenerate_service.py)
|
||||
@@ -127,6 +131,8 @@ def classify_file(rel_path: str, finalise_only: bool = False, stems_purgeable: b
|
||||
return (KEEP, "source_screen") if finalise_only else (PURGE, "screens_mov")
|
||||
return KEEP, "screens_image"
|
||||
if top == "stems":
|
||||
if stem == "lead_vocals":
|
||||
return KEEP, "training_lead_vocals"
|
||||
if stem.startswith(_KEPT_STEM_STEMS):
|
||||
return KEEP, "user_instrumental" if stem.startswith("custom_instrumental") else "stems_kept"
|
||||
if finalise_only:
|
||||
|
||||
@@ -54,6 +54,7 @@ class TestClassifyFile:
|
||||
"stems/custom_instrumental.flac",
|
||||
"stems/custom_instrumental.mp3",
|
||||
"stems/vocals_derived.flac",
|
||||
"stems/lead_vocals.flac",
|
||||
"custom_instrumental.mp3",
|
||||
"custom_instrumental.wav",
|
||||
"existing_instrumental.flac",
|
||||
@@ -88,7 +89,6 @@ class TestClassifyFile:
|
||||
("stems/instrumental_clean.flac", "stems"),
|
||||
("stems/instrumental_with_backing.flac", "stems"),
|
||||
("stems/vocals_clean.flac", "stems"),
|
||||
("stems/lead_vocals.flac", "stems"),
|
||||
("stems/backing_vocals.flac", "stems"),
|
||||
])
|
||||
def test_purged(self, rel, category):
|
||||
@@ -890,3 +890,11 @@ class TestDeletionLogFlushFailure:
|
||||
report = svc.purge_orphans(dry_run=False, now=NOW)
|
||||
assert storage.deleted == ["jobs/g1/finals/a.mp4"]
|
||||
assert "deletion log not persisted" in report["errors"][0]["error"]
|
||||
|
||||
|
||||
def test_lead_vocals_kept_as_training_data():
|
||||
# Word-timing training/eval data (full-auto v2): never purged, even when
|
||||
# the other separated stems are.
|
||||
assert classify_file("stems/lead_vocals.flac") == (KEEP, "training_lead_vocals")
|
||||
assert classify_file("stems/instrumental_clean.flac") == (PURGE, "stems")
|
||||
assert classify_file("stems/lead_vocals_backup.flac") == (PURGE, "stems")
|
||||
@@ -310,7 +310,7 @@ jobs completed (newest `complete` timeline entry) more than `STORAGE_RETENTION_M
|
||||
|
||||
- **Kept forever:** `input/*` (incl. `edited.flac`), `lyrics/`, `style/`, `review_sessions/`,
|
||||
`audio_edit*/`, `uploads/`, `packages/`, `analysis/`, `screens/*.png|jpg`, the 720p final (both the
|
||||
descriptive and `lossy_720p_mp4.mp4` copies), `stems/custom_instrumental.*`, `stems/vocals_derived.*`
|
||||
descriptive and `lossy_720p_mp4.mp4` copies), `stems/custom_instrumental.*`, `stems/vocals_derived.*`, `stems/lead_vocals.*` (word-timing training data, since v0.287.1)
|
||||
and the job-root `custom_instrumental.*` / `existing_instrumental.*`. Unknown files are kept.
|
||||
- **Purged:** other `finals/*`, `videos/*`, `previews/*`, `encoded/*`, `quick/*.mp4`, `screens/*.mov`,
|
||||
`review-audio/*`, and separated `stems/*` (only when the input audio is in `jobs/{id}/input/`).
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.287.0"
|
||||
version = "0.287.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user