mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary
`uploads/` had a bucket lifecycle **delete after 7 days**, and user
inputs were only partly copied out of it. The only copy step was the
audio worker (stem separation) copying the mix to `jobs/{id}/input/`,
best-effort. Everything else stayed in `uploads/`:
- bring-your-own-instrumental / tenant jobs, which skip separation;
- the uploaded instrumental, which was only staged at render time;
- lyrics and style uploads, which were never copied.
Result:
- randy-vild re-renders 404'd on their own input (2026-10-08). That data
was restored from Andrew's Dropbox.
- Firestore shows 64 completed Jan–Feb 2026 consumer jobs, plus
BYO-instrumental jobs, whose inputs are gone permanently.
## Changes
- **`backend/services/input_persistence.py`** (new):
- Server-side copies `uploads/{id}/**` to `jobs/{id}/input/**` and
repoints every job field that referenced the upload:
`input_media_gcs_path`, `existing_instrumental_gcs_path`,
`lyrics_file_gcs_path`, `style_params_gcs_path`, plus any matching value
under `file_urls` / `state_data` / `style_assets`.
- Idempotent: unchanged copies are skipped.
- The audio, lyrics, screens, render-video and video workers all call it
on entry, via `asyncio.to_thread`. On failure it logs an ERROR, which
alerts, but doesn't fail the job.
- **`POST /api/admin/persist-uploads`** (dry run by default): backfill
for jobs no worker will touch again. Lookup failures are isolated per
job.
- **GCE encoder:** prefers the explicitly downloaded top-level
`existing_instrumental.*`. The recursive glob could otherwise return the
raw, unconformed copy now kept under `input/audio/`.
- **Pulumi:** `uploads/` lifecycle goes from 7 to **180 days**; `temp/`
stays at 7. **Already applied** and verified directly on the bucket.
- **Docs:** LESSONS-LEARNED, DISASTER-RECOVERY (new user-inputs row; the
stale "30d noncurrent" figure corrected to 7d), API, README.
## Verification
- `make test` all green.
- 16 new tests:
- copy mapping, repointing, idempotency, re-copy when a kept copy
differs, already-expired paths left alone;
- the worker wrapper never raises;
- an AST guard that every pipeline worker calls the copy step via
`to_thread`;
- backfill dry run / apply / error isolation;
- the encoder's instrumental preference.
- Local CodeRabbit: 2 findings, both fixed (lyrics path, per-job
lookup); re-review clean.
## After merge
Run `POST /api/admin/persist-uploads` (dry run, then `?dry_run=false`)
to copy the inputs of the 51 jobs that still have objects in `uploads/`.
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
256 lines
11 KiB
Python
256 lines
11 KiB
Python
"""
|
|
User-uploaded inputs must outlive the uploads/ lifecycle delete.
|
|
|
|
2026-10-08: randy-vild jobs that skipped the audio worker's copy (bring-your-own
|
|
instrumental) or sat in review for >7 days lost their mix and instrumental, so
|
|
re-renders failed with a 404 on uploads/... . Every pipeline worker now copies
|
|
uploads/{job_id}/** to jobs/{job_id}/input/ on entry and repoints the job.
|
|
"""
|
|
import ast
|
|
import hashlib
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from backend.services.input_persistence import (
|
|
ensure_job_inputs_persisted,
|
|
persist_job_inputs,
|
|
persisted_path,
|
|
)
|
|
|
|
|
|
class FakeBlob:
|
|
def __init__(self, name, data):
|
|
self.name = name
|
|
self.data = data
|
|
self.size = len(data)
|
|
self.md5_hash = hashlib.md5(data).hexdigest()
|
|
|
|
|
|
class FakeBucket:
|
|
def __init__(self, objects):
|
|
self.objects = {n: FakeBlob(n, d) for n, d in objects.items()}
|
|
self.copies = []
|
|
|
|
def list_blobs(self, prefix=""):
|
|
return [b for n, b in sorted(self.objects.items()) if n.startswith(prefix)]
|
|
|
|
def get_blob(self, name):
|
|
return self.objects.get(name)
|
|
|
|
def copy_blob(self, blob, bucket, dest):
|
|
self.copies.append((blob.name, dest))
|
|
self.objects[dest] = FakeBlob(dest, blob.data)
|
|
|
|
|
|
def _job(**fields):
|
|
base = dict(
|
|
job_id="j1",
|
|
input_media_gcs_path="uploads/j1/audio/01-Song Mix.wav",
|
|
existing_instrumental_gcs_path="uploads/j1/conformed/existing_instrumental.flac",
|
|
style_params_gcs_path=None,
|
|
lyrics_file_gcs_path="uploads/j1/lyrics/user_lyrics.txt",
|
|
file_urls={"input": {"audio": "uploads/j1/audio/01-Song Mix.wav"}, "stems": {"x": "jobs/j1/stems/x.flac"}},
|
|
state_data={"instrumental_conformed": {"original_gcs_path": "uploads/j1/audio/existing_instrumental.wav", "correlation": 0.8}},
|
|
style_assets={},
|
|
)
|
|
base.update(fields)
|
|
return SimpleNamespace(**base)
|
|
|
|
|
|
UPLOADS = {
|
|
"uploads/j1/audio/01-Song Mix.wav": b"mix",
|
|
"uploads/j1/audio/existing_instrumental.wav": b"inst",
|
|
"uploads/j1/conformed/existing_instrumental.flac": b"conformed",
|
|
"uploads/j1/lyrics/user_lyrics.txt": b"la la",
|
|
"uploads/j10/audio/other.wav": b"another job", # prefix-sibling must not be touched
|
|
}
|
|
|
|
|
|
def test_persisted_path_keeps_relative_layout_and_only_maps_this_job():
|
|
assert persisted_path("j1", "uploads/j1/audio/a b.wav") == "jobs/j1/input/audio/a b.wav"
|
|
assert persisted_path("j1", "uploads/j10/audio/x.wav") is None
|
|
assert persisted_path("j1", "jobs/j1/input/x.wav") is None
|
|
assert persisted_path("j1", "uploads/j1/") is None
|
|
|
|
|
|
def test_copies_every_upload_and_repoints_every_reference():
|
|
bucket = FakeBucket(UPLOADS)
|
|
jm = MagicMock()
|
|
job = _job()
|
|
copied, updates = persist_job_inputs(job, SimpleNamespace(bucket=bucket), jm)
|
|
|
|
assert copied == 4
|
|
assert sorted(dest for _src, dest in bucket.copies) == [
|
|
"jobs/j1/input/audio/01-Song Mix.wav",
|
|
"jobs/j1/input/audio/existing_instrumental.wav",
|
|
"jobs/j1/input/conformed/existing_instrumental.flac",
|
|
"jobs/j1/input/lyrics/user_lyrics.txt",
|
|
]
|
|
assert updates == {
|
|
"input_media_gcs_path": "jobs/j1/input/audio/01-Song Mix.wav",
|
|
"existing_instrumental_gcs_path": "jobs/j1/input/conformed/existing_instrumental.flac",
|
|
"lyrics_file_gcs_path": "jobs/j1/input/lyrics/user_lyrics.txt",
|
|
"file_urls.input.audio": "jobs/j1/input/audio/01-Song Mix.wav",
|
|
"state_data.instrumental_conformed.original_gcs_path": "jobs/j1/input/audio/existing_instrumental.wav",
|
|
}
|
|
jm.update_job.assert_called_once_with("j1", updates)
|
|
# in-memory job follows, so the calling worker downloads from the kept copy
|
|
assert job.input_media_gcs_path == "jobs/j1/input/audio/01-Song Mix.wav"
|
|
assert job.existing_instrumental_gcs_path.startswith("jobs/j1/input/")
|
|
|
|
|
|
def test_idempotent_second_run_copies_nothing():
|
|
bucket = FakeBucket(UPLOADS)
|
|
jm = MagicMock()
|
|
persist_job_inputs(_job(), SimpleNamespace(bucket=bucket), jm)
|
|
bucket.copies.clear()
|
|
jm.reset_mock()
|
|
# Job already repointed (as Firestore would now hold)
|
|
job = _job(
|
|
input_media_gcs_path="jobs/j1/input/audio/01-Song Mix.wav",
|
|
existing_instrumental_gcs_path="jobs/j1/input/conformed/existing_instrumental.flac",
|
|
lyrics_file_gcs_path="jobs/j1/input/lyrics/user_lyrics.txt",
|
|
file_urls={"input": {"audio": "jobs/j1/input/audio/01-Song Mix.wav"}},
|
|
state_data={},
|
|
)
|
|
copied, updates = persist_job_inputs(job, SimpleNamespace(bucket=bucket), jm)
|
|
assert (copied, updates) == (0, {})
|
|
assert bucket.copies == []
|
|
jm.update_job.assert_not_called()
|
|
|
|
|
|
def test_recopies_when_the_kept_copy_differs():
|
|
"""A re-upload under the same name (e.g. a new conformed instrumental) replaces the kept copy."""
|
|
bucket = FakeBucket({**UPLOADS, "jobs/j1/input/conformed/existing_instrumental.flac": b"stale"})
|
|
persist_job_inputs(_job(), SimpleNamespace(bucket=bucket), MagicMock())
|
|
assert bucket.objects["jobs/j1/input/conformed/existing_instrumental.flac"].data == b"conformed"
|
|
|
|
|
|
def test_paths_to_already_expired_uploads_are_left_alone():
|
|
bucket = FakeBucket({}) # nothing left in uploads/
|
|
jm = MagicMock()
|
|
copied, updates = persist_job_inputs(_job(), SimpleNamespace(bucket=bucket), jm)
|
|
assert (copied, updates) == (0, {})
|
|
jm.update_job.assert_not_called()
|
|
|
|
|
|
def test_style_assets_and_style_params_are_repointed():
|
|
bucket = FakeBucket({"uploads/j1/style/style_params.json": b"{}", "uploads/j1/style/bg.png": b"png"})
|
|
job = _job(
|
|
input_media_gcs_path=None, existing_instrumental_gcs_path=None, lyrics_file_gcs_path=None,
|
|
file_urls={}, state_data={}, style_params_gcs_path="uploads/j1/style/style_params.json",
|
|
style_assets={"karaoke_background": "uploads/j1/style/bg.png", "font": "themes/nomad/assets/A.ttf"},
|
|
)
|
|
_copied, updates = persist_job_inputs(job, SimpleNamespace(bucket=bucket), MagicMock())
|
|
assert updates == {
|
|
"style_params_gcs_path": "jobs/j1/input/style/style_params.json",
|
|
"style_assets.karaoke_background": "jobs/j1/input/style/bg.png",
|
|
}
|
|
|
|
|
|
def test_worker_wrapper_never_raises_but_logs_an_error(caplog):
|
|
storage = SimpleNamespace(bucket=MagicMock(list_blobs=MagicMock(side_effect=RuntimeError("gcs down"))))
|
|
with caplog.at_level("ERROR"):
|
|
ensure_job_inputs_persisted(_job(), storage, MagicMock(), "lyrics")
|
|
assert any("failed to persist uploaded inputs" in r.message and "gcs down" in r.message for r in caplog.records)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"module",
|
|
["audio_worker", "lyrics_worker", "screens_worker", "render_video_worker", "video_worker"],
|
|
)
|
|
def test_every_pipeline_worker_persists_inputs_off_the_event_loop(module):
|
|
"""Each worker entry calls the persistence step via asyncio.to_thread (blocking GCS/Firestore I/O)."""
|
|
source = Path(__file__).resolve().parents[1].joinpath("workers", f"{module}.py").read_text()
|
|
calls = [
|
|
node for node in ast.walk(ast.parse(source))
|
|
if isinstance(node, ast.Call)
|
|
and getattr(node.func, "attr", None) == "to_thread"
|
|
and node.args and getattr(node.args[0], "id", None) == "ensure_job_inputs_persisted"
|
|
]
|
|
assert calls, f"{module} must call ensure_job_inputs_persisted on entry"
|
|
|
|
|
|
# --- admin backfill -------------------------------------------------------------------
|
|
|
|
|
|
def _admin_client():
|
|
from backend.api.dependencies import AuthResult, UserType, require_admin
|
|
from backend.api.routes.admin import router
|
|
|
|
app = FastAPI()
|
|
app.include_router(router, prefix="/api")
|
|
app.dependency_overrides[require_admin] = lambda: AuthResult(
|
|
is_valid=True, user_type=UserType.ADMIN, remaining_uses=999, message="ok",
|
|
user_email="admin@example.com", is_admin=True,
|
|
)
|
|
return TestClient(app)
|
|
|
|
|
|
def test_backfill_dry_run_reports_without_copying():
|
|
bucket = FakeBucket({**UPLOADS, "uploads/gone/audio/x.wav": b"orphan"})
|
|
jm = MagicMock()
|
|
jm.get_job.side_effect = lambda jid: _job() if jid == "j1" else (_job(job_id="j10") if jid == "j10" else None)
|
|
with patch("backend.api.routes.admin.StorageService", return_value=SimpleNamespace(bucket=bucket)), \
|
|
patch("backend.api.routes.admin.JobManager", return_value=jm):
|
|
resp = _admin_client().post("/api/admin/persist-uploads")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["dry_run"] is True
|
|
assert body["jobs"]["gone"] == {"orphan": 1}
|
|
assert body["jobs"]["j1"]["would_copy"]["uploads/j1/lyrics/user_lyrics.txt"] == "jobs/j1/input/lyrics/user_lyrics.txt"
|
|
assert bucket.copies == []
|
|
jm.update_job.assert_not_called()
|
|
|
|
|
|
def test_backfill_apply_persists_each_job():
|
|
bucket = FakeBucket(UPLOADS)
|
|
jm = MagicMock()
|
|
jm.get_job.side_effect = lambda jid: _job(job_id=jid, input_media_gcs_path=f"uploads/{jid}/audio/other.wav",
|
|
existing_instrumental_gcs_path=None, lyrics_file_gcs_path=None,
|
|
file_urls={}, state_data={}) \
|
|
if jid == "j10" else _job()
|
|
with patch("backend.api.routes.admin.StorageService", return_value=SimpleNamespace(bucket=bucket)), \
|
|
patch("backend.api.routes.admin.JobManager", return_value=jm):
|
|
body = _admin_client().post("/api/admin/persist-uploads?dry_run=false").json()
|
|
assert body["jobs"]["j1"]["copied"] == 4
|
|
assert body["jobs"]["j10"] == {"copied": 1, "updated": {"input_media_gcs_path": "jobs/j10/input/audio/other.wav"}}
|
|
|
|
|
|
# --- GCE encoder picks the configured instrumental, not a persisted raw copy -----------
|
|
|
|
|
|
def test_encoder_prefers_the_downloaded_existing_instrumental(tmp_path):
|
|
from backend.services.gce_encoding.main import resolve_instrumental
|
|
|
|
(tmp_path / "input" / "audio").mkdir(parents=True)
|
|
(tmp_path / "input" / "conformed").mkdir(parents=True)
|
|
(tmp_path / "input" / "audio" / "existing_instrumental.wav").write_bytes(b"raw, unaligned")
|
|
(tmp_path / "input" / "conformed" / "existing_instrumental.flac").write_bytes(b"aligned")
|
|
(tmp_path / "existing_instrumental.flac").write_bytes(b"configured")
|
|
picked = resolve_instrumental(tmp_path, {"existing_instrumental": "jobs/j1/input/conformed/existing_instrumental.flac"})
|
|
assert picked == tmp_path / "existing_instrumental.flac"
|
|
|
|
|
|
def test_backfill_keeps_going_when_one_job_lookup_fails():
|
|
bucket = FakeBucket(UPLOADS)
|
|
jm = MagicMock()
|
|
|
|
def get_job(jid):
|
|
if jid == "j1":
|
|
raise RuntimeError("firestore hiccup")
|
|
return _job(job_id=jid, input_media_gcs_path=f"uploads/{jid}/audio/other.wav",
|
|
existing_instrumental_gcs_path=None, lyrics_file_gcs_path=None, file_urls={}, state_data={})
|
|
|
|
jm.get_job.side_effect = get_job
|
|
with patch("backend.api.routes.admin.StorageService", return_value=SimpleNamespace(bucket=bucket)), \
|
|
patch("backend.api.routes.admin.JobManager", return_value=jm):
|
|
body = _admin_client().post("/api/admin/persist-uploads?dry_run=false").json()
|
|
assert body["jobs"]["j1"] == {"error": "firestore hiccup"}
|
|
assert body["jobs"]["j10"]["copied"] == 1
|