diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2259e015..e9da90de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2027,7 +2027,7 @@ jobs: --max-instances 20 \ --min-instances 0 \ --clear-vpc-connector \ - --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,USE_CLOUD_RUN_JOBS_FOR_RENDER=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-5-5,AUTO_CORRECT_COMPARE_MODELS=claude-opus-5-5;gemini-3.8-flash,AUTO_CORRECT_PROACTIVE_ENABLED=true,AUTO_APPROVAL_BACKING_KEEP_ENABLED=true,COMMUNITY_DAILY_PICK_ENABLED=true,EDGE_AUTH_MODE=enforce,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \ + --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,USE_CLOUD_RUN_JOBS_FOR_RENDER=true,EMAIL_PROVIDER=postmark,AUTO_CORRECT_MODEL=claude-opus-5-5,AUTO_CORRECT_COMPARE_MODELS=claude-opus-5-5;gemini-3.8-flash,AUTO_CORRECT_PROACTIVE_ENABLED=true,AUTO_APPROVAL_BACKING_KEEP_ENABLED=true,COMMUNITY_DAILY_PICK_ENABLED=true,STORAGE_RETENTION_DRY_RUN=false,STORAGE_RETENTION_MAX_JOBS_PER_RUN=50,EDGE_AUTH_MODE=enforce,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \ --set-secrets "ANTHROPIC_API_KEY=anthropic-api-key:latest,GEMINI_API_KEY=gemini-api-key:latest,AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,ENCODING_WORKER_FALLBACK_VMS=encoding-worker-fallback-vms:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest,EDGE_ORIGIN_SECRET=edge-origin-secret:latest,KARAOKEHUNT_FORWARDER_SECRET=karaokehunt-forwarder-secret:latest,KJBOX_PARTNER_SECRET=kjbox-partner-secret:latest" # R3/R4 — smoke the tagged candidate revision BEFORE it serves any traffic. @@ -2116,6 +2116,15 @@ jobs: --project ${{ env.PROJECT_ID }} \ --quiet + # Storage-retention job (CPU, us-east4): pin to this commit's image so + # the deletion code (generation-pinned deletes + audit log) always + # matches the API that starts it. + gcloud run jobs update storage-retention-job \ + --image "$CPU_JOBS_IMAGE" \ + --region us-east4 \ + --project ${{ env.PROJECT_ID }} \ + --quiet + # Audio separation job (GPU, us-east4) — uses GPU image gcloud run jobs update audio-separation-job \ --image "$GPU_IMAGE" \ diff --git a/backend/api/routes/internal.py b/backend/api/routes/internal.py index eb553a7e..1065b80c 100644 --- a/backend/api/routes/internal.py +++ b/backend/api/routes/internal.py @@ -776,6 +776,50 @@ async def storage_retention_run_endpoint( } +ORPHAN_DELETE_CONFIRMATION = "delete-orphan-job-folders" + + +@router.post("/storage-retention/orphans") +async def storage_retention_orphans_endpoint( + dry_run: bool = True, + confirm: Optional[str] = None, + max_folders: Optional[int] = None, + auth_data: Tuple[str, UserType, int] = Depends(require_admin), +): + """ + One-off cleanup of ``jobs/{id}/`` folders with no job record: keep ``input/``, + delete everything else (backend/services/storage_retention.py purge_orphans). + + DRY-RUN unless ``dry_run=false&confirm=delete-orphan-job-folders``. Independent + of STORAGE_RETENTION_DRY_RUN and never run by the scheduler. Runs as the + storage-retention-job Cloud Run Job; the report path is returned. + """ + from datetime import datetime, timezone + from backend.config import get_settings + from backend.services.storage_retention import REPORT_PREFIX + from backend.services.worker_service import get_worker_service + + if not dry_run and confirm != ORPHAN_DELETE_CONFIRMATION: + raise HTTPException( + status_code=400, + detail=f"A real orphan cleanup needs confirm={ORPHAN_DELETE_CONFIRMATION}", + ) + settings = get_settings() + now = datetime.now(timezone.utc) + report_path = f"{REPORT_PREFIX}/{now.strftime('%Y%m%dT%H%M%SZ')}-orphans-{'dry-run' if dry_run else 'run'}.json" + started = await get_worker_service().trigger_storage_retention_job( + dry_run=dry_run, report_path=report_path, mode="orphans", max_folders=max_folders, + ) + if not started: + raise HTTPException(status_code=503, detail="Couldn't start the storage-retention job") + return { + "status": "started", + "mode": "orphans", + "dry_run": dry_run, + "report_path": f"gs://{settings.gcs_bucket_name}/{report_path}", + } + + @router.post("/community-daily-pick") async def community_daily_pick_endpoint( http_request: Request, diff --git a/backend/services/storage_retention.py b/backend/services/storage_retention.py index c271d61b..8a3bbb05 100644 --- a/backend/services/storage_retention.py +++ b/backend/services/storage_retention.py @@ -52,6 +52,16 @@ logger = logging.getLogger(__name__) POLICY_VERSION = 1 REPORT_PREFIX = "storage-retention/reports" +# Audit trail of every object deleted (kept forever; never touched by any purge +# or lifecycle rule — see docs/STORAGE-RETENTION.md). +DELETION_LOG_PREFIX = "storage-retention/deletion-logs" +MODE_JOB_PURGE = "job_purge" +MODE_ORPHAN = "orphan" +ORPHAN_QUIET_DAYS = 30 +# Firestore collections whose doc id == the jobs/{id}/ folder name and that mean +# "this folder still belongs to something". encoding_worker_jobs is NOT one: it's +# the encoder's expiring result cache, left behind by deleted jobs. +ORPHAN_RECORD_COLLECTIONS = ("jobs", "jobs-dev", "youtube_upload_queue") STATE_COLLECTION = "storage_retention" STATE_DOC = "state" SCAN_PAGE_SIZE = 200 @@ -234,6 +244,7 @@ class JobPlan: stems_purgeable: bool = True stems_note: Optional[str] = None skip_reason: Optional[str] = None + generations: Dict[str, Any] = field(default_factory=dict) @property def purge_bytes(self) -> int: @@ -254,12 +265,26 @@ class JobPlan: return any(c in STEM_CATEGORIES for _, _, c in self.purge) -def plan_job(job: Dict[str, Any], listing: List[Tuple[str, int]]) -> JobPlan: +def _listing_entries(listing) -> List[Tuple[str, int, Any]]: + """Normalise a listing of ``(name, size)``, ``(name, size, gen)`` or meta dicts.""" + out = [] + for item in listing: + if isinstance(item, dict): + out.append((item["name"], int(item.get("size") or 0), item.get("generation"))) + elif len(item) >= 3: + out.append((item[0], int(item[1] or 0), item[2])) + else: + out.append((item[0], int(item[1] or 0), None)) + return out + + +def plan_job(job: Dict[str, Any], listing) -> JobPlan: """Decide what to purge for one (already eligible) job from its GCS listing.""" job_id = job["job_id"] prefix = f"jobs/{job_id}/" plan = JobPlan(job_id=job_id) - names = {name for name, _ in listing} + entries = _listing_entries(listing) + names = {name for name, _, _ in entries} # Re-rendering needs the input audio (render worker + separation). If it's # gone (e.g. pre-persistence jobs whose input expired from uploads/), nothing @@ -276,7 +301,7 @@ def plan_job(job: Dict[str, Any], listing: List[Tuple[str, int]]) -> JobPlan: plan.stems_purgeable = False plan.stems_note = "existing_instrumental" - for name, size in listing: + for name, size, generation in entries: if not name.startswith(prefix): # defence in depth: never outside jobs/{id}/ continue action, category = classify_file( @@ -284,11 +309,152 @@ def plan_job(job: Dict[str, Any], listing: List[Tuple[str, int]]) -> JobPlan: ) if action == PURGE: plan.purge.append((name, size, category)) + plan.generations[name] = generation else: plan.kept_bytes += size return plan +def assert_deletable(path: str) -> None: + """Hard guard: retention code only ever deletes inside a jobs/{id}/ folder.""" + parts = path.split("/") + if len(parts) < 3 or parts[0] != "jobs" or not parts[1] or path.startswith(DELETION_LOG_PREFIX): + raise ValueError(f"refusing to delete outside jobs/{{id}}/: {path}") + + +def log_category(category: str) -> str: + """Category label used in the deletion log (finals, review-audio, screens-mov, ...).""" + return category.replace("_", "-") + + +class DeletionLog: + """Append-only JSONL audit log of one run's deletions, flushed to GCS in batches. + + ``storage-retention/deletion-logs/-.jsonl`` holds one line per + attempted delete; it is re-uploaded (whole) after every batch so a crash + mid-run still leaves a complete record of what was deleted up to the last + flush (the batch in flight is at most ``flush_every`` lines / one job). + """ + + def __init__(self, storage, run_ts: datetime, mode: str, flush_every: int = 200): + self.storage = storage + self.mode = mode + self.run_id = f"{run_ts.strftime('%Y%m%dT%H%M%SZ')}-{mode}" + self.path = f"{DELETION_LOG_PREFIX}/{self.run_id}.jsonl" + self.summary_path = f"{DELETION_LOG_PREFIX}/{self.run_id}.summary.json" + self.flush_every = flush_every + self._lines: List[str] = [] + self._unflushed = 0 + self.totals: Counter = Counter() + self.counts: Counter = Counter() + self.failures = 0 + self.jobs: set = set() + # Set when the audit log couldn't be written: callers stop deleting. + self.flush_error: Optional[str] = None + + def record(self, job_id: str, path: str, size: int, generation, category: str, + result: str, error: Optional[str] = None) -> None: + import json + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "run_id": self.run_id, + "mode": self.mode, + "job_id": job_id, + "path": path, + "size_bytes": int(size or 0), + "generation": str(generation) if generation is not None else None, + "category": category, + "result": result, + } + if error: + entry["error"] = error[:500] + self._lines.append(json.dumps(entry, ensure_ascii=False)) + self._unflushed += 1 + if result == "deleted": + self.totals[category] += int(size or 0) + self.counts[category] += 1 + self.jobs.add(job_id) + elif result == "failed": + self.failures += 1 + if self._unflushed >= self.flush_every: + self.flush() + + def flush(self) -> None: + if not self._lines: + return + try: + self.storage.upload_text(self.path, "\n".join(self._lines) + "\n", content_type="application/x-ndjson") + self._unflushed = 0 + self.flush_error = None + except Exception as e: # noqa: BLE001 - recorded; the run stops deleting + self.flush_error = str(e) + logger.error(f"storage retention: deletion log flush failed ({self.path}): {e}") + + def write_summary(self, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + summary = { + "run_id": self.run_id, + "mode": self.mode, + "deletion_log": self.path, + "objects_deleted": sum(self.counts.values()), + "jobs": len(self.jobs), + "bytes_deleted": sum(self.totals.values()), + "gib_deleted": round(sum(self.totals.values()) / 2**30, 3), + "bytes_by_category": dict(self.totals), + "objects_by_category": dict(self.counts), + "failures": self.failures, + "log_flush_error": self.flush_error, + "written_at": datetime.now(timezone.utc).isoformat(), + **(extra or {}), + } + self.flush() + try: + self.storage.upload_json(self.summary_path, summary) + except Exception as e: # noqa: BLE001 + logger.error(f"storage retention: summary write failed ({self.summary_path}): {e}") + return summary + + +def orphan_candidates(entries: Iterable[Dict[str, Any]], known_ids: set) -> Dict[str, List[Dict[str, Any]]]: + """Group ``jobs/{id}/...`` objects whose id has no job record.""" + folders: Dict[str, List[Dict[str, Any]]] = {} + for meta in entries: + parts = meta["name"].split("/") + if len(parts) < 3 or parts[0] != "jobs" or not parts[1]: + continue + if parts[1] in known_ids: + continue + folders.setdefault(parts[1], []).append(meta) + return folders + + +def plan_orphan_folder(job_id: str, objects: List[Dict[str, Any]], now: datetime, + quiet_days: int = ORPHAN_QUIET_DAYS) -> Dict[str, Any]: + """Keep ``input/``; everything else in an orphan folder is deleted (if quiet).""" + prefix = f"jobs/{job_id}/" + latest = None + for meta in objects: + updated = _parse_ts(meta.get("updated")) + if updated and (latest is None or updated > latest): + latest = updated + keep, purge = [], [] + for meta in objects: + if not meta["name"].startswith(prefix): + continue + (keep if meta["name"][len(prefix):].startswith("input/") else purge).append(meta) + skip = None + if latest is None or now - latest < timedelta(days=quiet_days): + skip = "recently_modified" + return { + "job_id": job_id, + "latest_update": latest.isoformat() if latest else None, + "skip_reason": skip, + "purge": purge, + "keep": keep, + "purge_bytes": sum(int(m.get("size") or 0) for m in purge), + "keep_bytes": sum(int(m.get("size") or 0) for m in keep), + } + + def file_url_deletions(file_urls: Dict[str, Any], purged_paths: Iterable[str]) -> List[str]: """Firestore field paths under ``file_urls`` that reference purged blobs.""" purged = set(purged_paths) @@ -395,6 +561,10 @@ class StorageRetentionService: "errors": [], } totals: Counter = Counter() + deletion_log = None if dry_run else DeletionLog(self.storage, now, MODE_JOB_PURGE) + if deletion_log: + report["deletion_log"] = deletion_log.path + report["deletion_summary"] = deletion_log.summary_path processed = 0 scanned = 0 last_id = None @@ -413,7 +583,7 @@ class StorageRetentionService: report["skipped"][reason] += 1 continue try: - listing = self.storage.list_files_with_sizes(f"jobs/{job['job_id']}/") + listing = self.storage.list_blob_meta(f"jobs/{job['job_id']}/") plan = plan_job(job, listing) except Exception as e: # noqa: BLE001 - one bad job never stops the pass logger.exception(f"[job:{job.get('job_id')}] storage retention: planning failed") @@ -438,12 +608,18 @@ class StorageRetentionService: } if not dry_run: try: - entry["result"] = self._execute(job["job_id"], plan, now, min_age, excluded) + entry["result"] = self._execute(job["job_id"], plan, now, min_age, excluded, deletion_log) except Exception as e: # noqa: BLE001 logger.exception(f"[job:{plan.job_id}] storage retention: purge failed") entry["result"] = {"status": "error", "error": str(e)} report["errors"].append({"job_id": plan.job_id, "error": str(e)}) report["jobs"].append(entry) + if deletion_log and deletion_log.flush_error: + # The audit log is a required safety record: no more deletes. + report["errors"].append({"job_id": plan.job_id, + "error": f"deletion log not persisted, run stopped: {deletion_log.flush_error}"}) + exhausted = False + break if not dry_run and entry["result"].get("status") != "purged": continue processed += 1 @@ -452,6 +628,9 @@ class StorageRetentionService: if use_cursor: self._save_cursor(None if exhausted else last_id) + if deletion_log: + report["deletion_totals"] = deletion_log.write_summary( + {"report_path": report_path or self.default_report_path(now, dry_run)}) report["skipped"] = dict(report["skipped"]) report["summary"] = { @@ -593,7 +772,8 @@ class StorageRetentionService: return _run_in_transaction(self.db, claim) - def _execute(self, job_id: str, plan: JobPlan, now: datetime, min_age: int, excluded: List[str]) -> Dict[str, Any]: + def _execute(self, job_id: str, plan: JobPlan, now: datetime, min_age: int, excluded: List[str], + deletion_log: Optional["DeletionLog"] = None) -> Dict[str, Any]: from google.cloud.firestore_v1 import DELETE_FIELD if not self._claim(job_id, plan, now, min_age, excluded): @@ -606,11 +786,19 @@ class StorageRetentionService: for path, size, category in plan.purge: if not path.startswith(f"jobs/{job_id}/"): continue + generation = plan.generations.get(path) try: - self.storage.delete_file(path, ignore_missing=True) + assert_deletable(path) + self.storage.delete_file(path, ignore_missing=True, if_generation_match=generation) deleted.append((path, size, category)) + if deletion_log: + deletion_log.record(job_id, path, size, generation, log_category(category), "deleted") except Exception as e: # noqa: BLE001 failures.append({"path": path, "error": str(e)}) + if deletion_log: + deletion_log.record(job_id, path, size, generation, log_category(category), "failed", str(e)) + if deletion_log: + deletion_log.flush() snap = ref.get() file_urls = (snap.to_dict() or {}).get("file_urls") or {} if snap.exists else {} @@ -630,8 +818,11 @@ class StorageRetentionService: "policy_version": POLICY_VERSION, "bytes": sum(by_category.values()), "bytes_by_category": dict(by_category), - "files": [{"path": p, "bytes": s, "category": c} for p, s, c in deleted], + "files": [{"path": p, "bytes": s, "category": c, + "generation": str(plan.generations.get(p)) if plan.generations.get(p) is not None else None} + for p, s, c in deleted], "failures": failures, + "deletion_log": deletion_log.path if deletion_log else None, } ref.update(update) except Exception: @@ -657,11 +848,14 @@ class StorageRetentionService: "failures": failures, } - # --- orphans (report only, never deleted) ------------------------------------- + # --- orphan folders ----------------------------------------------------------- + + def _known_job_ids(self) -> set: + return {snap.id for snap in self.jobs_collection.select(["status"]).stream()} def orphan_job_folders(self) -> Dict[str, Any]: """``jobs/{id}/`` folders with no Firestore job doc (report only).""" - known = {snap.id for snap in self.jobs_collection.select(["status"]).stream()} + known = self._known_job_ids() sizes: Counter = Counter() for name, size in self.storage.list_files_with_sizes("jobs/"): parts = name.split("/") @@ -673,3 +867,126 @@ class StorageRetentionService: "gib_total": round(total / 2**30, 2), "largest": [{"job_id": k, "gib": round(v / 2**30, 2)} for k, v in sizes.most_common(20)], } + + def _record_holder(self, job_id: str) -> Optional[str]: + """Collection that (still) has a record for this folder id, checked live.""" + for name in ORPHAN_RECORD_COLLECTIONS: + if self.db.collection(name).document(job_id).get().exists: + return name + return None + + def purge_orphans( + self, + dry_run: bool = True, + max_folders: Optional[int] = None, + quiet_days: int = ORPHAN_QUIET_DAYS, + report_path: Optional[str] = None, + now: Optional[datetime] = None, + ) -> Dict[str, Any]: + """Delete everything but ``input/`` from job folders that have no job record. + + Candidates come from one bucket listing vs. the ``jobs`` collection; each + folder is then re-checked LIVE before acting: no record in any of + ORPHAN_RECORD_COLLECTIONS, re-listed fresh, and no object modified in the + last ``quiet_days`` days. Dry-run by default. + """ + started = time.time() + now = now or datetime.now(timezone.utc) + known = self._known_job_ids() + candidates = orphan_candidates(self.storage.list_blob_meta("jobs/"), known) + deletion_log = None if dry_run else DeletionLog(self.storage, now, MODE_ORPHAN) + report: Dict[str, Any] = { + "mode": MODE_ORPHAN, + "dry_run": dry_run, + "started_at": now.isoformat(), + "quiet_days": quiet_days, + "record_collections": list(ORPHAN_RECORD_COLLECTIONS), + "candidates": len(candidates), + "folders": [], + "skipped": Counter(), + "not_orphans": [], + "errors": [], + } + if deletion_log: + report["deletion_log"] = deletion_log.path + report["deletion_summary"] = deletion_log.summary_path + acted = 0 + purge_bytes = keep_bytes = skipped_bytes = 0 + by_ext: Counter = Counter() + for job_id in sorted(candidates): + if max_folders is not None and acted >= max_folders: + report["skipped"]["batch_limit"] += 1 + continue + try: + holder = self._record_holder(job_id) + if holder: + report["skipped"]["has_record"] += 1 + report["not_orphans"].append({"job_id": job_id, "collection": holder}) + continue + fresh = self.storage.list_blob_meta(f"jobs/{job_id}/") + plan = plan_orphan_folder(job_id, fresh, now, quiet_days) + except Exception as e: # noqa: BLE001 + report["errors"].append({"job_id": job_id, "error": str(e)}) + continue + if plan["skip_reason"]: + report["skipped"][plan["skip_reason"]] += 1 + skipped_bytes += plan["purge_bytes"] + plan["keep_bytes"] + continue + if not plan["purge"]: + report["skipped"]["nothing_to_purge"] += 1 + keep_bytes += plan["keep_bytes"] + continue + acted += 1 + purge_bytes += plan["purge_bytes"] + keep_bytes += plan["keep_bytes"] + for meta in plan["purge"]: + rel = meta["name"].split("/", 2)[2] + by_ext[rel.split("/", 1)[0] if "/" in rel else "(root)"] += int(meta.get("size") or 0) + folder = { + "job_id": job_id, "latest_update": plan["latest_update"], + "purge_bytes": plan["purge_bytes"], "keep_bytes": plan["keep_bytes"], + "purge_objects": len(plan["purge"]), "kept_input_objects": len(plan["keep"]), + } + if not dry_run: + deleted = failed = 0 + for meta in plan["purge"]: + path, size, gen = meta["name"], int(meta.get("size") or 0), meta.get("generation") + try: + assert_deletable(path) + self.storage.delete_file(path, ignore_missing=True, if_generation_match=gen) + deletion_log.record(job_id, path, size, gen, "orphan-nonInput", "deleted") + deleted += 1 + except Exception as e: # noqa: BLE001 + deletion_log.record(job_id, path, size, gen, "orphan-nonInput", "failed", str(e)) + failed += 1 + deletion_log.flush() + folder.update({"deleted": deleted, "failed": failed}) + report["folders"].append(folder) + if deletion_log and deletion_log.flush_error: + report["errors"].append({"job_id": job_id, + "error": f"deletion log not persisted, run stopped: {deletion_log.flush_error}"}) + break + + report["skipped"] = dict(report["skipped"]) + report["summary"] = { + "folders_acted" if not dry_run else "folders_to_purge": acted, + "gib_to_delete" if dry_run else "gib_deleted": round(purge_bytes / 2**30, 2), + "gib_input_kept": round(keep_bytes / 2**30, 2), + "gib_skipped_recent": round(skipped_bytes / 2**30, 2), + "gib_by_top_level_dir": {k: round(v / 2**30, 2) for k, v in by_ext.most_common()}, + "duration_seconds": round(time.time() - started, 1), + } + report["report_path"] = report_path or ( + f"{REPORT_PREFIX}/{now.strftime('%Y%m%dT%H%M%SZ')}-orphans-{'dry-run' if dry_run else 'run'}.json") + if deletion_log: + report["deletion_totals"] = deletion_log.write_summary({"report_path": report["report_path"]}) + try: + self.storage.upload_json(report["report_path"], report) + except Exception as e: # noqa: BLE001 + report["report_write_error"] = str(e) + logger.info( + f"STORAGE_RETENTION ORPHANS {'DRY-RUN' if dry_run else 'RUN'}: candidates={len(candidates)} " + f"summary={report['summary']} skipped={report['skipped']} errors={len(report['errors'])} " + f"report=gs://{self.settings.gcs_bucket_name}/{report['report_path']}" + ) + return report diff --git a/backend/services/storage_service.py b/backend/services/storage_service.py index 406b87d0..48be0493 100644 --- a/backend/services/storage_service.py +++ b/backend/services/storage_service.py @@ -298,7 +298,8 @@ class StorageService: logger.error(f"Error generating signed {method} URL for {blob_path}: {e}") raise - def delete_file(self, blob_path: str, ignore_missing: bool = False) -> bool: + def delete_file(self, blob_path: str, ignore_missing: bool = False, + if_generation_match: Optional[int] = None) -> bool: """Delete a file from GCS. Args: @@ -314,7 +315,12 @@ class StorageService: """ try: blob = self.bucket.blob(blob_path) - blob.delete() + if if_generation_match is not None: + # Only delete the exact version that was planned/logged; a newer + # write since then raises PreconditionFailed. + blob.delete(if_generation_match=if_generation_match) + else: + blob.delete() logger.info(f"Deleted gs://{settings.gcs_bucket_name}/{blob_path}") return True except NotFound: @@ -373,6 +379,24 @@ class StorageService: logger.error(f"Error listing files with prefix {prefix}: {e}") raise + def list_blob_meta(self, prefix: str) -> list: + """List ``{name, size, generation, updated}`` for every object under ``prefix``.""" + try: + return [ + {"name": b.name, "size": int(b.size or 0), "generation": b.generation, "updated": b.updated} + for b in self.bucket.list_blobs(prefix=prefix) + ] + except Exception as e: + logger.error(f"Error listing files with prefix {prefix}: {e}") + raise + + def upload_text(self, destination_path: str, text: str, content_type: str = "text/plain") -> str: + """Upload a text object (overwrites).""" + blob = self.bucket.blob(destination_path) + blob.cache_control = NO_STORE_CACHE_CONTROL + blob.upload_from_string(text, content_type=content_type) + return destination_path + def get_file_size(self, blob_path: str) -> Optional[int]: """Size in bytes of a GCS object, or None if it doesn't exist.""" blob = self.bucket.get_blob(blob_path) diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 7c4be6f2..cfc16cbb 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -459,6 +459,8 @@ class WorkerService: report_path: str, max_jobs: Optional[int] = None, include_orphans: bool = False, + mode: str = "job_purge", + max_folders: Optional[int] = None, ) -> bool: """Start the storage-retention pass as the ``storage-retention-job`` Cloud Run Job. @@ -470,6 +472,10 @@ class WorkerService: args += ["--max-jobs", str(int(max_jobs))] if include_orphans: args.append("--include-orphans") + if mode != "job_purge": + args += ["--mode", mode] + if max_folders is not None: + args += ["--max-folders", str(int(max_folders))] if not self._use_cloud_tasks and not is_production(): return self._run_worker_module_locally("storage_retention_worker", args, log_prefix="[storage-retention]") try: diff --git a/backend/tests/test_regenerate_service.py b/backend/tests/test_regenerate_service.py index 9e3ce26f..23ffb279 100644 --- a/backend/tests/test_regenerate_service.py +++ b/backend/tests/test_regenerate_service.py @@ -723,6 +723,35 @@ class TestInternalRetentionEndpoint: assert kwargs["dry_run"] is True and kwargs["include_orphans"] is True assert kwargs["report_path"] == body["report_path"].removeprefix("gs://bucket/") + def _orphans(self, client, url): + test_client, app = client + from backend.api.dependencies import require_admin + + async def admin(): + return _auth(email="admin@x.com", is_admin=True) + + app.dependency_overrides[require_admin] = admin + ws = MagicMock() + ws.trigger_storage_retention_job = AsyncMock(return_value=True) + with patch("backend.services.worker_service.get_worker_service", return_value=ws): + return test_client.post(url), ws + + def test_orphan_endpoint_defaults_to_dry_run(self, client): + resp, ws = self._orphans(client, "/api/internal/storage-retention/orphans") + assert resp.status_code == 200 and resp.json()["dry_run"] is True + kwargs = ws.trigger_storage_retention_job.await_args.kwargs + assert kwargs["mode"] == "orphans" and kwargs["dry_run"] is True + assert "-orphans-dry-run.json" in kwargs["report_path"] + + def test_orphan_real_run_needs_confirmation(self, client): + resp, ws = self._orphans(client, "/api/internal/storage-retention/orphans?dry_run=false") + assert resp.status_code == 400 + ws.trigger_storage_retention_job.assert_not_awaited() + resp, ws = self._orphans( + client, "/api/internal/storage-retention/orphans?dry_run=false&confirm=delete-orphan-job-folders") + assert resp.status_code == 200 + assert ws.trigger_storage_retention_job.await_args.kwargs["dry_run"] is False + def test_job_trigger_failure_is_503(self, client): ws = MagicMock() ws.trigger_storage_retention_job = AsyncMock(return_value=False) @@ -914,6 +943,15 @@ class TestStorageRetentionWorker: assert svc.run.call_args.kwargs == {"dry_run": False, "max_jobs": 5, "include_orphans": True, "report_path": "r.json"} + def test_cli_orphan_mode(self): + from backend.workers import storage_retention_worker + svc = MagicMock() + svc.purge_orphans.return_value = {} + with patch("backend.services.storage_retention.StorageRetentionService", return_value=svc): + assert storage_retention_worker.main(["--mode", "orphans", "--max-folders", "7"]) == 0 + assert svc.purge_orphans.call_args.kwargs == {"dry_run": True, "max_folders": 7, "report_path": None} + svc.run.assert_not_called() + def test_cli_defaults_to_dry_run(self): from backend.workers import storage_retention_worker svc = MagicMock() @@ -948,6 +986,9 @@ class TestStorageRetentionWorker: args = list(request.overrides.container_overrides[0].args) assert args[:3] == ["python", "-m", "backend.workers.storage_retention_worker"] assert args[3:] == ["--dry-run", "false", "--report-path", "r.json", "--max-jobs", "3", "--include-orphans"] + await ws.trigger_storage_retention_job(True, "o.json", mode="orphans", max_folders=2) + args = list(ws._run_job_with_retry.await_args.args[1].overrides.container_overrides[0].args) + assert args[3:] == ["--dry-run", "true", "--report-path", "o.json", "--mode", "orphans", "--max-folders", "2"] diff --git a/backend/tests/test_storage_retention.py b/backend/tests/test_storage_retention.py index e2be47c7..2e72184b 100644 --- a/backend/tests/test_storage_retention.py +++ b/backend/tests/test_storage_retention.py @@ -310,16 +310,25 @@ class TestFileUrlDeletions: # --- Service ---------------------------------------------------------------------- class FakeStorage: - def __init__(self, files): + def __init__(self, files, updated=None): self.files = dict(files) + self.updated = dict(updated or {}) self.deleted = [] self.reports = {} + self.texts = {} + self.delete_generations = {} def list_files_with_sizes(self, prefix): return [(k, v) for k, v in sorted(self.files.items()) if k.startswith(prefix)] - def delete_file(self, path, ignore_missing=False): + def list_blob_meta(self, prefix): + old = datetime(2026, 1, 1, tzinfo=timezone.utc) + return [{"name": k, "size": v, "generation": 1000 + i, "updated": self.updated.get(k, old)} + for i, (k, v) in enumerate(sorted(self.files.items())) if k.startswith(prefix)] + + def delete_file(self, path, ignore_missing=False, if_generation_match=None): self.deleted.append(path) + self.delete_generations[path] = if_generation_match self.files.pop(path, None) return True @@ -327,6 +336,10 @@ class FakeStorage: self.reports[path] = data return path + def upload_text(self, path, text, content_type="text/plain"): + self.texts[path] = text + return path + class FakeDoc: def __init__(self, db, doc_id): @@ -588,10 +601,10 @@ class TestServiceRealRun: svc, db, storage = _service([_job(job_id="a")]) real_delete = storage.delete_file - def flaky(path, ignore_missing=False): + def flaky(path, ignore_missing=False, if_generation_match=None): if path.endswith("lossy_4k_mp4.mp4"): raise RuntimeError("boom") - return real_delete(path, ignore_missing) + return real_delete(path, ignore_missing, if_generation_match) storage.delete_file = flaky svc.run(dry_run=False, now=NOW) @@ -643,7 +656,7 @@ class TestInterruptedPurge: svc, db, storage = _service([_job(job_id="a")]) seen = {} - def delete(path, ignore_missing=False): + def delete(path, ignore_missing=False, if_generation_match=None): # at the first delete the job must already say what's being purged if not seen: doc = db.data["a"] @@ -662,11 +675,11 @@ class TestInterruptedPurge: calls = {"n": 0} real = storage.delete_file - def dies_after_two(path, ignore_missing=False): + def dies_after_two(path, ignore_missing=False, if_generation_match=None): calls["n"] += 1 if calls["n"] > 2: raise SystemExit("instance killed") - return real(path, ignore_missing) + return real(path, ignore_missing, if_generation_match) storage.delete_file = dies_after_two with pytest.raises(SystemExit): @@ -679,3 +692,201 @@ class TestInterruptedPurge: report = svc.run(dry_run=False, now=NOW + timedelta(hours=1)) assert report["summary"]["jobs_purged"] == 1 assert db.data["a"]["storage_purge"]["status"] == "complete" + + + +# --- Deletion audit log ------------------------------------------------------------- + +import json as _json + + +def _log_lines(storage, path): + return [_json.loads(line) for line in storage.texts[path].splitlines() if line] + + +class TestDeletionLog: + def test_real_run_logs_every_deleted_object_with_generation(self): + svc, db, storage = _service([_job(job_id="a")]) + report = svc.run(dry_run=False, now=NOW) + log_path = report["deletion_log"] + assert log_path == "storage-retention/deletion-logs/20261003T120000Z-job_purge.jsonl" + lines = _log_lines(storage, log_path) + assert {l["path"] for l in lines} == set(storage.deleted) + assert all(l["result"] == "deleted" and l["mode"] == "job_purge" and l["job_id"] == "a" for l in lines) + assert all(l["generation"] is not None for l in lines) + cats = {l["category"] for l in lines} + assert {"finals", "videos", "stems", "review-audio", "previews", "screens-mov"} == cats + # deletes are generation-pinned to exactly the logged version + for l in lines: + assert str(storage.delete_generations[l["path"]]) == l["generation"] + summary = storage.reports[report["deletion_summary"]] + assert summary["objects_deleted"] == len(lines) + assert summary["bytes_deleted"] == sum(l["size_bytes"] for l in lines) + assert summary["jobs"] == 1 and summary["failures"] == 0 + # manifest on the job doc carries generations + log path + manifest = db.data["a"]["storage_purge"] + assert manifest["deletion_log"] == log_path + assert all(f["generation"] for f in manifest["files"]) + + def test_dry_run_writes_no_deletion_log(self): + svc, db, storage = _service([_job(job_id="a")]) + report = svc.run(dry_run=True, now=NOW) + assert "deletion_log" not in report and storage.texts == {} + + def test_failures_are_logged(self): + svc, db, storage = _service([_job(job_id="a")]) + real = storage.delete_file + + def flaky(path, ignore_missing=False, if_generation_match=None): + if path.endswith("lossy_4k_mp4.mp4"): + raise RuntimeError("412 precondition failed") + return real(path, ignore_missing, if_generation_match) + + storage.delete_file = flaky + report = svc.run(dry_run=False, now=NOW) + failed = [l for l in _log_lines(storage, report["deletion_log"]) if l["result"] == "failed"] + assert len(failed) == 1 and "precondition" in failed[0]["error"] + assert storage.reports[report["deletion_summary"]]["failures"] == 1 + + def test_log_is_flushed_per_job_so_a_crash_keeps_the_record(self): + jobs = [_job(job_id="a"), _job(job_id="b")] + svc, db, storage = _service(jobs) + real = storage.delete_file + + def die_on_b(path, ignore_missing=False, if_generation_match=None): + if path.startswith("jobs/b/"): + raise SystemExit("killed") + return real(path, ignore_missing, if_generation_match) + + storage.delete_file = die_on_b + with pytest.raises(SystemExit): + svc.run(dry_run=False, now=NOW) + path = "storage-retention/deletion-logs/20261003T120000Z-job_purge.jsonl" + lines = _log_lines(storage, path) + assert lines and {l["job_id"] for l in lines} == {"a"} + assert {l["path"] for l in lines} == {p for p in storage.deleted if p.startswith("jobs/a/")} + + @pytest.mark.parametrize("path", [ + "storage-retention/deletion-logs/x.jsonl", "uploads/a/x.flac", "jobs/", "jobs//x", "temp/x", + ]) + def test_guard_refuses_paths_outside_job_folders(self, path): + with pytest.raises(ValueError): + sr.assert_deletable(path) + + def test_guard_allows_job_files(self): + sr.assert_deletable("jobs/a/finals/x.mp4") + + +# --- Orphan folders ---------------------------------------------------------------- + +def _orphan_service(files, updated=None, records=None): + db = FakeDb([_job(job_id="live")]) + for coll, ids in (records or {}).items(): + db.extra = getattr(db, "extra", {}) + db.extra.update({(coll, i): True for i in ids}) + + real_collection = db.collection + + def collection(name): + if name in ("jobs-dev", "youtube_upload_queue"): + extra = getattr(db, "extra", {}) + return SimpleNamespace(document=lambda i: SimpleNamespace( + get=lambda: SimpleNamespace(exists=extra.get((name, i), False)))) + return real_collection(name) + + db.collection = collection + storage = FakeStorage(files, updated) + return StorageRetentionService(db=db, storage=storage, settings=_settings()), db, storage + + +ORPHAN_FILES = { + "jobs/live/finals/lossy_4k_mp4.mp4": 10, + "jobs/ghost/input/song.flac": 100, + "jobs/ghost/finals/lossy_720p_mp4.mp4": 30, + "jobs/ghost/videos/with_vocals.mkv": 200, + "jobs/ghost/stems/custom_instrumental.flac": 40, + "jobs/ghost/custom_instrumental.mp3": 9, + "jobs/fresh/finals/x.mp4": 50, + "jobs/queued/finals/x.mp4": 70, + "jobs/_preview123/review-audio/mixed.ogg": 4, + "storage-retention/deletion-logs/old.jsonl": 1, + "uploads/ghost/audio/x.flac": 5, +} + + +class TestOrphans: + def _svc(self): + updated = {"jobs/fresh/finals/x.mp4": NOW - timedelta(days=2)} + return _orphan_service(ORPHAN_FILES, updated, records={"youtube_upload_queue": ["queued"]}) + + def test_plan_keeps_only_input(self): + objs = [{"name": k, "size": v, "updated": NOW - timedelta(days=90)} + for k, v in ORPHAN_FILES.items() if k.startswith("jobs/ghost/")] + plan = sr.plan_orphan_folder("ghost", objs, NOW) + assert [m["name"] for m in plan["keep"]] == ["jobs/ghost/input/song.flac"] + assert plan["purge_bytes"] == 30 + 200 + 40 + 9 and plan["skip_reason"] is None + + def test_recent_folder_is_skipped(self): + plan = sr.plan_orphan_folder("x", [{"name": "jobs/x/a.mp4", "size": 1, "updated": NOW - timedelta(days=29)}], NOW) + assert plan["skip_reason"] == "recently_modified" + + def test_dry_run_reports_without_deleting(self): + svc, db, storage = self._svc() + report = svc.purge_orphans(dry_run=True, now=NOW) + assert storage.deleted == [] and storage.texts == {} + assert report["candidates"] == 4 # ghost, fresh, queued, _preview123 (never "live") + assert sorted(f["job_id"] for f in report["folders"]) == ["_preview123", "ghost"] + assert report["skipped"] == {"recently_modified": 1, "has_record": 1} + assert report["not_orphans"] == [{"job_id": "queued", "collection": "youtube_upload_queue"}] + assert report["summary"]["gib_input_kept"] == round(100 / 2**30, 2) + assert report["report_path"].endswith("-orphans-dry-run.json") + + def test_real_run_deletes_non_input_and_logs(self): + svc, db, storage = self._svc() + report = svc.purge_orphans(dry_run=False, now=NOW) + assert sorted(storage.deleted) == sorted([ + "jobs/ghost/finals/lossy_720p_mp4.mp4", "jobs/ghost/videos/with_vocals.mkv", + "jobs/ghost/stems/custom_instrumental.flac", "jobs/ghost/custom_instrumental.mp3", + "jobs/_preview123/review-audio/mixed.ogg", + ]) + for kept in ("jobs/ghost/input/song.flac", "jobs/live/finals/lossy_4k_mp4.mp4", + "jobs/fresh/finals/x.mp4", "jobs/queued/finals/x.mp4", + "storage-retention/deletion-logs/old.jsonl", "uploads/ghost/audio/x.flac"): + assert kept in storage.files + lines = _log_lines(storage, report["deletion_log"]) + assert report["deletion_log"].endswith("-orphan.jsonl") + assert {l["category"] for l in lines} == {"orphan-nonInput"} and {l["mode"] for l in lines} == {"orphan"} + assert storage.reports[report["deletion_summary"]]["objects_deleted"] == 5 + + def test_record_created_after_listing_is_respected(self): + svc, db, storage = self._svc() + db.extra = {("jobs-dev", "ghost"): True} + svc.purge_orphans(dry_run=False, now=NOW) + assert not any(p.startswith("jobs/ghost/") for p in storage.deleted) + + def test_batch_limit(self): + svc, db, storage = self._svc() + report = svc.purge_orphans(dry_run=False, max_folders=1, now=NOW) + assert len(report["folders"]) == 1 and report["skipped"]["batch_limit"] == 3 + + +class TestDeletionLogFlushFailure: + def test_run_stops_deleting_when_log_cannot_be_written(self): + svc, db, storage = _service([_job(job_id="a"), _job(job_id="b")]) + + def broken(path, text, content_type="text/plain"): + raise RuntimeError("gcs write denied") + + storage.upload_text = broken + report = svc.run(dry_run=False, now=NOW) + assert not any(p.startswith("jobs/b/") for p in storage.deleted) + assert report["errors"] and "deletion log not persisted" in report["errors"][0]["error"] + assert storage.reports[report["deletion_summary"]]["log_flush_error"] == "gcs write denied" + + def test_orphan_run_stops_too(self): + files = {"jobs/g1/finals/a.mp4": 1, "jobs/g2/finals/b.mp4": 1} + svc, db, storage = _orphan_service(files) + storage.upload_text = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("denied")) + report = svc.purge_orphans(dry_run=False, now=NOW) + assert storage.deleted == ["jobs/g1/finals/a.mp4"] + assert "deletion log not persisted" in report["errors"][0]["error"] diff --git a/backend/workers/storage_retention_worker.py b/backend/workers/storage_retention_worker.py index b66046c2..fbaf5889 100644 --- a/backend/workers/storage_retention_worker.py +++ b/backend/workers/storage_retention_worker.py @@ -25,16 +25,23 @@ def main(argv=None) -> int: parser.add_argument("--max-jobs", type=int, default=None) parser.add_argument("--include-orphans", action="store_true") parser.add_argument("--report-path", default=None) + parser.add_argument("--mode", choices=["job_purge", "orphans"], default="job_purge") + parser.add_argument("--max-folders", type=int, default=None) args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") from backend.services.storage_retention import StorageRetentionService try: - report = StorageRetentionService().run( - dry_run=args.dry_run, max_jobs=args.max_jobs, - include_orphans=args.include_orphans, report_path=args.report_path, - ) + if args.mode == "orphans": + report = StorageRetentionService().purge_orphans( + dry_run=args.dry_run, max_folders=args.max_folders, report_path=args.report_path, + ) + else: + report = StorageRetentionService().run( + dry_run=args.dry_run, max_jobs=args.max_jobs, + include_orphans=args.include_orphans, report_path=args.report_path, + ) except Exception: logging.getLogger(__name__).exception("STORAGE_RETENTION job crashed") return 1 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d97b7665..7b9cd24a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -253,8 +253,11 @@ jobs completed (newest `complete` timeline entry) more than `STORAGE_RETENTION_M (regenerate, admin/theme re-render, private→public, Edit, retry) passes the screens worker, whose **stems gate** (`backend/services/stems_restore.py`) re-runs the audio-separation job in restore mode when `stems_purged_at` is set; the audio worker re-triggers screens when the stems are back. -- Bucket: versioning on, overwritten versions kept 7 days, soft delete 7 days — purged bytes remain - recoverable for about a week. +- Bucket: versioning on, noncurrent (overwritten or deleted) versions kept 7 days, soft delete 7 days — + purged bytes remain recoverable for about two weeks. +- Every deleted object is logged (path, size, generation) under + `storage-retention/deletion-logs/`; review/restore commands and the one-off orphan-folder cleanup are in + `docs/STORAGE-RETENTION.md`. ## LyricsTranscriber Integration diff --git a/docs/STORAGE-RETENTION.md b/docs/STORAGE-RETENTION.md new file mode 100644 index 00000000..43d3fadf --- /dev/null +++ b/docs/STORAGE-RETENTION.md @@ -0,0 +1,73 @@ +# Storage retention: operations + +Policy + design: `docs/ARCHITECTURE.md` § Storage Retention. Code: +`backend/services/storage_retention.py`, `backend/workers/storage_retention_worker.py`. + +## Runs + +| Mode | Trigger | Deletes | +|---|---|---| +| `job_purge` | Cloud Scheduler `storage-retention-daily` (10:30 UTC) → `POST /api/internal/storage-retention/run` → Cloud Run Job `storage-retention-job` (us-east4) | Regenerable files of jobs completed >30 days ago, up to `STORAGE_RETENTION_MAX_JOBS_PER_RUN` (50) per run. Real deletion is on (`STORAGE_RETENTION_DRY_RUN=false` in the CI `--set-env-vars` of the backend deploy — the API passes the flag to the job; that's the only place it's set). | +| `orphan` | Manual: `POST /api/internal/storage-retention/orphans` (dry-run) / `?dry_run=false&confirm=delete-orphan-job-folders` | Everything except `input/` in `jobs/{id}/` folders with no record in `jobs`, `jobs-dev` or `youtube_upload_queue` (checked live per folder) and nothing modified in the last 30 days. | + +Manual trigger (admin token): + +```bash +TOK=$(gcloud secrets versions access latest --secret=admin-tokens --project=nomadkaraoke | cut -d, -f1) +curl -X POST https://api.nomadkaraoke.com/api/internal/storage-retention/run -H "X-Admin-Token: $TOK" +curl -X POST https://api.nomadkaraoke.com/api/internal/storage-retention/orphans -H "X-Admin-Token: $TOK" # dry-run +``` + +## Where the record is + +- `gs://karaoke-gen-storage-nomadkaraoke/storage-retention/reports/-{dry-run,run}.json`, + `-orphans-{dry-run,run}.json`: the plan/outcome of each pass (dry-runs included). +- `gs://karaoke-gen-storage-nomadkaraoke/storage-retention/deletion-logs/-.jsonl`: **one line + per object a real run deleted (or failed to)**: + `timestamp, run_id, mode (job_purge|orphan), job_id, path, size_bytes, generation, category + (finals|videos|stems|review-audio|previews|screens-mov|encoded|quick|orphan-nonInput), result + (deleted|failed), error`. Rewritten after every job/folder, so a crash leaves a complete record up to + the last finished job. +- `.../deletion-logs/-.summary.json`: totals by category, job count, GiB, failures. +- Each purged job's Firestore doc: `storage_purge` (files + generations + `deletion_log`), + `renders_purged_at`, `stems_purged_at`. + +The `storage-retention/` prefix is never touched by retention code (it only deletes inside +`jobs/{id}/`, enforced by `assert_deletable`) and no lifecycle rule matches live objects there +(the rules are: age 7 on `temp/`+`uploads/`, and noncurrent versions after 7 days). Logs are kept +forever. + +## Reviewing + +```bash +B=gs://karaoke-gen-storage-nomadkaraoke/storage-retention/deletion-logs +gcloud storage ls "$B/" # all runs +RUN=20261004T150000Z-job_purge # pick one from the listing +gcloud storage cat "$B/$RUN.summary.json" | jq . # one run's totals +gcloud storage cat "$B/$RUN.jsonl" | jq -s 'map(select(.result=="deleted")) | group_by(.category) + | map({category: .[0].category, objects: length, gib: (map(.size_bytes)|add/1073741824)})' +gcloud storage cat "$B/$RUN.jsonl" | jq -s 'group_by(.job_id) | map({job: .[0].job_id, + gib: (map(.size_bytes)|add/1073741824)}) | sort_by(-.gib)' # per job +gcloud storage cat "$B/$RUN.jsonl" | jq -c 'select(.result=="failed")' +JOB=abc12345 +gcloud storage cat "$B/*.jsonl" | jq -c --arg j "$JOB" 'select(.job_id==$j)' # everything deleted for a job +``` + +Or: `python scripts/storage_retention_report.py` (all runs), `... ` (one run, per category and +job), `... --find ` (which run deleted it, with a restore command). + +## Restoring + +The bucket keeps deleted objects as noncurrent versions for 7 days, then soft delete keeps them 7 more +days. With the `generation` from the log: + +```bash +OBJ='jobs/abc12345/finals/lossy_4k_mp4.mp4'; GEN=1790000000000000 # from a result=="deleted" log line +# within ~7 days (noncurrent version): +gcloud storage cp --no-clobber "gs://karaoke-gen-storage-nomadkaraoke/$OBJ#$GEN" "gs://karaoke-gen-storage-nomadkaraoke/$OBJ" +# days 7-14 (soft-deleted): +gcloud storage restore "gs://karaoke-gen-storage-nomadkaraoke/$OBJ#$GEN" +``` + +After restoring a purged job's files, prefer simply using "Regenerate video" (or +`POST /api/admin/jobs/{id}/regenerate`) — it rebuilds everything and clears the purge markers. diff --git a/pyproject.toml b/pyproject.toml index 3150dfcd..4cc31d4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.265.1" +version = "0.266.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/storage_retention_report.py b/scripts/storage_retention_report.py new file mode 100644 index 00000000..79f23aab --- /dev/null +++ b/scripts/storage_retention_report.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +Review what the storage-retention process deleted (docs/STORAGE-RETENTION.md). + + python scripts/storage_retention_report.py # list all runs (summaries) + python scripts/storage_retention_report.py RUN_ID # one run: totals by category + per job + python scripts/storage_retention_report.py --find jobs/abc/finals/x.mp4 # which run deleted it + python scripts/storage_retention_report.py --find abc123 # everything deleted for a job id + +RUN_ID is the log name without extension, e.g. 20261004T103000Z-job_purge. +Uses Application Default Credentials (read-only access is enough). +""" +import argparse +import json +import sys +from collections import Counter, defaultdict + +from google.cloud import storage + +BUCKET = "karaoke-gen-storage-nomadkaraoke" +PREFIX = "storage-retention/deletion-logs/" + + +def _bucket(): + return storage.Client(project="nomadkaraoke").bucket(BUCKET) + + +def _gib(n): + return f"{n / 2**30:.2f} GiB" + + +def list_runs(bucket): + for blob in sorted(bucket.list_blobs(prefix=PREFIX), key=lambda b: b.name): + if not blob.name.endswith(".summary.json"): + continue + s = json.loads(blob.download_as_text()) + print(f"{s['run_id']:<40} jobs={s['jobs']:<5} objects={s['objects_deleted']:<6} " + f"{_gib(s['bytes_deleted']):>11} failures={s['failures']}") + + +def _lines(bucket, run_id): + blob = bucket.blob(f"{PREFIX}{run_id}.jsonl") + for line in blob.download_as_text().splitlines(): + if line.strip(): + yield json.loads(line) + + +def show_run(bucket, run_id): + by_cat, n_cat, by_job = Counter(), Counter(), defaultdict(Counter) + failures = [] + for e in _lines(bucket, run_id): + if e["result"] == "deleted": + by_cat[e["category"]] += e["size_bytes"] + n_cat[e["category"]] += 1 + by_job[e["job_id"]][e["category"]] += e["size_bytes"] + else: + failures.append(e) + print(f"Run {run_id}: {len(by_job)} jobs, {sum(n_cat.values())} objects, {_gib(sum(by_cat.values()))}") + for cat, size in by_cat.most_common(): + print(f" {cat:<18} {n_cat[cat]:>6} objects {_gib(size):>11}") + print("Per job:") + for job_id, cats in sorted(by_job.items(), key=lambda kv: -sum(kv[1].values())): + print(f" {job_id:<16} {_gib(sum(cats.values())):>11} " + ", ".join(f"{c}={_gib(v)}" for c, v in cats.items())) + if failures: + print(f"Failures ({len(failures)}):") + for e in failures: + print(f" {e['path']}: {e.get('error')}") + + +def find(bucket, needle): + for blob in sorted(bucket.list_blobs(prefix=PREFIX), key=lambda b: b.name): + if not blob.name.endswith(".jsonl"): + continue + run_id = blob.name[len(PREFIX):-len(".jsonl")] + for e in _lines(bucket, run_id): + if needle == e["job_id"] or needle in e["path"]: + # Only offer a restore for objects actually deleted at a known + # generation (a failed delete may have a newer live version). + restore = "" + if e["result"] == "deleted" and e.get("generation"): + restore = (f" restore: gcloud storage cp --no-clobber 'gs://{BUCKET}/{e['path']}#{e['generation']}' " + f"'gs://{BUCKET}/{e['path']}'") + print(f"{run_id} {e['result']:<7} {e['path']} gen={e.get('generation')} {e['size_bytes']}B{restore}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("run_id", nargs="?") + parser.add_argument("--find") + args = parser.parse_args() + bucket = _bucket() + if args.find: + find(bucket, args.find) + elif args.run_id: + show_run(bucket, args.run_id) + else: + list_runs(bucket) + + +if __name__ == "__main__": + sys.exit(main())