diff --git a/backend/api/routes/jobs.py b/backend/api/routes/jobs.py index 02d091c2..cdddf3f0 100644 --- a/backend/api/routes/jobs.py +++ b/backend/api/routes/jobs.py @@ -887,7 +887,12 @@ def delete_job( detail=t(locale, "jobs.cleanupFailed", services=", ".join(incomplete)), ) - job_manager.delete_job(job_id, delete_files=delete_files) + job_manager.delete_job( + job_id, + delete_files=delete_files, + archived_by=auth_result.user_email, + reason="admin_delete" if auth_result.is_admin else "owner_delete", + ) # A requests-board pick whose owner deleted the job goes back on the board (or is # removed if nobody backs it) instead of staying "in progress" on a dead job. @@ -2810,7 +2815,10 @@ def cleanup_distribution( # Delete the job if requested if delete_job: try: - job_manager.delete_job(job_id, delete_files=True) + job_manager.delete_job( + job_id, delete_files=True, archived_by=auth_result.user_email, + reason="cleanup_distribution_delete", + ) results["job_deleted"] = True logger.info(f"Deleted job {job_id} after distribution cleanup") except Exception as e: diff --git a/backend/services/account_deletion_service.py b/backend/services/account_deletion_service.py index ec4a1a94..be9b6d0d 100644 --- a/backend/services/account_deletion_service.py +++ b/backend/services/account_deletion_service.py @@ -260,6 +260,20 @@ class AccountDeletionService: def _owned_jobs(self, email: str) -> List[Any]: return list(self._jobs().where(filter=FieldFilter("user_email", "==", email)).stream()) + def _archived_jobs(self, email: str) -> List[Any]: + """Deleted jobs kept in ``_archive`` (system data, redacted like live jobs). + + Single-field queries only, so the archive needs no composite index; + made-for-you is filtered here rather than in the query. + """ + archive = self.db.collection(f"{self.jobs_collection}_archive") + owned = list(archive.where(filter=FieldFilter("user_email", "==", email)).stream()) + customer = [ + snap for snap in archive.where(filter=FieldFilter("customer_email", "==", email)).stream() + if (snap.to_dict() or {}).get("made_for_you") + ] + return owned + customer + def _customer_jobs(self, email: str) -> List[Any]: """Made-for-you orders: owned by the admin, delivered to ``customer_email``.""" return list( @@ -442,10 +456,12 @@ class AccountDeletionService: def _scrub_jobs(self, email: str, deletion_id: str) -> dict: totals = {"jobs": 0, "input_objects_deleted": 0, "dropbox_links_revoked": 0} seen = set() - for snap in self._owned_jobs(email) + self._customer_jobs(email): - if snap.id in seen: + tagged = [("live", snap) for snap in self._owned_jobs(email) + self._customer_jobs(email)] + tagged += [("archive", snap) for snap in self._archived_jobs(email)] + for where, snap in tagged: + if (where, snap.id) in seen: continue - seen.add(snap.id) + seen.add((where, snap.id)) result = self._scrub_job(snap, email, deletion_id) totals["jobs"] += 1 totals["input_objects_deleted"] += result["input_objects_deleted"] @@ -517,7 +533,8 @@ class AccountDeletionService: if "client_ip" in (data.get("request_metadata") or {}): updates["request_metadata.client_ip"] = DELETE_FIELD if updates: - self._jobs().document(job_id).update(updates) + # snap.reference: the live job or its archived copy in _archive. + snap.reference.update(updates) return {"input_objects_deleted": deleted, "dropbox_link_revoked": revoked} # Step 8 diff --git a/backend/services/firestore_service.py b/backend/services/firestore_service.py index 3eb91a11..20e4b583 100644 --- a/backend/services/firestore_service.py +++ b/backend/services/firestore_service.py @@ -481,6 +481,56 @@ class FirestoreService: logger.error(f"Error deleting jobs by filter: {e}") raise + # Deleted jobs are archived here (forever) instead of being lost: the job doc + # plus its logs and audio-edit sessions, so a deleted job can still be debugged. + # Logs go under ``archived_logs`` without ``ttl_expiry``: the TTL policy on the + # ``logs`` collection group would otherwise expire them on the original schedule. + ARCHIVE_SUBCOLLECTIONS = {"logs": "archived_logs", "audio_edit_sessions": "audio_edit_sessions"} + + @property + def archive_collection(self) -> str: + return f"{self.collection}_archive" + + def archive_job( + self, job_id: str, archived_by: Optional[str] = None, reason: Optional[str] = None + ) -> bool: + """Copy a job (doc + subcollections) into the archive collection. + + Returns False if the job doesn't exist. Raises on any write failure, so the + caller can refuse to delete a job whose archive copy didn't land. + """ + src = self.db.collection(self.collection).document(job_id) + snap = src.get() + if not snap.exists: + return False + dest = self.db.collection(self.archive_collection).document(job_id) + data = snap.to_dict() or {} + data["archived_at"] = datetime.now(timezone.utc) + data["archived_by"] = archived_by + data["archive_reason"] = reason + dest.set(data) + + copied = {} + for src_name, dest_name in self.ARCHIVE_SUBCOLLECTIONS.items(): + count = 0 + batch = self.db.batch() + pending = 0 + for doc in src.collection(src_name).stream(): + entry = doc.to_dict() or {} + entry.pop("ttl_expiry", None) + batch.set(dest.collection(dest_name).document(doc.id), entry) + pending += 1 + count += 1 + if pending >= 400: + batch.commit() + batch = self.db.batch() + pending = 0 + if pending: + batch.commit() + copied[src_name] = count + logger.info(f"Archived job {job_id} to {self.archive_collection} ({copied}) by {archived_by}: {reason}") + return True + def delete_job(self, job_id: str) -> None: """Delete a job from Firestore.""" try: diff --git a/backend/services/job_manager.py b/backend/services/job_manager.py index fc8d66ff..0ac42ed2 100644 --- a/backend/services/job_manager.py +++ b/backend/services/job_manager.py @@ -567,9 +567,23 @@ class JobManager: return urls - def delete_job(self, job_id: str, delete_files: bool = True) -> None: - """Delete a job, its files, and its logs subcollection.""" + def delete_job( + self, + job_id: str, + delete_files: bool = True, + archived_by: Optional[str] = None, + reason: str = "deleted", + ) -> None: + """Delete a job, its files, and its logs subcollection. + + The job doc, logs and audio-edit sessions are first copied to the archive + collection (``jobs_archive``). If that copy fails the delete is aborted, so a + deleted job is never lost. Bulk test cleanup (``delete_jobs_by_filter``) does + not archive. + """ job = self.get_job(job_id) + if job: + self.firestore.archive_job(job_id, archived_by=archived_by, reason=reason) # Refund credit if job never completed (and wasn't already refunded via fail/cancel) if job: diff --git a/backend/tests/emulator/test_job_archive_emulator.py b/backend/tests/emulator/test_job_archive_emulator.py new file mode 100644 index 00000000..78fa618d --- /dev/null +++ b/backend/tests/emulator/test_job_archive_emulator.py @@ -0,0 +1,64 @@ +""" +Emulator test: deleting a job archives it (doc + logs + audio-edit sessions). + +Run with: ./scripts/run-emulator-tests.sh +""" +import os +import time +from datetime import datetime, timedelta, timezone + +import pytest +import requests + + +def emulators_running() -> bool: + try: + requests.get("http://127.0.0.1:8080", timeout=1) # Firestore + requests.get("http://127.0.0.1:4443", timeout=1) # GCS (same check as conftest) + return True + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + return False + + +pytestmark = pytest.mark.skipif( + not emulators_running(), + reason="GCP emulators not running. Start with: scripts/start-emulators.sh", +) + +os.environ["FIRESTORE_EMULATOR_HOST"] = "127.0.0.1:8080" +os.environ["GOOGLE_CLOUD_PROJECT"] = "test-project" + + +def test_archive_job_copies_doc_and_subcollections_without_ttl(): + from google.cloud import firestore + from backend.services.firestore_service import FirestoreService + + svc = FirestoreService() + svc.db = firestore.Client(project="test-project") + svc.collection = f"test-archive-jobs-{int(time.time() * 1000)}" + + job_id = "arch-job-1" + job_ref = svc.db.collection(svc.collection).document(job_id) + job_ref.set({"job_id": job_id, "status": "complete", "artist": "Invented Band", "title": "Made Up Song"}) + expiry = datetime.now(timezone.utc) + timedelta(days=30) + for i in range(3): + job_ref.collection("logs").document(f"log{i}").set( + {"message": f"line {i}", "ttl_expiry": expiry, "level": "INFO"} + ) + job_ref.collection("audio_edit_sessions").document("s1").set({"edits": 2}) + + assert svc.archive_job(job_id, archived_by="admin@nomadkaraoke.com", reason="admin_delete") is True + + arch = svc.db.collection(svc.archive_collection).document(job_id) + data = arch.get().to_dict() + assert data["artist"] == "Invented Band" + assert data["archived_by"] == "admin@nomadkaraoke.com" + assert data["archive_reason"] == "admin_delete" + assert data["archived_at"] is not None + + logs = {d.id: d.to_dict() for d in arch.collection("archived_logs").stream()} + assert sorted(logs) == ["log0", "log1", "log2"] + assert all("ttl_expiry" not in entry for entry in logs.values()) + assert [d.id for d in arch.collection("audio_edit_sessions").stream()] == ["s1"] + + assert svc.archive_job("does-not-exist") is False diff --git a/backend/tests/test_account_deletion.py b/backend/tests/test_account_deletion.py index 83c3607c..ad1cfa50 100644 --- a/backend/tests/test_account_deletion.py +++ b/backend/tests/test_account_deletion.py @@ -699,6 +699,30 @@ def test_completed_made_for_you_order_anonymised(client, db): assert job["customer_notes"] is None +def test_archived_jobs_redacted_not_deleted(client, db): + """Deleted jobs live on in jobs_archive as system data; account deletion redacts them.""" + _seed_user(db) + archived = dict(_seed_job(db, "arch1"), archived_by="admin@nomadkaraoke.com", archive_reason="admin_delete") + db.collection("jobs").docs.pop("arch1") + db.collection("jobs_archive").docs["arch1"] = archived + mfy = dict(_seed_job(db, "arch2", email="admin@nomadkaraoke.com", made_for_you=True, customer_email=EMAIL)) + db.collection("jobs").docs.pop("arch2") + db.collection("jobs_archive").docs["arch2"] = mfy + + body = _delete(client).json() + tomb = f"deleted:{body['summary']['deletion_id']}" + a1 = db.collection("jobs_archive").docs["arch1"] + assert a1["user_email"] == tomb + assert a1["creation_ip"] is None + assert "client_ip" not in a1["request_metadata"] + assert "customer_email" not in a1["state_data"] + assert a1["artist"] == "Invented Band" # system data kept + assert a1["archive_reason"] == "admin_delete" + a2 = db.collection("jobs_archive").docs["arch2"] + assert a2["user_email"] == "admin@nomadkaraoke.com" + assert a2["customer_email"] == tomb + + def test_other_users_jobs_untouched(client, db): _seed_user(db) _seed_job(db, "j9", email=OTHER) diff --git a/backend/tests/test_job_manager.py b/backend/tests/test_job_manager.py index e4cf852e..4675e0cf 100644 --- a/backend/tests/test_job_manager.py +++ b/backend/tests/test_job_manager.py @@ -515,6 +515,40 @@ class TestJobDeletion: mock_firestore_service.delete_job.assert_called_once_with("test123") + def test_delete_job_archives_first(self, job_manager, mock_firestore_service): + """The job is copied to jobs_archive before the doc is deleted.""" + existing_job = Job( + job_id="test123", status=JobStatus.COMPLETE, progress=100, + created_at=datetime.now(UTC), updated_at=datetime.now(UTC), output_files={}, + ) + mock_firestore_service.get_job.return_value = existing_job + order = [] + mock_firestore_service.archive_job.side_effect = lambda *a, **k: order.append("archive") or True + mock_firestore_service.delete_job.side_effect = lambda *a, **k: order.append("delete") + + job_manager.delete_job("test123", archived_by="admin@nomadkaraoke.com", reason="admin_delete") + + mock_firestore_service.archive_job.assert_called_once_with( + "test123", archived_by="admin@nomadkaraoke.com", reason="admin_delete" + ) + assert order == ["archive", "delete"] + + def test_delete_job_aborts_when_archive_fails(self, job_manager, mock_firestore_service): + """A job whose archive copy didn't land is never deleted.""" + existing_job = Job( + job_id="test123", status=JobStatus.COMPLETE, progress=100, + created_at=datetime.now(UTC), updated_at=datetime.now(UTC), output_files={}, + ) + mock_firestore_service.get_job.return_value = existing_job + mock_firestore_service.archive_job.side_effect = RuntimeError("firestore down") + + with pytest.raises(RuntimeError): + job_manager.delete_job("test123") + + mock_firestore_service.delete_job.assert_not_called() + mock_firestore_service.delete_logs_subcollection.assert_not_called() + + class TestDeleteStateDataKeys: """Tests for delete_state_data_key and delete_state_data_keys methods. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2d63a168..c25403b6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -467,6 +467,8 @@ karaoke-gen shares a GCP project (`nomadkaraoke`) with karaoke-decide, but uses **Worker Logs**: Stored in subcollection (`jobs/{job_id}/logs`) instead of embedded array to avoid Firestore's 1MB document size limit. TTL policy auto-deletes logs after 30 days. Feature flag: `USE_LOG_SUBCOLLECTION` (default: true). See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#firestore-document-1mb-limit-with-embedded-arrays). +**Deleted jobs are archived, not lost** (since v0.295.0): `JobManager.delete_job` first copies the job doc to `jobs_archive/{job_id}` (plus `archived_at`, `archived_by`, `archive_reason`: `admin_delete` / `owner_delete` / `cleanup_distribution_delete`), its logs to `jobs_archive/{job_id}/archived_logs` (without `ttl_expiry`, because the TTL policy is on the `logs` collection group, so archived logs are kept forever), and its `audio_edit_sessions`. If the copy fails, the delete is aborted. Archived jobs are kept forever as system data. Account deletion redacts them like live jobs (owner/customer email → `deleted:`, IPs, notes and contact fields removed). Bulk test cleanup (`DELETE /api/jobs?…`, `delete_jobs_by_filter`) still hard-deletes. `scripts/get_job.py ` falls back to the archive. + ## Encoding Worker: blue-green primary pair + multi-family fallback pool > **2026-11 update:** the c4d Spot pair (`encoding-worker-a`/`-b`) is **retired and deleted**. The preferred encoder (and distribution worker) is the AWS EC2 box `aws-encoding-worker-1`; the only GCE VM is the on-demand fallback `encoding-worker-fallback-c2df` (us-central1-f). `config/encoding-worker` keeps `primary_vm`/`secondary_vm` empty, so every selection path routes to a fallback and records it as `active_override_*`. CI blue-green deploys now just validate the new wheel on c2df and stop it again (the AWS override keeps serving); the worker-change gate diffs against `validated_version`. The pair-specific text below is historical. diff --git a/pyproject.toml b/pyproject.toml index 94b8cf05..b0dbde04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.294.3" +version = "0.295.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/get_job.py b/scripts/get_job.py index b0774b3e..65b2487c 100755 --- a/scripts/get_job.py +++ b/scripts/get_job.py @@ -22,6 +22,11 @@ def get_job(job_id: str) -> dict | None: if doc.exists: return doc.to_dict() + # Deleted jobs are archived (doc + archived_logs) instead of lost. + archived = db.collection('jobs_archive').document(job_id).get() + if archived.exists: + print(f"(job {job_id} was deleted; showing jobs_archive copy)", file=sys.stderr) + return archived.to_dict() return None