mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: gate brand code recycling on full GDrive cleanup confirmation (#418)
## Summary
- Brand code `NOMAD-1271` was reused for a real song (Hot Mulligan - Pop
Shuvit) while old E2E test job files still existed in Google Drive,
creating a duplicate in the public share
- Root cause: `cleanup-distribution` endpoint had two bugs that allowed
brand codes to be recycled before GDrive was confirmed clean
## Root Causes
1. **Empty dict was falsy**: `if gdrive_files:` skips cleanup when
`gdrive_files = {}` (empty dict). This happened when GDrive uploads
silently failed (pre-PR #417), so test job files accumulated in GDrive
without ever being cleaned
2. **Brand code recycled too early**: Brand code was recycled
immediately after Dropbox cleanup succeeded, *without* confirming GDrive
was also clean — a new job could claim the recycled code while old
GDrive files still existed
## Changes
- **`gdrive_service.py`**: Added `find_files_by_brand_code()` — searches
CDG/, MP4/, MP4-720p/ subfolders by brand code prefix as a fallback when
tracked file IDs aren't available (old jobs, or jobs where upload failed
silently). Raises `RuntimeError` if any subfolder search fails so
callers can't misread an incomplete search as "GDrive is clean"
- **`jobs.py` (`cleanup-distribution`)**: Now uses tracked file IDs
(fast path) OR falls back to brand_code search; brand code is only
recycled after **both** Dropbox AND GDrive are confirmed clean. Fixed
`method` key being lost when results dict was replaced (use local
variable instead of interim mutation)
- **`scripts/cleanup_test_job_gdrive.py`**: Remediation script to delete
existing stranded test job files from GDrive
## Testing
- 41 tests passing, 94% coverage on `gdrive_service.py`
- New `TestDeleteFile` class (5 tests for `delete_file`/`delete_files` —
previously uncovered)
- New `test_cleanup_distribution.py` (9 tests covering fast path,
fallback, and brand code gating logic)
- Updated error handling test to verify `RuntimeError` is raised (not
swallowed) on subfolder search failure
- [x] CodeRabbit CLI review completed, 2 findings fixed
@coderabbitai ignore
---
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
0f5ddb0092
commit
c7b7f4bee5
8 files changed
+1074
-29
No files matched your search
+80
-23
@@ -1655,6 +1655,7 @@ async def cleanup_distribution(
|
||||
# Clean up Dropbox
|
||||
brand_code = state_data.get('brand_code')
|
||||
dropbox_path = getattr(job, 'dropbox_path', None)
|
||||
dropbox_cleaned = False
|
||||
if brand_code and dropbox_path:
|
||||
try:
|
||||
from backend.services.dropbox_service import get_dropbox_service
|
||||
@@ -1664,22 +1665,11 @@ async def cleanup_distribution(
|
||||
folder_name = f"{brand_code} - {base_name}"
|
||||
full_path = f"{dropbox_path}/{folder_name}"
|
||||
success = dropbox.delete_folder(full_path)
|
||||
dropbox_cleaned = success
|
||||
results["dropbox"] = {
|
||||
"status": "success" if success else "failed",
|
||||
"path": full_path
|
||||
}
|
||||
# Recycle the brand code number so it can be reused
|
||||
if success:
|
||||
try:
|
||||
from backend.services.brand_code_service import (
|
||||
BrandCodeService, get_brand_code_service
|
||||
)
|
||||
prefix, number = BrandCodeService.parse_brand_code(brand_code)
|
||||
get_brand_code_service().recycle_brand_code(prefix, number)
|
||||
results["dropbox"]["recycled_brand_code"] = True
|
||||
except (ValueError, Exception) as e:
|
||||
logger.warning(f"Failed to recycle brand code {brand_code}: {e}")
|
||||
results["dropbox"]["recycled_brand_code"] = False
|
||||
else:
|
||||
results["dropbox"] = {"status": "failed", "reason": "Dropbox credentials not configured"}
|
||||
except Exception as e:
|
||||
@@ -1687,25 +1677,92 @@ async def cleanup_distribution(
|
||||
results["dropbox"] = {"status": "error", "error": str(e)}
|
||||
|
||||
# Clean up Google Drive
|
||||
# Use tracked file IDs if available; otherwise fall back to searching by brand code.
|
||||
# This handles old jobs where gdrive_files was not persisted (e.g., uploaded before
|
||||
# file ID tracking was implemented, or when GDrive upload silently failed and a
|
||||
# previous test run left files behind).
|
||||
gdrive_files = state_data.get('gdrive_files')
|
||||
if gdrive_files:
|
||||
gdrive_folder_id = getattr(job, 'gdrive_folder_id', None)
|
||||
gdrive_cleaned = False
|
||||
|
||||
if gdrive_files or (gdrive_folder_id and brand_code):
|
||||
try:
|
||||
from backend.services.gdrive_service import get_gdrive_service
|
||||
gdrive = get_gdrive_service()
|
||||
if gdrive.is_configured:
|
||||
# gdrive_files is a dict like {"mp4": "file_id", "mp4_720p": "file_id", "cdg": "file_id"}
|
||||
file_ids = list(gdrive_files.values()) if isinstance(gdrive_files, dict) else []
|
||||
delete_results = gdrive.delete_files(file_ids)
|
||||
all_success = all(delete_results.values())
|
||||
results["gdrive"] = {
|
||||
"status": "success" if all_success else "partial",
|
||||
"files": delete_results
|
||||
}
|
||||
file_ids: list = []
|
||||
gdrive_method = None # track how files were located for response
|
||||
|
||||
if gdrive_files and isinstance(gdrive_files, dict):
|
||||
# Fast path: delete using tracked file IDs from state_data
|
||||
file_ids = [v for v in gdrive_files.values() if v]
|
||||
logger.info(
|
||||
f"GDrive cleanup for job {job_id}: "
|
||||
f"using {len(file_ids)} tracked file IDs"
|
||||
)
|
||||
|
||||
if not file_ids and gdrive_folder_id and brand_code:
|
||||
# Fallback: search by brand code prefix in CDG/, MP4/, MP4-720p/ subfolders.
|
||||
# Handles jobs where gdrive_files was not tracked (old jobs, or jobs where
|
||||
# GDrive upload succeeded but file IDs were lost).
|
||||
logger.info(
|
||||
f"GDrive cleanup for job {job_id}: "
|
||||
f"no tracked file IDs, searching by brand_code '{brand_code}'"
|
||||
)
|
||||
file_ids = gdrive.find_files_by_brand_code(gdrive_folder_id, brand_code)
|
||||
gdrive_method = "brand_code_search"
|
||||
|
||||
if file_ids:
|
||||
delete_results = gdrive.delete_files(file_ids)
|
||||
all_success = all(delete_results.values())
|
||||
gdrive_cleaned = all_success
|
||||
results["gdrive"] = {
|
||||
"status": "success" if all_success else "partial",
|
||||
"files": delete_results,
|
||||
}
|
||||
if gdrive_method:
|
||||
results["gdrive"]["method"] = gdrive_method
|
||||
else:
|
||||
gdrive_cleaned = True # Nothing to delete
|
||||
results["gdrive"] = {
|
||||
"status": "skipped",
|
||||
"reason": "no files found to delete",
|
||||
}
|
||||
else:
|
||||
results["gdrive"] = {"status": "failed", "reason": "Google Drive credentials not configured"}
|
||||
results["gdrive"] = {
|
||||
"status": "failed",
|
||||
"reason": "Google Drive credentials not configured",
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up Google Drive for job {job_id}: {e}", exc_info=True)
|
||||
logger.error(
|
||||
f"Error cleaning up Google Drive for job {job_id}: {e}", exc_info=True
|
||||
)
|
||||
results["gdrive"] = {"status": "error", "error": str(e)}
|
||||
else:
|
||||
if not gdrive_folder_id:
|
||||
results["gdrive"]["reason"] = "no gdrive_folder_id on job and no tracked file IDs"
|
||||
|
||||
# Recycle the brand code so it can be reused — but only after GDrive is confirmed clean.
|
||||
# If we recycle before GDrive cleanup, a new job could claim the same brand code and
|
||||
# collide with leftover files on GDrive (duplicate brand code in public share).
|
||||
if dropbox_cleaned and brand_code and gdrive_cleaned:
|
||||
try:
|
||||
from backend.services.brand_code_service import (
|
||||
BrandCodeService, get_brand_code_service
|
||||
)
|
||||
prefix, number = BrandCodeService.parse_brand_code(brand_code)
|
||||
get_brand_code_service().recycle_brand_code(prefix, number)
|
||||
results["dropbox"]["recycled_brand_code"] = True
|
||||
logger.info(f"Recycled brand code {brand_code} after full distribution cleanup")
|
||||
except (ValueError, Exception) as e:
|
||||
logger.warning(f"Failed to recycle brand code {brand_code}: {e}")
|
||||
results["dropbox"]["recycled_brand_code"] = False
|
||||
elif dropbox_cleaned and brand_code:
|
||||
logger.warning(
|
||||
f"Brand code {brand_code} NOT recycled: GDrive cleanup did not confirm success "
|
||||
f"(gdrive_cleaned={gdrive_cleaned}). Brand code will remain reserved to prevent "
|
||||
f"collisions with leftover GDrive files."
|
||||
)
|
||||
|
||||
# Delete the job if requested
|
||||
if delete_job:
|
||||
|
||||
@@ -387,6 +387,85 @@ class GoogleDriveService:
|
||||
results[file_id] = self.delete_file(file_id)
|
||||
return results
|
||||
|
||||
def find_files_by_brand_code(self, root_folder_id: str, brand_code: str) -> list[str]:
|
||||
"""
|
||||
Search for files in public share subfolders matching a brand code prefix.
|
||||
|
||||
Used as a fallback when gdrive_files is not tracked in state_data (e.g., old
|
||||
jobs uploaded before file ID tracking was implemented, or jobs where GDrive
|
||||
upload succeeded but file IDs were not saved due to a bug).
|
||||
|
||||
Searches CDG/, MP4/, MP4-720p/ subfolders for files whose name starts with
|
||||
'{brand_code} - '.
|
||||
|
||||
Args:
|
||||
root_folder_id: Root Google Drive folder ID for the public share
|
||||
brand_code: Brand code prefix to search for (e.g., "NOMAD-1271")
|
||||
|
||||
Returns:
|
||||
List of file IDs found matching the brand code
|
||||
"""
|
||||
found_ids = []
|
||||
subfolders = ["CDG", "MP4", "MP4-720p"]
|
||||
search_errors: list[Exception] = []
|
||||
|
||||
for subfolder in subfolders:
|
||||
try:
|
||||
# Find the subfolder
|
||||
escaped_subfolder = subfolder.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped_subfolder}' and '{root_folder_id}' in parents "
|
||||
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(
|
||||
q=query, fields="files(id, name)", supportsAllDrives=True,
|
||||
includeItemsFromAllDrives=True
|
||||
).execute()
|
||||
|
||||
if not results.get("files"):
|
||||
logger.debug(f"Subfolder '{subfolder}' not found in {root_folder_id}")
|
||||
continue
|
||||
|
||||
subfolder_id = results["files"][0]["id"]
|
||||
|
||||
# Search for files whose name starts with the brand code
|
||||
# Use contains for efficiency, then filter by exact prefix
|
||||
escaped_brand = brand_code.replace("'", "\\'")
|
||||
file_query = (
|
||||
f"name contains '{escaped_brand}' and '{subfolder_id}' in parents "
|
||||
f"and trashed=false"
|
||||
)
|
||||
file_results = self.service.files().list(
|
||||
q=file_query, fields="files(id, name)", supportsAllDrives=True,
|
||||
includeItemsFromAllDrives=True
|
||||
).execute()
|
||||
|
||||
for f in file_results.get("files", []):
|
||||
if f["name"].startswith(f"{brand_code} - "):
|
||||
found_ids.append(f["id"])
|
||||
logger.info(
|
||||
f"Found GDrive file to clean up in {subfolder}/: "
|
||||
f"{f['name']} ({f['id']})"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error searching for brand_code '{brand_code}' in '{subfolder}': {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
search_errors.append(e)
|
||||
|
||||
if search_errors:
|
||||
# Raise so the caller knows the search was incomplete.
|
||||
# This prevents the cleanup endpoint from treating an uncertain result
|
||||
# as "GDrive is clean" and incorrectly recycling the brand code.
|
||||
raise RuntimeError(
|
||||
f"GDrive brand_code search incomplete: "
|
||||
f"{len(search_errors)}/{len(subfolders)} subfolder(s) failed"
|
||||
)
|
||||
|
||||
return found_ids
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_gdrive_service: Optional[GoogleDriveService] = None
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""
|
||||
Tests for the cleanup-distribution endpoint in jobs.py.
|
||||
|
||||
Verifies:
|
||||
- GDrive cleanup uses tracked file IDs from state_data when available (fast path)
|
||||
- GDrive cleanup falls back to brand_code search when gdrive_files is empty (fallback path)
|
||||
- Brand code is recycled ONLY after both Dropbox AND GDrive are confirmed clean
|
||||
- Brand code is NOT recycled when GDrive cleanup fails or is skipped
|
||||
- Endpoint returns correct status for each distribution service
|
||||
|
||||
The cleanup-distribution endpoint is used by the E2E happy-path test (Step 12)
|
||||
to remove test job files from YouTube, Dropbox, and Google Drive after the test.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from fastapi.testclient import TestClient
|
||||
from fastapi import FastAPI
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.api.routes.jobs import router
|
||||
from backend.api.dependencies import require_admin, require_auth, AuthResult, UserType
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
# ─── Test App ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_mock_admin():
|
||||
return AuthResult(
|
||||
is_valid=True,
|
||||
user_type=UserType.ADMIN,
|
||||
remaining_uses=999,
|
||||
message="Admin authenticated",
|
||||
user_email="admin@example.com",
|
||||
is_admin=True,
|
||||
)
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api")
|
||||
app.dependency_overrides[require_admin] = get_mock_admin
|
||||
app.dependency_overrides[require_auth] = get_mock_admin
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
# ─── Fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_job(
|
||||
*,
|
||||
brand_code="NOMAD-1271",
|
||||
gdrive_files=None,
|
||||
gdrive_folder_id="root-folder-id",
|
||||
dropbox_path="/Tracks-PublicShare",
|
||||
artist="piri",
|
||||
title="dog",
|
||||
):
|
||||
"""Make a mock Job in COMPLETE state with the given distribution data."""
|
||||
job = Mock(spec=Job)
|
||||
job.job_id = "test-job-id"
|
||||
job.status = JobStatus.COMPLETE
|
||||
job.artist = artist
|
||||
job.title = title
|
||||
job.dropbox_path = dropbox_path
|
||||
job.gdrive_folder_id = gdrive_folder_id
|
||||
job.state_data = {
|
||||
"brand_code": brand_code,
|
||||
"gdrive_files": gdrive_files if gdrive_files is not None else {},
|
||||
}
|
||||
return job
|
||||
|
||||
|
||||
# ─── GDrive Cleanup — Fast Path (tracked file IDs) ───────────────────────────
|
||||
|
||||
class TestCleanupDistributionGDriveTrackedFileIDs:
|
||||
"""GDrive cleanup uses tracked file IDs from state_data when available."""
|
||||
|
||||
def test_gdrive_cleanup_uses_tracked_file_ids(self, client):
|
||||
"""When gdrive_files has file IDs, delete_files is called with those IDs."""
|
||||
job = _make_job(gdrive_files={"mp4": "file-id-mp4", "cdg": "file-id-cdg"})
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.delete_files.return_value = {"file-id-mp4": True, "file-id-cdg": True}
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should have used the tracked file IDs (fast path)
|
||||
mock_gdrive.delete_files.assert_called_once_with(["file-id-mp4", "file-id-cdg"])
|
||||
# Should NOT have searched by brand code
|
||||
mock_gdrive.find_files_by_brand_code.assert_not_called()
|
||||
|
||||
assert data["gdrive"]["status"] == "success"
|
||||
|
||||
def test_gdrive_cleanup_reports_partial_when_some_deletes_fail(self, client):
|
||||
"""When some file deletes fail, status is 'partial'."""
|
||||
job = _make_job(gdrive_files={"mp4": "file-id-mp4", "cdg": "file-id-cdg"})
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.delete_files.return_value = {
|
||||
"file-id-mp4": True,
|
||||
"file-id-cdg": False, # This one failed
|
||||
}
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["gdrive"]["status"] == "partial"
|
||||
|
||||
|
||||
# ─── GDrive Cleanup — Fallback Path (brand_code search) ──────────────────────
|
||||
|
||||
class TestCleanupDistributionGDriveFallback:
|
||||
"""GDrive cleanup searches by brand_code when gdrive_files is empty."""
|
||||
|
||||
def test_falls_back_to_brand_code_search_when_gdrive_files_empty(self, client):
|
||||
"""When gdrive_files={}, search GDrive by brand_code and delete found files."""
|
||||
job = _make_job(gdrive_files={}) # Empty — simulates old jobs / silently-failed uploads
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
# find_files_by_brand_code returns file IDs found by searching GDrive
|
||||
mock_gdrive.find_files_by_brand_code.return_value = ["old-file-id-cdg", "old-file-id-mp4"]
|
||||
mock_gdrive.delete_files.return_value = {
|
||||
"old-file-id-cdg": True,
|
||||
"old-file-id-mp4": True,
|
||||
}
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should have searched by brand code (fallback path)
|
||||
mock_gdrive.find_files_by_brand_code.assert_called_once_with(
|
||||
"root-folder-id", "NOMAD-1271"
|
||||
)
|
||||
# Should have deleted the found files
|
||||
mock_gdrive.delete_files.assert_called_once()
|
||||
assert data["gdrive"]["status"] == "success"
|
||||
|
||||
def test_falls_back_to_brand_code_search_when_gdrive_files_none(self, client):
|
||||
"""When gdrive_files not in state_data (old job), search by brand_code."""
|
||||
job = _make_job(gdrive_files=None)
|
||||
job.state_data = {"brand_code": "NOMAD-1271"} # No gdrive_files key at all
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.find_files_by_brand_code.return_value = ["legacy-file-id"]
|
||||
mock_gdrive.delete_files.return_value = {"legacy-file-id": True}
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_gdrive.find_files_by_brand_code.assert_called_once()
|
||||
|
||||
def test_reports_skipped_when_no_files_found_by_brand_code(self, client):
|
||||
"""When brand_code search finds nothing, GDrive status is 'skipped'."""
|
||||
job = _make_job(gdrive_files={})
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.find_files_by_brand_code.return_value = [] # Nothing found
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["gdrive"]["status"] == "skipped"
|
||||
assert "no files found" in data["gdrive"]["reason"]
|
||||
|
||||
|
||||
# ─── Brand Code Recycling — Gated on GDrive Success ──────────────────────────
|
||||
|
||||
class TestCleanupDistributionBrandCodeRecycling:
|
||||
"""Brand code is recycled only after BOTH Dropbox AND GDrive are confirmed clean."""
|
||||
|
||||
def test_brand_code_recycled_when_both_dropbox_and_gdrive_cleaned(self, client):
|
||||
"""Brand code is recycled when Dropbox delete succeeds and GDrive cleanup succeeds."""
|
||||
job = _make_job(gdrive_files={"cdg": "file-id"})
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True # Dropbox success
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.delete_files.return_value = {"file-id": True} # GDrive success
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.BrandCodeService") as mock_bcs_class, \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
mock_bcs_class.parse_brand_code.return_value = ("NOMAD", 1271)
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Brand code should be recycled
|
||||
mock_brand_service.recycle_brand_code.assert_called_once()
|
||||
assert data["dropbox"].get("recycled_brand_code") is True
|
||||
|
||||
def test_brand_code_not_recycled_when_gdrive_cleanup_partial(self, client):
|
||||
"""Brand code is NOT recycled when GDrive delete only partially succeeded.
|
||||
|
||||
If GDrive still has files with this brand code, recycling would allow a new
|
||||
job to get the same brand code, creating a duplicate in the public share.
|
||||
"""
|
||||
job = _make_job(gdrive_files={"mp4": "file-id-mp4", "cdg": "file-id-cdg"})
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True # Dropbox success
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.delete_files.return_value = {
|
||||
"file-id-mp4": True,
|
||||
"file-id-cdg": False, # CDG file still on GDrive
|
||||
}
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Brand code should NOT be recycled because GDrive still has a file
|
||||
mock_brand_service.recycle_brand_code.assert_not_called()
|
||||
# recycled_brand_code key should be absent (recycling never attempted)
|
||||
assert "recycled_brand_code" not in data.get("dropbox", {})
|
||||
|
||||
def test_brand_code_not_recycled_when_gdrive_not_configured(self, client):
|
||||
"""Brand code is NOT recycled when GDrive is not configured.
|
||||
|
||||
Without GDrive credentials, we cannot confirm that GDrive was cleaned.
|
||||
A previous upload may have succeeded (e.g., different credentials rotation).
|
||||
"""
|
||||
job = _make_job(gdrive_files={"cdg": "file-id"})
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = False # GDrive not configured
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
# Brand code NOT recycled — GDrive might still have files
|
||||
mock_brand_service.recycle_brand_code.assert_not_called()
|
||||
|
||||
def test_brand_code_recycled_when_gdrive_finds_nothing_to_delete(self, client):
|
||||
"""Brand code IS recycled when GDrive search finds nothing (already clean)."""
|
||||
job = _make_job(gdrive_files={}) # Empty — no tracked IDs
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.delete_folder.return_value = True
|
||||
|
||||
mock_gdrive = Mock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.find_files_by_brand_code.return_value = [] # GDrive already clean
|
||||
|
||||
mock_brand_service = Mock()
|
||||
|
||||
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
|
||||
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
|
||||
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
|
||||
patch("backend.services.brand_code_service.BrandCodeService") as mock_bcs_class, \
|
||||
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
|
||||
mock_jm.get_job.return_value = job
|
||||
mock_jm.delete_job.return_value = None
|
||||
mock_bcs_class.parse_brand_code.return_value = ("NOMAD", 1271)
|
||||
|
||||
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# GDrive was confirmed clean (nothing found), so brand code IS recycled
|
||||
mock_brand_service.recycle_brand_code.assert_called_once()
|
||||
assert data["dropbox"].get("recycled_brand_code") is True
|
||||
@@ -682,24 +682,271 @@ class TestRetryBehavior:
|
||||
assert mock_list_result.execute.call_count == 1
|
||||
|
||||
|
||||
class TestDeleteFile:
|
||||
"""Test delete_file and delete_files methods."""
|
||||
|
||||
def _make_service(self, mock_get_settings):
|
||||
"""Helper: make a GoogleDriveService with fake credentials pre-loaded."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
mock_settings = Mock()
|
||||
mock_get_settings.return_value = mock_settings
|
||||
service = GoogleDriveService()
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
return service
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_delete_file_success(self, mock_get_settings):
|
||||
"""Successful delete returns True."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_delete = Mock()
|
||||
mock_delete.execute.return_value = None
|
||||
mock_files_api.delete.return_value = mock_delete
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.delete_file("file-id-123")
|
||||
|
||||
assert result is True
|
||||
mock_files_api.delete.assert_called_once_with(fileId="file-id-123")
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_delete_file_returns_true_on_404(self, mock_get_settings):
|
||||
"""404 (already deleted) returns True — idempotent cleanup."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_delete = Mock()
|
||||
not_found = Exception("File not found")
|
||||
not_found.resp = Mock()
|
||||
not_found.resp.status = 404
|
||||
mock_delete.execute.side_effect = not_found
|
||||
mock_files_api.delete.return_value = mock_delete
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.delete_file("already-gone-id")
|
||||
|
||||
assert result is True
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_delete_file_returns_false_on_error(self, mock_get_settings):
|
||||
"""Non-404 errors return False."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_delete = Mock()
|
||||
mock_delete.execute.side_effect = Exception("Permission denied")
|
||||
mock_files_api.delete.return_value = mock_delete
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.delete_file("restricted-file")
|
||||
|
||||
assert result is False
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_delete_files_calls_delete_file_for_each_id(self, mock_get_settings):
|
||||
"""delete_files returns a per-ID success dict."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
# Stub delete_file: first ID succeeds, second fails
|
||||
results_by_id = {"id-1": True, "id-2": False}
|
||||
service.delete_file = lambda fid: results_by_id[fid]
|
||||
|
||||
result = service.delete_files(["id-1", "id-2"])
|
||||
|
||||
assert result == {"id-1": True, "id-2": False}
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_delete_files_empty_list(self, mock_get_settings):
|
||||
"""delete_files with empty list returns empty dict."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
result = service.delete_files([])
|
||||
assert result == {}
|
||||
|
||||
|
||||
class TestFindFilesByBrandCode:
|
||||
"""Test find_files_by_brand_code method — used as fallback in cleanup-distribution."""
|
||||
|
||||
def _make_service(self, mock_get_settings):
|
||||
"""Helper: make a GoogleDriveService with fake credentials pre-loaded."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
service = GoogleDriveService()
|
||||
service._loaded = True
|
||||
service._credentials_data = {
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
}
|
||||
return service
|
||||
|
||||
def _attach_mock_drive(self, service, list_side_effects):
|
||||
"""Attach a mock Drive client whose files().list().execute() returns given values."""
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = list_side_effects
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
return mock_files_api
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_finds_files_in_all_subfolders(self, mock_get_settings):
|
||||
"""Test finds matching files across CDG/, MP4/, MP4-720p/ subfolders."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
# Responses: for each subfolder: (1) find subfolder, (2) search for files
|
||||
mock_list_result.execute.side_effect = [
|
||||
{"files": [{"id": "cdg-folder"}]}, # CDG subfolder found
|
||||
{"files": [{"id": "cdg-file-1", "name": "NOMAD-1271 - piri - dog.zip"}]},
|
||||
{"files": [{"id": "mp4-folder"}]}, # MP4 subfolder found
|
||||
{"files": [{"id": "mp4-file-1", "name": "NOMAD-1271 - piri - dog.mp4"}]},
|
||||
{"files": [{"id": "mp4720-folder"}]}, # MP4-720p subfolder found
|
||||
{"files": [{"id": "mp4720-file-1", "name": "NOMAD-1271 - piri - dog.mp4"}]},
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
|
||||
|
||||
assert len(result) == 3
|
||||
assert "cdg-file-1" in result
|
||||
assert "mp4-file-1" in result
|
||||
assert "mp4720-file-1" in result
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_returns_empty_when_subfolder_not_found(self, mock_get_settings):
|
||||
"""Test returns empty list when public share subfolders don't exist."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
# All subfolder lookups return empty
|
||||
mock_list_result.execute.return_value = {"files": []}
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
|
||||
|
||||
assert result == []
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_filters_by_exact_brand_code_prefix(self, mock_get_settings):
|
||||
"""Test only returns files whose name starts with '{brand_code} - '."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = [
|
||||
{"files": [{"id": "cdg-folder"}]},
|
||||
{
|
||||
"files": [
|
||||
{"id": "match-id", "name": "NOMAD-1271 - piri - dog.zip"},
|
||||
# This one contains NOMAD-1271 but doesn't START with it — should be excluded
|
||||
{"id": "no-match-id", "name": "BONUS-NOMAD-1271 - other.zip"},
|
||||
]
|
||||
},
|
||||
{"files": []}, # MP4 subfolder not found
|
||||
{"files": []}, # MP4-720p subfolder not found
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
|
||||
|
||||
assert result == ["match-id"]
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_returns_empty_when_no_files_match_brand_code(self, mock_get_settings):
|
||||
"""Test returns empty list when subfolder exists but has no matching files."""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = [
|
||||
{"files": [{"id": "cdg-folder"}]},
|
||||
{"files": [{"id": "other-id", "name": "NOMAD-1300 - someone else.zip"}]},
|
||||
{"files": []}, # MP4 not found
|
||||
{"files": []}, # MP4-720p not found
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
|
||||
|
||||
assert result == []
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_raises_after_all_subfolders_when_any_error(self, mock_get_settings):
|
||||
"""When any subfolder search errors, raises RuntimeError after processing all subfolders.
|
||||
|
||||
This prevents the cleanup endpoint from treating an incomplete search result
|
||||
as "GDrive is clean" and incorrectly recycling the brand code.
|
||||
"""
|
||||
service = self._make_service(mock_get_settings)
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = [
|
||||
{"files": [{"id": "cdg-folder"}]},
|
||||
Exception("API error"), # CDG file search fails
|
||||
{"files": [{"id": "mp4-folder"}]},
|
||||
{"files": [{"id": "mp4-file-1", "name": "NOMAD-1271 - piri - dog.mp4"}]},
|
||||
{"files": [{"id": "mp4720-folder"}]},
|
||||
{"files": []},
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
|
||||
# Should raise after processing all subfolders — an incomplete search must not
|
||||
# be silently treated as "nothing found" by the caller
|
||||
with pytest.raises(RuntimeError, match="search incomplete"):
|
||||
service.find_files_by_brand_code("root-folder", "NOMAD-1271")
|
||||
|
||||
|
||||
class TestGetGdriveService:
|
||||
"""Test get_gdrive_service singleton."""
|
||||
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_get_gdrive_service_singleton(self, mock_get_settings):
|
||||
"""Test get_gdrive_service returns singleton instance."""
|
||||
from backend.services.gdrive_service import get_gdrive_service
|
||||
import backend.services.gdrive_service as gdrive_module
|
||||
|
||||
|
||||
# Reset singleton
|
||||
gdrive_module._gdrive_service = None
|
||||
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
|
||||
service1 = get_gdrive_service()
|
||||
service2 = get_gdrive_service()
|
||||
|
||||
|
||||
assert service1 is service2
|
||||
|
||||
@@ -30,6 +30,17 @@ When two sequential human review steps can be combined into one, do it. We origi
|
||||
|
||||
## Common Gotchas
|
||||
|
||||
### Gate Resource Recycling on Full Cleanup Confirmation (Feb 2026)
|
||||
**What happened**: Brand code NOMAD-1271 was recycled and reused for a real song while old E2E test job files still existed in Google Drive, creating a duplicate in the public share.
|
||||
|
||||
**Root causes**:
|
||||
1. `if gdrive_files:` skips cleanup when `gdrive_files = {}` (empty dict is falsy) — any falsy check on a collection will silently skip work
|
||||
2. Brand code recycled after Dropbox cleanup success, *before* confirming GDrive was clean — partial cleanup allowed resource reuse with orphaned files
|
||||
|
||||
**Fix**: Use `if gdrive_files is not None` (or check explicitly). Gate resource recycling on **all** cleanup steps succeeding, not just the first one. When a resource has been distributed to N places, it's only safe to recycle when all N locations are confirmed clean.
|
||||
|
||||
**Pattern**: Treat resource IDs (brand codes, sequential IDs) like distributed transactions — either all distribution points are cleaned, or the ID stays reserved.
|
||||
|
||||
### Fail Fast, Don't Fall Back
|
||||
Silent fallbacks hide configuration errors. When critical configuration is missing (themes, credentials, etc.), raise clear errors instead of falling back to defaults. Better to fail loudly during testing than silently produce incorrect output in production. Example: Theme validation (v0.109.0) now raises `ValueError` on incomplete themes instead of merging with defaults. This ensures all cloud jobs use complete, explicit themes with no silent degradation.
|
||||
|
||||
|
||||
@@ -42,6 +42,8 @@
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- **GDrive Cleanup Robustness + Brand Code Race Fix** (2026-02-27): Fixed duplicate brand codes appearing in the public Google Drive share after the daily E2E test (NOMAD-1271 appearing for both the test job "piri - dog" and real song "Hot Mulligan - Pop Shuvit"). **Root causes**: (1) `if gdrive_files:` was falsy for empty dict `{}` — when GDrive upload silently failed (pre-PR #417), the entire GDrive cleanup block was skipped, (2) Brand code was recycled immediately after Dropbox cleanup succeeded, without confirming GDrive was also clean — a new job could claim the recycled code while old GDrive files still existed. **Fixes**: Added `find_files_by_brand_code()` fallback to `GoogleDriveService` that searches CDG/MP4/MP4-720p subfolders by brand code prefix when tracked file IDs aren't available. `cleanup-distribution` endpoint now: (a) uses tracked file IDs (fast path) or falls back to brand_code search, (b) only recycles the brand code after **both** Dropbox AND GDrive are confirmed clean. Added `scripts/cleanup_test_job_gdrive.py` remediation script. Added 14 new tests (94% coverage on gdrive_service.py).
|
||||
|
||||
- **CDG Packaging Fail-Fast Validation** (2026-02-27): Fixed silent CDG generation failures where jobs completed with `enable_cdg=True` but no CDG ZIP was produced. **Root cause**: Jobs 5b6aba25 and 5161b069 had `instrumental_selection: custom` but before commit b671b83f, the orchestrator had no `custom` case — it fell through to `"Backing"` suffix. The CDG generator then hit a `FileNotFoundError` looking for the wrong filename, which was silently caught. **Fix**: (1) Orchestrator now validates CDG/TXT outputs after packaging stage and before encoding — if `enable_cdg=True` but no CDG ZIP was produced, the pipeline fails fast instead of silently continuing to expensive encoding, (2) Missing LRC file now raises when CDG/TXT is enabled instead of silently skipping, (3) Added `scripts/regenerate_cdg.py` remediation script for affected jobs. Removed "CDG requires additional style configuration" known issue (the nomad theme has complete CDG styles). Added 7 new tests for validation logic.
|
||||
|
||||
- **GCE Encoding Pipeline Resilience** (2026-02-26): Made encoding pipeline resilient to Cloud Run deployment interruptions. A deployment killed a poller mid-encoding, leaving job a03ad39c permanently stuck at `encoding` status. Three fixes: (1) GCE worker `/encode` endpoint now idempotent — returns cached results for completed jobs, allows retry on failure, returns status for in-progress (previously hard 409 reject), (2) `encoding_service.py` handles cached/in_progress responses and falls back to status check on 409, (3) Health service detects jobs stuck in `encoding` >50 min. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#resilient-remote-worker-interactions-feb-2026).
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.119.8"
|
||||
version = "0.119.9"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Remove stranded E2E test job files from the Google Drive public share.
|
||||
|
||||
These files were left behind by the daily happy-path E2E test before the
|
||||
cleanup-distribution endpoint existed (pre-PR #411) or when the GDrive
|
||||
cleanup was silently skipped because gdrive_files was empty.
|
||||
|
||||
Usage:
|
||||
# Dry run - show what would be deleted
|
||||
python scripts/cleanup_test_job_gdrive.py --dry-run
|
||||
|
||||
# Delete all test job files (files whose name contains test song patterns)
|
||||
python scripts/cleanup_test_job_gdrive.py
|
||||
|
||||
# Delete files with a specific brand code
|
||||
python scripts/cleanup_test_job_gdrive.py --brand-code NOMAD-1271
|
||||
|
||||
# Use a different GDrive folder ID
|
||||
python scripts/cleanup_test_job_gdrive.py --gdrive-folder-id 1abc...
|
||||
|
||||
Requires:
|
||||
- google-api-python-client, google-auth
|
||||
- DEFAULT_GDRIVE_FOLDER_ID env var (or --gdrive-folder-id flag)
|
||||
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
||||
(alternatively: gdrive-oauth-credentials secret in Secret Manager)
|
||||
|
||||
The test song used by the E2E happy path is:
|
||||
Artist: piri
|
||||
Title: dog
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Patterns that identify test job files
|
||||
# These match files created by the E2E happy-path test
|
||||
TEST_FILE_PATTERNS = [
|
||||
"piri - dog", # Current test song (artist: piri, title: dog)
|
||||
]
|
||||
|
||||
PUBLIC_SHARE_SUBFOLDERS = ["CDG", "MP4", "MP4-720p"]
|
||||
|
||||
|
||||
def build_gdrive_service():
|
||||
"""Build Google Drive API service from credentials."""
|
||||
try:
|
||||
from google.oauth2.credentials import Credentials
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
# Try Secret Manager first
|
||||
try:
|
||||
import google.cloud.secretmanager as sm
|
||||
client = sm.SecretManagerServiceClient()
|
||||
name = "projects/nomadkaraoke/secrets/gdrive-oauth-credentials/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
creds_data = json.loads(response.payload.data.decode("utf-8"))
|
||||
|
||||
credentials = Credentials(
|
||||
token=creds_data.get("token"),
|
||||
refresh_token=creds_data.get("refresh_token"),
|
||||
token_uri=creds_data.get("token_uri", "https://oauth2.googleapis.com/token"),
|
||||
client_id=creds_data.get("client_id"),
|
||||
client_secret=creds_data.get("client_secret"),
|
||||
scopes=creds_data.get("scopes", ["https://www.googleapis.com/auth/drive.file"]),
|
||||
)
|
||||
logger.info("Using GDrive credentials from Secret Manager")
|
||||
except Exception as e:
|
||||
logger.warning(f"Secret Manager failed ({e}), falling back to ADC")
|
||||
import google.auth
|
||||
credentials, _ = google.auth.default(
|
||||
scopes=["https://www.googleapis.com/auth/drive"]
|
||||
)
|
||||
|
||||
return build("drive", "v3", credentials=credentials)
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(f"Missing dependency: {e}")
|
||||
logger.error("Install with: pip install google-api-python-client google-auth google-cloud-secret-manager")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def find_test_files(service, root_folder_id: str, patterns: list[str]) -> list[dict]:
|
||||
"""
|
||||
Search CDG/, MP4/, MP4-720p/ subfolders for files matching test song patterns.
|
||||
|
||||
Returns list of dicts: {name, id, subfolder}
|
||||
"""
|
||||
found = []
|
||||
|
||||
for subfolder in PUBLIC_SHARE_SUBFOLDERS:
|
||||
escaped = subfolder.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped}' and '{root_folder_id}' in parents "
|
||||
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
)
|
||||
results = service.files().list(
|
||||
q=query, fields="files(id, name)",
|
||||
supportsAllDrives=True, includeItemsFromAllDrives=True
|
||||
).execute()
|
||||
|
||||
if not results.get("files"):
|
||||
logger.debug(f"Subfolder '{subfolder}' not found, skipping")
|
||||
continue
|
||||
|
||||
subfolder_id = results["files"][0]["id"]
|
||||
logger.debug(f"Found subfolder '{subfolder}': {subfolder_id}")
|
||||
|
||||
# List all files in this subfolder
|
||||
page_token = None
|
||||
while True:
|
||||
kwargs = dict(
|
||||
q=f"'{subfolder_id}' in parents and trashed=false",
|
||||
fields="nextPageToken, files(id, name)",
|
||||
supportsAllDrives=True,
|
||||
includeItemsFromAllDrives=True,
|
||||
pageSize=200,
|
||||
)
|
||||
if page_token:
|
||||
kwargs["pageToken"] = page_token
|
||||
|
||||
response = service.files().list(**kwargs).execute()
|
||||
|
||||
for f in response.get("files", []):
|
||||
name_lower = f["name"].lower()
|
||||
for pattern in patterns:
|
||||
if pattern.lower() in name_lower:
|
||||
found.append({
|
||||
"name": f["name"],
|
||||
"id": f["id"],
|
||||
"subfolder": subfolder,
|
||||
})
|
||||
break
|
||||
|
||||
page_token = response.get("nextPageToken")
|
||||
if not page_token:
|
||||
break
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def find_files_by_brand_code(service, root_folder_id: str, brand_code: str) -> list[dict]:
|
||||
"""Find all files with a specific brand code prefix in the public share subfolders."""
|
||||
found = []
|
||||
|
||||
for subfolder in PUBLIC_SHARE_SUBFOLDERS:
|
||||
escaped = subfolder.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped}' and '{root_folder_id}' in parents "
|
||||
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
)
|
||||
results = service.files().list(
|
||||
q=query, fields="files(id, name)",
|
||||
supportsAllDrives=True, includeItemsFromAllDrives=True
|
||||
).execute()
|
||||
|
||||
if not results.get("files"):
|
||||
continue
|
||||
|
||||
subfolder_id = results["files"][0]["id"]
|
||||
escaped_brand = brand_code.replace("'", "\\'")
|
||||
file_query = (
|
||||
f"name contains '{escaped_brand}' and '{subfolder_id}' in parents and trashed=false"
|
||||
)
|
||||
file_results = service.files().list(
|
||||
q=file_query, fields="files(id, name)",
|
||||
supportsAllDrives=True, includeItemsFromAllDrives=True
|
||||
).execute()
|
||||
|
||||
for f in file_results.get("files", []):
|
||||
if f["name"].startswith(f"{brand_code} - "):
|
||||
found.append({
|
||||
"name": f["name"],
|
||||
"id": f["id"],
|
||||
"subfolder": subfolder,
|
||||
})
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Remove stranded E2E test job files from Google Drive public share"
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show what would be deleted")
|
||||
parser.add_argument(
|
||||
"--gdrive-folder-id",
|
||||
default=os.getenv("DEFAULT_GDRIVE_FOLDER_ID"),
|
||||
help="Root Google Drive folder ID (defaults to DEFAULT_GDRIVE_FOLDER_ID env var)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--brand-code",
|
||||
help="Delete only files with this specific brand code (e.g., NOMAD-1271)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--patterns",
|
||||
nargs="+",
|
||||
default=TEST_FILE_PATTERNS,
|
||||
help="Filename patterns to match (default: piri - dog)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.gdrive_folder_id:
|
||||
logger.error(
|
||||
"No GDrive folder ID. Set DEFAULT_GDRIVE_FOLDER_ID env var "
|
||||
"or pass --gdrive-folder-id"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
logger.info(f"GDrive root folder: {args.gdrive_folder_id}")
|
||||
if args.dry_run:
|
||||
logger.info("DRY RUN - no files will be deleted")
|
||||
|
||||
service = build_gdrive_service()
|
||||
|
||||
if args.brand_code:
|
||||
logger.info(f"Searching for files with brand code: {args.brand_code}")
|
||||
files = find_files_by_brand_code(service, args.gdrive_folder_id, args.brand_code)
|
||||
else:
|
||||
logger.info(f"Searching for test job files matching patterns: {args.patterns}")
|
||||
files = find_test_files(service, args.gdrive_folder_id, args.patterns)
|
||||
|
||||
if not files:
|
||||
logger.info("No test job files found")
|
||||
return
|
||||
|
||||
logger.info(f"\nFound {len(files)} file(s) to delete:")
|
||||
for f in files:
|
||||
logger.info(f" [{f['subfolder']}] {f['name']} (id: {f['id']})")
|
||||
|
||||
if args.dry_run:
|
||||
logger.info("\nDry run complete - no files deleted")
|
||||
return
|
||||
|
||||
confirm = input(f"\nDelete {len(files)} file(s)? [y/N] ")
|
||||
if confirm.lower() != "y":
|
||||
logger.info("Aborted")
|
||||
return
|
||||
|
||||
deleted = 0
|
||||
failed = 0
|
||||
for f in files:
|
||||
try:
|
||||
service.files().delete(
|
||||
fileId=f["id"], supportsAllDrives=True
|
||||
).execute()
|
||||
logger.info(f"Deleted: [{f['subfolder']}] {f['name']}")
|
||||
deleted += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete {f['name']}: {e}")
|
||||
failed += 1
|
||||
|
||||
logger.info(f"\nDone: {deleted} deleted, {failed} failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user