fix: gate brand code recycling on full GDrive cleanup confirmation (#418)

## Summary

- Brand code `NOMAD-1271` was reused for a real song (Hot Mulligan - Pop
Shuvit) while old E2E test job files still existed in Google Drive,
creating a duplicate in the public share
- Root cause: `cleanup-distribution` endpoint had two bugs that allowed
brand codes to be recycled before GDrive was confirmed clean

## Root Causes

1. **Empty dict was falsy**: `if gdrive_files:` skips cleanup when
`gdrive_files = {}` (empty dict). This happened when GDrive uploads
silently failed (pre-PR #417), so test job files accumulated in GDrive
without ever being cleaned
2. **Brand code recycled too early**: Brand code was recycled
immediately after Dropbox cleanup succeeded, *without* confirming GDrive
was also clean — a new job could claim the recycled code while old
GDrive files still existed

## Changes

- **`gdrive_service.py`**: Added `find_files_by_brand_code()` — searches
CDG/, MP4/, MP4-720p/ subfolders by brand code prefix as a fallback when
tracked file IDs aren't available (old jobs, or jobs where upload failed
silently). Raises `RuntimeError` if any subfolder search fails so
callers can't misread an incomplete search as "GDrive is clean"
- **`jobs.py` (`cleanup-distribution`)**: Now uses tracked file IDs
(fast path) OR falls back to brand_code search; brand code is only
recycled after **both** Dropbox AND GDrive are confirmed clean. Fixed
`method` key being lost when results dict was replaced (use local
variable instead of interim mutation)
- **`scripts/cleanup_test_job_gdrive.py`**: Remediation script to delete
existing stranded test job files from GDrive

## Testing

- 41 tests passing, 94% coverage on `gdrive_service.py`
- New `TestDeleteFile` class (5 tests for `delete_file`/`delete_files` —
previously uncovered)
- New `test_cleanup_distribution.py` (9 tests covering fast path,
fallback, and brand code gating logic)
- Updated error handling test to verify `RuntimeError` is raised (not
swallowed) on subfolder search failure
- [x] CodeRabbit CLI review completed, 2 findings fixed

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude authored and GitHub committed 2026-02-27 17:35:50 -05:00
1 parent 0f5ddb0092
commit c7b7f4bee5
8 files changed
+1074 -29

No files matched your search

+80 -23
View File
@@ -1655,6 +1655,7 @@ async def cleanup_distribution(
# Clean up Dropbox
brand_code = state_data.get('brand_code')
dropbox_path = getattr(job, 'dropbox_path', None)
dropbox_cleaned = False
if brand_code and dropbox_path:
try:
from backend.services.dropbox_service import get_dropbox_service
@@ -1664,22 +1665,11 @@ async def cleanup_distribution(
folder_name = f"{brand_code} - {base_name}"
full_path = f"{dropbox_path}/{folder_name}"
success = dropbox.delete_folder(full_path)
dropbox_cleaned = success
results["dropbox"] = {
"status": "success" if success else "failed",
"path": full_path
}
# Recycle the brand code number so it can be reused
if success:
try:
from backend.services.brand_code_service import (
BrandCodeService, get_brand_code_service
)
prefix, number = BrandCodeService.parse_brand_code(brand_code)
get_brand_code_service().recycle_brand_code(prefix, number)
results["dropbox"]["recycled_brand_code"] = True
except (ValueError, Exception) as e:
logger.warning(f"Failed to recycle brand code {brand_code}: {e}")
results["dropbox"]["recycled_brand_code"] = False
else:
results["dropbox"] = {"status": "failed", "reason": "Dropbox credentials not configured"}
except Exception as e:
@@ -1687,25 +1677,92 @@ async def cleanup_distribution(
results["dropbox"] = {"status": "error", "error": str(e)}
# Clean up Google Drive
# Use tracked file IDs if available; otherwise fall back to searching by brand code.
# This handles old jobs where gdrive_files was not persisted (e.g., uploaded before
# file ID tracking was implemented, or when GDrive upload silently failed and a
# previous test run left files behind).
gdrive_files = state_data.get('gdrive_files')
if gdrive_files:
gdrive_folder_id = getattr(job, 'gdrive_folder_id', None)
gdrive_cleaned = False
if gdrive_files or (gdrive_folder_id and brand_code):
try:
from backend.services.gdrive_service import get_gdrive_service
gdrive = get_gdrive_service()
if gdrive.is_configured:
# gdrive_files is a dict like {"mp4": "file_id", "mp4_720p": "file_id", "cdg": "file_id"}
file_ids = list(gdrive_files.values()) if isinstance(gdrive_files, dict) else []
delete_results = gdrive.delete_files(file_ids)
all_success = all(delete_results.values())
results["gdrive"] = {
"status": "success" if all_success else "partial",
"files": delete_results
}
file_ids: list = []
gdrive_method = None # track how files were located for response
if gdrive_files and isinstance(gdrive_files, dict):
# Fast path: delete using tracked file IDs from state_data
file_ids = [v for v in gdrive_files.values() if v]
logger.info(
f"GDrive cleanup for job {job_id}: "
f"using {len(file_ids)} tracked file IDs"
)
if not file_ids and gdrive_folder_id and brand_code:
# Fallback: search by brand code prefix in CDG/, MP4/, MP4-720p/ subfolders.
# Handles jobs where gdrive_files was not tracked (old jobs, or jobs where
# GDrive upload succeeded but file IDs were lost).
logger.info(
f"GDrive cleanup for job {job_id}: "
f"no tracked file IDs, searching by brand_code '{brand_code}'"
)
file_ids = gdrive.find_files_by_brand_code(gdrive_folder_id, brand_code)
gdrive_method = "brand_code_search"
if file_ids:
delete_results = gdrive.delete_files(file_ids)
all_success = all(delete_results.values())
gdrive_cleaned = all_success
results["gdrive"] = {
"status": "success" if all_success else "partial",
"files": delete_results,
}
if gdrive_method:
results["gdrive"]["method"] = gdrive_method
else:
gdrive_cleaned = True # Nothing to delete
results["gdrive"] = {
"status": "skipped",
"reason": "no files found to delete",
}
else:
results["gdrive"] = {"status": "failed", "reason": "Google Drive credentials not configured"}
results["gdrive"] = {
"status": "failed",
"reason": "Google Drive credentials not configured",
}
except Exception as e:
logger.error(f"Error cleaning up Google Drive for job {job_id}: {e}", exc_info=True)
logger.error(
f"Error cleaning up Google Drive for job {job_id}: {e}", exc_info=True
)
results["gdrive"] = {"status": "error", "error": str(e)}
else:
if not gdrive_folder_id:
results["gdrive"]["reason"] = "no gdrive_folder_id on job and no tracked file IDs"
# Recycle the brand code so it can be reused — but only after GDrive is confirmed clean.
# If we recycle before GDrive cleanup, a new job could claim the same brand code and
# collide with leftover files on GDrive (duplicate brand code in public share).
if dropbox_cleaned and brand_code and gdrive_cleaned:
try:
from backend.services.brand_code_service import (
BrandCodeService, get_brand_code_service
)
prefix, number = BrandCodeService.parse_brand_code(brand_code)
get_brand_code_service().recycle_brand_code(prefix, number)
results["dropbox"]["recycled_brand_code"] = True
logger.info(f"Recycled brand code {brand_code} after full distribution cleanup")
except (ValueError, Exception) as e:
logger.warning(f"Failed to recycle brand code {brand_code}: {e}")
results["dropbox"]["recycled_brand_code"] = False
elif dropbox_cleaned and brand_code:
logger.warning(
f"Brand code {brand_code} NOT recycled: GDrive cleanup did not confirm success "
f"(gdrive_cleaned={gdrive_cleaned}). Brand code will remain reserved to prevent "
f"collisions with leftover GDrive files."
)
# Delete the job if requested
if delete_job:
+79
View File
@@ -387,6 +387,85 @@ class GoogleDriveService:
results[file_id] = self.delete_file(file_id)
return results
def find_files_by_brand_code(self, root_folder_id: str, brand_code: str) -> list[str]:
"""
Search for files in public share subfolders matching a brand code prefix.
Used as a fallback when gdrive_files is not tracked in state_data (e.g., old
jobs uploaded before file ID tracking was implemented, or jobs where GDrive
upload succeeded but file IDs were not saved due to a bug).
Searches CDG/, MP4/, MP4-720p/ subfolders for files whose name starts with
'{brand_code} - '.
Args:
root_folder_id: Root Google Drive folder ID for the public share
brand_code: Brand code prefix to search for (e.g., "NOMAD-1271")
Returns:
List of file IDs found matching the brand code
"""
found_ids = []
subfolders = ["CDG", "MP4", "MP4-720p"]
search_errors: list[Exception] = []
for subfolder in subfolders:
try:
# Find the subfolder
escaped_subfolder = subfolder.replace("'", "\\'")
query = (
f"name='{escaped_subfolder}' and '{root_folder_id}' in parents "
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
)
results = self.service.files().list(
q=query, fields="files(id, name)", supportsAllDrives=True,
includeItemsFromAllDrives=True
).execute()
if not results.get("files"):
logger.debug(f"Subfolder '{subfolder}' not found in {root_folder_id}")
continue
subfolder_id = results["files"][0]["id"]
# Search for files whose name starts with the brand code
# Use contains for efficiency, then filter by exact prefix
escaped_brand = brand_code.replace("'", "\\'")
file_query = (
f"name contains '{escaped_brand}' and '{subfolder_id}' in parents "
f"and trashed=false"
)
file_results = self.service.files().list(
q=file_query, fields="files(id, name)", supportsAllDrives=True,
includeItemsFromAllDrives=True
).execute()
for f in file_results.get("files", []):
if f["name"].startswith(f"{brand_code} - "):
found_ids.append(f["id"])
logger.info(
f"Found GDrive file to clean up in {subfolder}/: "
f"{f['name']} ({f['id']})"
)
except Exception as e:
logger.error(
f"Error searching for brand_code '{brand_code}' in '{subfolder}': {e}",
exc_info=True,
)
search_errors.append(e)
if search_errors:
# Raise so the caller knows the search was incomplete.
# This prevents the cleanup endpoint from treating an uncertain result
# as "GDrive is clean" and incorrectly recycling the brand code.
raise RuntimeError(
f"GDrive brand_code search incomplete: "
f"{len(search_errors)}/{len(subfolders)} subfolder(s) failed"
)
return found_ids
# Singleton instance
_gdrive_service: Optional[GoogleDriveService] = None
+381
View File
@@ -0,0 +1,381 @@
"""
Tests for the cleanup-distribution endpoint in jobs.py.
Verifies:
- GDrive cleanup uses tracked file IDs from state_data when available (fast path)
- GDrive cleanup falls back to brand_code search when gdrive_files is empty (fallback path)
- Brand code is recycled ONLY after both Dropbox AND GDrive are confirmed clean
- Brand code is NOT recycled when GDrive cleanup fails or is skipped
- Endpoint returns correct status for each distribution service
The cleanup-distribution endpoint is used by the E2E happy-path test (Step 12)
to remove test job files from YouTube, Dropbox, and Google Drive after the test.
"""
import pytest
from unittest.mock import Mock, patch, MagicMock
from fastapi.testclient import TestClient
from fastapi import FastAPI
from datetime import datetime, UTC
from backend.api.routes.jobs import router
from backend.api.dependencies import require_admin, require_auth, AuthResult, UserType
from backend.models.job import Job, JobStatus
# ─── Test App ────────────────────────────────────────────────────────────────
def get_mock_admin():
return AuthResult(
is_valid=True,
user_type=UserType.ADMIN,
remaining_uses=999,
message="Admin authenticated",
user_email="admin@example.com",
is_admin=True,
)
app = FastAPI()
app.include_router(router, prefix="/api")
app.dependency_overrides[require_admin] = get_mock_admin
app.dependency_overrides[require_auth] = get_mock_admin
@pytest.fixture
def client():
return TestClient(app)
# ─── Fixtures ────────────────────────────────────────────────────────────────
def _make_job(
*,
brand_code="NOMAD-1271",
gdrive_files=None,
gdrive_folder_id="root-folder-id",
dropbox_path="/Tracks-PublicShare",
artist="piri",
title="dog",
):
"""Make a mock Job in COMPLETE state with the given distribution data."""
job = Mock(spec=Job)
job.job_id = "test-job-id"
job.status = JobStatus.COMPLETE
job.artist = artist
job.title = title
job.dropbox_path = dropbox_path
job.gdrive_folder_id = gdrive_folder_id
job.state_data = {
"brand_code": brand_code,
"gdrive_files": gdrive_files if gdrive_files is not None else {},
}
return job
# ─── GDrive Cleanup — Fast Path (tracked file IDs) ───────────────────────────
class TestCleanupDistributionGDriveTrackedFileIDs:
"""GDrive cleanup uses tracked file IDs from state_data when available."""
def test_gdrive_cleanup_uses_tracked_file_ids(self, client):
"""When gdrive_files has file IDs, delete_files is called with those IDs."""
job = _make_job(gdrive_files={"mp4": "file-id-mp4", "cdg": "file-id-cdg"})
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.delete_files.return_value = {"file-id-mp4": True, "file-id-cdg": True}
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
# Should have used the tracked file IDs (fast path)
mock_gdrive.delete_files.assert_called_once_with(["file-id-mp4", "file-id-cdg"])
# Should NOT have searched by brand code
mock_gdrive.find_files_by_brand_code.assert_not_called()
assert data["gdrive"]["status"] == "success"
def test_gdrive_cleanup_reports_partial_when_some_deletes_fail(self, client):
"""When some file deletes fail, status is 'partial'."""
job = _make_job(gdrive_files={"mp4": "file-id-mp4", "cdg": "file-id-cdg"})
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.delete_files.return_value = {
"file-id-mp4": True,
"file-id-cdg": False, # This one failed
}
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
assert data["gdrive"]["status"] == "partial"
# ─── GDrive Cleanup — Fallback Path (brand_code search) ──────────────────────
class TestCleanupDistributionGDriveFallback:
"""GDrive cleanup searches by brand_code when gdrive_files is empty."""
def test_falls_back_to_brand_code_search_when_gdrive_files_empty(self, client):
"""When gdrive_files={}, search GDrive by brand_code and delete found files."""
job = _make_job(gdrive_files={}) # Empty — simulates old jobs / silently-failed uploads
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = True
# find_files_by_brand_code returns file IDs found by searching GDrive
mock_gdrive.find_files_by_brand_code.return_value = ["old-file-id-cdg", "old-file-id-mp4"]
mock_gdrive.delete_files.return_value = {
"old-file-id-cdg": True,
"old-file-id-mp4": True,
}
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
# Should have searched by brand code (fallback path)
mock_gdrive.find_files_by_brand_code.assert_called_once_with(
"root-folder-id", "NOMAD-1271"
)
# Should have deleted the found files
mock_gdrive.delete_files.assert_called_once()
assert data["gdrive"]["status"] == "success"
def test_falls_back_to_brand_code_search_when_gdrive_files_none(self, client):
"""When gdrive_files not in state_data (old job), search by brand_code."""
job = _make_job(gdrive_files=None)
job.state_data = {"brand_code": "NOMAD-1271"} # No gdrive_files key at all
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.find_files_by_brand_code.return_value = ["legacy-file-id"]
mock_gdrive.delete_files.return_value = {"legacy-file-id": True}
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
mock_gdrive.find_files_by_brand_code.assert_called_once()
def test_reports_skipped_when_no_files_found_by_brand_code(self, client):
"""When brand_code search finds nothing, GDrive status is 'skipped'."""
job = _make_job(gdrive_files={})
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.find_files_by_brand_code.return_value = [] # Nothing found
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
assert data["gdrive"]["status"] == "skipped"
assert "no files found" in data["gdrive"]["reason"]
# ─── Brand Code Recycling — Gated on GDrive Success ──────────────────────────
class TestCleanupDistributionBrandCodeRecycling:
"""Brand code is recycled only after BOTH Dropbox AND GDrive are confirmed clean."""
def test_brand_code_recycled_when_both_dropbox_and_gdrive_cleaned(self, client):
"""Brand code is recycled when Dropbox delete succeeds and GDrive cleanup succeeds."""
job = _make_job(gdrive_files={"cdg": "file-id"})
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True # Dropbox success
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.delete_files.return_value = {"file-id": True} # GDrive success
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.BrandCodeService") as mock_bcs_class, \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
mock_bcs_class.parse_brand_code.return_value = ("NOMAD", 1271)
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
# Brand code should be recycled
mock_brand_service.recycle_brand_code.assert_called_once()
assert data["dropbox"].get("recycled_brand_code") is True
def test_brand_code_not_recycled_when_gdrive_cleanup_partial(self, client):
"""Brand code is NOT recycled when GDrive delete only partially succeeded.
If GDrive still has files with this brand code, recycling would allow a new
job to get the same brand code, creating a duplicate in the public share.
"""
job = _make_job(gdrive_files={"mp4": "file-id-mp4", "cdg": "file-id-cdg"})
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True # Dropbox success
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.delete_files.return_value = {
"file-id-mp4": True,
"file-id-cdg": False, # CDG file still on GDrive
}
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
# Brand code should NOT be recycled because GDrive still has a file
mock_brand_service.recycle_brand_code.assert_not_called()
# recycled_brand_code key should be absent (recycling never attempted)
assert "recycled_brand_code" not in data.get("dropbox", {})
def test_brand_code_not_recycled_when_gdrive_not_configured(self, client):
"""Brand code is NOT recycled when GDrive is not configured.
Without GDrive credentials, we cannot confirm that GDrive was cleaned.
A previous upload may have succeeded (e.g., different credentials rotation).
"""
job = _make_job(gdrive_files={"cdg": "file-id"})
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = False # GDrive not configured
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
# Brand code NOT recycled — GDrive might still have files
mock_brand_service.recycle_brand_code.assert_not_called()
def test_brand_code_recycled_when_gdrive_finds_nothing_to_delete(self, client):
"""Brand code IS recycled when GDrive search finds nothing (already clean)."""
job = _make_job(gdrive_files={}) # Empty — no tracked IDs
mock_dropbox = Mock()
mock_dropbox.is_configured = True
mock_dropbox.delete_folder.return_value = True
mock_gdrive = Mock()
mock_gdrive.is_configured = True
mock_gdrive.find_files_by_brand_code.return_value = [] # GDrive already clean
mock_brand_service = Mock()
with patch("backend.api.routes.jobs.job_manager") as mock_jm, \
patch("backend.services.dropbox_service.get_dropbox_service", return_value=mock_dropbox), \
patch("backend.services.gdrive_service.get_gdrive_service", return_value=mock_gdrive), \
patch("backend.services.brand_code_service.BrandCodeService") as mock_bcs_class, \
patch("backend.services.brand_code_service.get_brand_code_service", return_value=mock_brand_service):
mock_jm.get_job.return_value = job
mock_jm.delete_job.return_value = None
mock_bcs_class.parse_brand_code.return_value = ("NOMAD", 1271)
response = client.post("/api/jobs/test-job-id/cleanup-distribution")
assert response.status_code == 200
data = response.json()
# GDrive was confirmed clean (nothing found), so brand code IS recycled
mock_brand_service.recycle_brand_code.assert_called_once()
assert data["dropbox"].get("recycled_brand_code") is True
+252 -5
View File
@@ -682,24 +682,271 @@ class TestRetryBehavior:
assert mock_list_result.execute.call_count == 1
class TestDeleteFile:
"""Test delete_file and delete_files methods."""
def _make_service(self, mock_get_settings):
"""Helper: make a GoogleDriveService with fake credentials pre-loaded."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
service._loaded = True
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
return service
@patch("backend.services.gdrive_service.get_settings")
def test_delete_file_success(self, mock_get_settings):
"""Successful delete returns True."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_delete = Mock()
mock_delete.execute.return_value = None
mock_files_api.delete.return_value = mock_delete
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.delete_file("file-id-123")
assert result is True
mock_files_api.delete.assert_called_once_with(fileId="file-id-123")
@patch("backend.services.gdrive_service.get_settings")
def test_delete_file_returns_true_on_404(self, mock_get_settings):
"""404 (already deleted) returns True — idempotent cleanup."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_delete = Mock()
not_found = Exception("File not found")
not_found.resp = Mock()
not_found.resp.status = 404
mock_delete.execute.side_effect = not_found
mock_files_api.delete.return_value = mock_delete
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.delete_file("already-gone-id")
assert result is True
@patch("backend.services.gdrive_service.get_settings")
def test_delete_file_returns_false_on_error(self, mock_get_settings):
"""Non-404 errors return False."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_delete = Mock()
mock_delete.execute.side_effect = Exception("Permission denied")
mock_files_api.delete.return_value = mock_delete
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.delete_file("restricted-file")
assert result is False
@patch("backend.services.gdrive_service.get_settings")
def test_delete_files_calls_delete_file_for_each_id(self, mock_get_settings):
"""delete_files returns a per-ID success dict."""
service = self._make_service(mock_get_settings)
# Stub delete_file: first ID succeeds, second fails
results_by_id = {"id-1": True, "id-2": False}
service.delete_file = lambda fid: results_by_id[fid]
result = service.delete_files(["id-1", "id-2"])
assert result == {"id-1": True, "id-2": False}
@patch("backend.services.gdrive_service.get_settings")
def test_delete_files_empty_list(self, mock_get_settings):
"""delete_files with empty list returns empty dict."""
service = self._make_service(mock_get_settings)
result = service.delete_files([])
assert result == {}
class TestFindFilesByBrandCode:
"""Test find_files_by_brand_code method — used as fallback in cleanup-distribution."""
def _make_service(self, mock_get_settings):
"""Helper: make a GoogleDriveService with fake credentials pre-loaded."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_settings.get_secret.return_value = json.dumps({
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
})
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
service._loaded = True
service._credentials_data = {
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
}
return service
def _attach_mock_drive(self, service, list_side_effects):
"""Attach a mock Drive client whose files().list().execute() returns given values."""
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = list_side_effects
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
return mock_files_api
@patch("backend.services.gdrive_service.get_settings")
def test_finds_files_in_all_subfolders(self, mock_get_settings):
"""Test finds matching files across CDG/, MP4/, MP4-720p/ subfolders."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_list_result = Mock()
# Responses: for each subfolder: (1) find subfolder, (2) search for files
mock_list_result.execute.side_effect = [
{"files": [{"id": "cdg-folder"}]}, # CDG subfolder found
{"files": [{"id": "cdg-file-1", "name": "NOMAD-1271 - piri - dog.zip"}]},
{"files": [{"id": "mp4-folder"}]}, # MP4 subfolder found
{"files": [{"id": "mp4-file-1", "name": "NOMAD-1271 - piri - dog.mp4"}]},
{"files": [{"id": "mp4720-folder"}]}, # MP4-720p subfolder found
{"files": [{"id": "mp4720-file-1", "name": "NOMAD-1271 - piri - dog.mp4"}]},
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
assert len(result) == 3
assert "cdg-file-1" in result
assert "mp4-file-1" in result
assert "mp4720-file-1" in result
@patch("backend.services.gdrive_service.get_settings")
def test_returns_empty_when_subfolder_not_found(self, mock_get_settings):
"""Test returns empty list when public share subfolders don't exist."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_list_result = Mock()
# All subfolder lookups return empty
mock_list_result.execute.return_value = {"files": []}
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
assert result == []
@patch("backend.services.gdrive_service.get_settings")
def test_filters_by_exact_brand_code_prefix(self, mock_get_settings):
"""Test only returns files whose name starts with '{brand_code} - '."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = [
{"files": [{"id": "cdg-folder"}]},
{
"files": [
{"id": "match-id", "name": "NOMAD-1271 - piri - dog.zip"},
# This one contains NOMAD-1271 but doesn't START with it — should be excluded
{"id": "no-match-id", "name": "BONUS-NOMAD-1271 - other.zip"},
]
},
{"files": []}, # MP4 subfolder not found
{"files": []}, # MP4-720p subfolder not found
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
assert result == ["match-id"]
@patch("backend.services.gdrive_service.get_settings")
def test_returns_empty_when_no_files_match_brand_code(self, mock_get_settings):
"""Test returns empty list when subfolder exists but has no matching files."""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = [
{"files": [{"id": "cdg-folder"}]},
{"files": [{"id": "other-id", "name": "NOMAD-1300 - someone else.zip"}]},
{"files": []}, # MP4 not found
{"files": []}, # MP4-720p not found
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
result = service.find_files_by_brand_code("root-folder", "NOMAD-1271")
assert result == []
@patch("backend.services.gdrive_service.get_settings")
def test_raises_after_all_subfolders_when_any_error(self, mock_get_settings):
"""When any subfolder search errors, raises RuntimeError after processing all subfolders.
This prevents the cleanup endpoint from treating an incomplete search result
as "GDrive is clean" and incorrectly recycling the brand code.
"""
service = self._make_service(mock_get_settings)
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = [
{"files": [{"id": "cdg-folder"}]},
Exception("API error"), # CDG file search fails
{"files": [{"id": "mp4-folder"}]},
{"files": [{"id": "mp4-file-1", "name": "NOMAD-1271 - piri - dog.mp4"}]},
{"files": [{"id": "mp4720-folder"}]},
{"files": []},
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
# Should raise after processing all subfolders — an incomplete search must not
# be silently treated as "nothing found" by the caller
with pytest.raises(RuntimeError, match="search incomplete"):
service.find_files_by_brand_code("root-folder", "NOMAD-1271")
class TestGetGdriveService:
"""Test get_gdrive_service singleton."""
@patch("backend.services.gdrive_service.get_settings")
def test_get_gdrive_service_singleton(self, mock_get_settings):
"""Test get_gdrive_service returns singleton instance."""
from backend.services.gdrive_service import get_gdrive_service
import backend.services.gdrive_service as gdrive_module
# Reset singleton
gdrive_module._gdrive_service = None
mock_settings = Mock()
mock_settings.get_secret.return_value = None
mock_get_settings.return_value = mock_settings
service1 = get_gdrive_service()
service2 = get_gdrive_service()
assert service1 is service2
+11
View File
@@ -30,6 +30,17 @@ When two sequential human review steps can be combined into one, do it. We origi
## Common Gotchas
### Gate Resource Recycling on Full Cleanup Confirmation (Feb 2026)
**What happened**: Brand code NOMAD-1271 was recycled and reused for a real song while old E2E test job files still existed in Google Drive, creating a duplicate in the public share.
**Root causes**:
1. `if gdrive_files:` skips cleanup when `gdrive_files = {}` (empty dict is falsy) — any falsy check on a collection will silently skip work
2. Brand code recycled after Dropbox cleanup success, *before* confirming GDrive was clean — partial cleanup allowed resource reuse with orphaned files
**Fix**: Use `if gdrive_files is not None` (or check explicitly). Gate resource recycling on **all** cleanup steps succeeding, not just the first one. When a resource has been distributed to N places, it's only safe to recycle when all N locations are confirmed clean.
**Pattern**: Treat resource IDs (brand codes, sequential IDs) like distributed transactions — either all distribution points are cleaned, or the ID stays reserved.
### Fail Fast, Don't Fall Back
Silent fallbacks hide configuration errors. When critical configuration is missing (themes, credentials, etc.), raise clear errors instead of falling back to defaults. Better to fail loudly during testing than silently produce incorrect output in production. Example: Theme validation (v0.109.0) now raises `ValueError` on incomplete themes instead of merging with defaults. This ensures all cloud jobs use complete, explicit themes with no silent degradation.
+2
View File
@@ -42,6 +42,8 @@
## Recent Changes
- **GDrive Cleanup Robustness + Brand Code Race Fix** (2026-02-27): Fixed duplicate brand codes appearing in the public Google Drive share after the daily E2E test (NOMAD-1271 appearing for both the test job "piri - dog" and real song "Hot Mulligan - Pop Shuvit"). **Root causes**: (1) `if gdrive_files:` was falsy for empty dict `{}` — when GDrive upload silently failed (pre-PR #417), the entire GDrive cleanup block was skipped, (2) Brand code was recycled immediately after Dropbox cleanup succeeded, without confirming GDrive was also clean — a new job could claim the recycled code while old GDrive files still existed. **Fixes**: Added `find_files_by_brand_code()` fallback to `GoogleDriveService` that searches CDG/MP4/MP4-720p subfolders by brand code prefix when tracked file IDs aren't available. `cleanup-distribution` endpoint now: (a) uses tracked file IDs (fast path) or falls back to brand_code search, (b) only recycles the brand code after **both** Dropbox AND GDrive are confirmed clean. Added `scripts/cleanup_test_job_gdrive.py` remediation script. Added 14 new tests (94% coverage on gdrive_service.py).
- **CDG Packaging Fail-Fast Validation** (2026-02-27): Fixed silent CDG generation failures where jobs completed with `enable_cdg=True` but no CDG ZIP was produced. **Root cause**: Jobs 5b6aba25 and 5161b069 had `instrumental_selection: custom` but before commit b671b83f, the orchestrator had no `custom` case — it fell through to `"Backing"` suffix. The CDG generator then hit a `FileNotFoundError` looking for the wrong filename, which was silently caught. **Fix**: (1) Orchestrator now validates CDG/TXT outputs after packaging stage and before encoding — if `enable_cdg=True` but no CDG ZIP was produced, the pipeline fails fast instead of silently continuing to expensive encoding, (2) Missing LRC file now raises when CDG/TXT is enabled instead of silently skipping, (3) Added `scripts/regenerate_cdg.py` remediation script for affected jobs. Removed "CDG requires additional style configuration" known issue (the nomad theme has complete CDG styles). Added 7 new tests for validation logic.
- **GCE Encoding Pipeline Resilience** (2026-02-26): Made encoding pipeline resilient to Cloud Run deployment interruptions. A deployment killed a poller mid-encoding, leaving job a03ad39c permanently stuck at `encoding` status. Three fixes: (1) GCE worker `/encode` endpoint now idempotent — returns cached results for completed jobs, allows retry on failure, returns status for in-progress (previously hard 409 reject), (2) `encoding_service.py` handles cached/in_progress responses and falls back to status check on 409, (3) Health service detects jobs stuck in `encoding` >50 min. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#resilient-remote-worker-interactions-feb-2026).
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.119.8"
version = "0.119.9"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+268
View File
@@ -0,0 +1,268 @@
#!/usr/bin/env python3
"""
Remove stranded E2E test job files from the Google Drive public share.
These files were left behind by the daily happy-path E2E test before the
cleanup-distribution endpoint existed (pre-PR #411) or when the GDrive
cleanup was silently skipped because gdrive_files was empty.
Usage:
# Dry run - show what would be deleted
python scripts/cleanup_test_job_gdrive.py --dry-run
# Delete all test job files (files whose name contains test song patterns)
python scripts/cleanup_test_job_gdrive.py
# Delete files with a specific brand code
python scripts/cleanup_test_job_gdrive.py --brand-code NOMAD-1271
# Use a different GDrive folder ID
python scripts/cleanup_test_job_gdrive.py --gdrive-folder-id 1abc...
Requires:
- google-api-python-client, google-auth
- DEFAULT_GDRIVE_FOLDER_ID env var (or --gdrive-folder-id flag)
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
(alternatively: gdrive-oauth-credentials secret in Secret Manager)
The test song used by the E2E happy path is:
Artist: piri
Title: dog
"""
import argparse
import json
import logging
import os
import sys
# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger(__name__)
# Patterns that identify test job files
# These match files created by the E2E happy-path test
TEST_FILE_PATTERNS = [
"piri - dog", # Current test song (artist: piri, title: dog)
]
PUBLIC_SHARE_SUBFOLDERS = ["CDG", "MP4", "MP4-720p"]
def build_gdrive_service():
"""Build Google Drive API service from credentials."""
try:
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
# Try Secret Manager first
try:
import google.cloud.secretmanager as sm
client = sm.SecretManagerServiceClient()
name = "projects/nomadkaraoke/secrets/gdrive-oauth-credentials/versions/latest"
response = client.access_secret_version(request={"name": name})
creds_data = json.loads(response.payload.data.decode("utf-8"))
credentials = Credentials(
token=creds_data.get("token"),
refresh_token=creds_data.get("refresh_token"),
token_uri=creds_data.get("token_uri", "https://oauth2.googleapis.com/token"),
client_id=creds_data.get("client_id"),
client_secret=creds_data.get("client_secret"),
scopes=creds_data.get("scopes", ["https://www.googleapis.com/auth/drive.file"]),
)
logger.info("Using GDrive credentials from Secret Manager")
except Exception as e:
logger.warning(f"Secret Manager failed ({e}), falling back to ADC")
import google.auth
credentials, _ = google.auth.default(
scopes=["https://www.googleapis.com/auth/drive"]
)
return build("drive", "v3", credentials=credentials)
except ImportError as e:
logger.error(f"Missing dependency: {e}")
logger.error("Install with: pip install google-api-python-client google-auth google-cloud-secret-manager")
sys.exit(1)
def find_test_files(service, root_folder_id: str, patterns: list[str]) -> list[dict]:
"""
Search CDG/, MP4/, MP4-720p/ subfolders for files matching test song patterns.
Returns list of dicts: {name, id, subfolder}
"""
found = []
for subfolder in PUBLIC_SHARE_SUBFOLDERS:
escaped = subfolder.replace("'", "\\'")
query = (
f"name='{escaped}' and '{root_folder_id}' in parents "
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
)
results = service.files().list(
q=query, fields="files(id, name)",
supportsAllDrives=True, includeItemsFromAllDrives=True
).execute()
if not results.get("files"):
logger.debug(f"Subfolder '{subfolder}' not found, skipping")
continue
subfolder_id = results["files"][0]["id"]
logger.debug(f"Found subfolder '{subfolder}': {subfolder_id}")
# List all files in this subfolder
page_token = None
while True:
kwargs = dict(
q=f"'{subfolder_id}' in parents and trashed=false",
fields="nextPageToken, files(id, name)",
supportsAllDrives=True,
includeItemsFromAllDrives=True,
pageSize=200,
)
if page_token:
kwargs["pageToken"] = page_token
response = service.files().list(**kwargs).execute()
for f in response.get("files", []):
name_lower = f["name"].lower()
for pattern in patterns:
if pattern.lower() in name_lower:
found.append({
"name": f["name"],
"id": f["id"],
"subfolder": subfolder,
})
break
page_token = response.get("nextPageToken")
if not page_token:
break
return found
def find_files_by_brand_code(service, root_folder_id: str, brand_code: str) -> list[dict]:
"""Find all files with a specific brand code prefix in the public share subfolders."""
found = []
for subfolder in PUBLIC_SHARE_SUBFOLDERS:
escaped = subfolder.replace("'", "\\'")
query = (
f"name='{escaped}' and '{root_folder_id}' in parents "
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
)
results = service.files().list(
q=query, fields="files(id, name)",
supportsAllDrives=True, includeItemsFromAllDrives=True
).execute()
if not results.get("files"):
continue
subfolder_id = results["files"][0]["id"]
escaped_brand = brand_code.replace("'", "\\'")
file_query = (
f"name contains '{escaped_brand}' and '{subfolder_id}' in parents and trashed=false"
)
file_results = service.files().list(
q=file_query, fields="files(id, name)",
supportsAllDrives=True, includeItemsFromAllDrives=True
).execute()
for f in file_results.get("files", []):
if f["name"].startswith(f"{brand_code} - "):
found.append({
"name": f["name"],
"id": f["id"],
"subfolder": subfolder,
})
return found
def main():
parser = argparse.ArgumentParser(
description="Remove stranded E2E test job files from Google Drive public share"
)
parser.add_argument("--dry-run", action="store_true", help="Show what would be deleted")
parser.add_argument(
"--gdrive-folder-id",
default=os.getenv("DEFAULT_GDRIVE_FOLDER_ID"),
help="Root Google Drive folder ID (defaults to DEFAULT_GDRIVE_FOLDER_ID env var)",
)
parser.add_argument(
"--brand-code",
help="Delete only files with this specific brand code (e.g., NOMAD-1271)",
)
parser.add_argument(
"--patterns",
nargs="+",
default=TEST_FILE_PATTERNS,
help="Filename patterns to match (default: piri - dog)",
)
args = parser.parse_args()
if not args.gdrive_folder_id:
logger.error(
"No GDrive folder ID. Set DEFAULT_GDRIVE_FOLDER_ID env var "
"or pass --gdrive-folder-id"
)
sys.exit(1)
logger.info(f"GDrive root folder: {args.gdrive_folder_id}")
if args.dry_run:
logger.info("DRY RUN - no files will be deleted")
service = build_gdrive_service()
if args.brand_code:
logger.info(f"Searching for files with brand code: {args.brand_code}")
files = find_files_by_brand_code(service, args.gdrive_folder_id, args.brand_code)
else:
logger.info(f"Searching for test job files matching patterns: {args.patterns}")
files = find_test_files(service, args.gdrive_folder_id, args.patterns)
if not files:
logger.info("No test job files found")
return
logger.info(f"\nFound {len(files)} file(s) to delete:")
for f in files:
logger.info(f" [{f['subfolder']}] {f['name']} (id: {f['id']})")
if args.dry_run:
logger.info("\nDry run complete - no files deleted")
return
confirm = input(f"\nDelete {len(files)} file(s)? [y/N] ")
if confirm.lower() != "y":
logger.info("Aborted")
return
deleted = 0
failed = 0
for f in files:
try:
service.files().delete(
fileId=f["id"], supportsAllDrives=True
).execute()
logger.info(f"Deleted: [{f['subfolder']}] {f['name']}")
deleted += 1
except Exception as e:
logger.error(f"Failed to delete {f['name']}: {e}")
failed += 1
logger.info(f"\nDone: {deleted} deleted, {failed} failed")
if __name__ == "__main__":
main()