feat: Integrate flacfetch cache management API (#180)

## Summary

- Integrate karaoke-gen with flacfetch's new cache management API
(v0.15.4)
- When admins clear a job's cache, both Firestore AND flacfetch GCS
cache are now cleared
- Add bulk cache management (clear all, view stats) to admin UI

## Changes

### Backend
- **FlacfetchClient** (`backend/services/flacfetch_client.py`): Add
cache management methods
  - `clear_search_cache(artist, title)` - clears specific cache entry
  - `clear_all_cache()` - clears entire flacfetch cache  
  - `get_cache_stats()` - returns cache statistics

- **Admin API** (`backend/api/routes/admin.py`): 
- Update `POST /api/admin/audio-searches/{job_id}/clear-cache` to also
call flacfetch cache API
  - Add `DELETE /api/admin/cache` - clear entire flacfetch cache
  - Add `GET /api/admin/cache/stats` - get cache statistics

### Frontend
- **API Client** (`frontend/lib/api.ts`): Add `clearAllCache()` and
`getCacheStats()` methods
- **Admin Searches Page** (`frontend/app/admin/searches/page.tsx`):
  - Add "Clear All Cache" button with confirmation dialog
  - Display cache stats (entry count, size, newest entry)

### Documentation
- Update API.md with new endpoints
- Update README.md with recent changes
- Update LESSONS-LEARNED.md with two-layer caching insight

## Architecture

```
User clicks "Clear Cache" on job
    ↓
karaoke-gen clears Firestore state_data (existing)
    ↓
karaoke-gen calls flacfetch DELETE /cache/search (NEW)
    ↓
flacfetch deletes GCS cache entry
    ↓
Next search hits trackers fresh ✓
```

## Testing

- [x] Backend tests pass (`make test` - 57 tests)
- [x] Frontend tests pass (`npm run test:unit` - 128 tests)
- [x] Frontend builds successfully
- [x] CodeRabbit CLI review completed locally

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude authored and GitHub committed 2026-01-03 22:48:29 -05:00
1 parent 3ffed23880
commit 8a7cbb4a8b
9 files changed
+471 -20

No files matched your search

+133 -2
View File
@@ -18,6 +18,7 @@ from backend.api.dependencies import require_admin
from backend.services.auth_service import UserType
from backend.services.user_service import get_user_service, UserService, USERS_COLLECTION
from backend.services.job_manager import JobManager
from backend.services.flacfetch_client import get_flacfetch_client, FlacfetchServiceError
from backend.models.job import JobStatus
@@ -243,6 +244,24 @@ class ClearSearchCacheResponse(BaseModel):
previous_status: str
new_status: str
results_cleared: int
flacfetch_cache_cleared: bool = False
flacfetch_error: Optional[str] = None
class ClearAllCacheResponse(BaseModel):
"""Response for clearing all flacfetch cache."""
status: str
message: str
deleted_count: int
class CacheStatsResponse(BaseModel):
"""Response for cache statistics."""
count: int
total_size_bytes: int
oldest_entry: Optional[str] = None
newest_entry: Optional[str] = None
configured: bool
# =============================================================================
@@ -343,6 +362,7 @@ async def clear_audio_search_cache(
This will:
1. Remove the cached search results from job.state_data
2. Reset the job status to 'pending' so a new search can be performed
3. Clear the flacfetch GCS cache for this artist/title (if available)
Use this when:
- Cached results are stale (e.g., flacfetch was updated)
@@ -399,16 +419,127 @@ async def clear_audio_search_cache(
"updated_at": datetime.utcnow(),
})
# Also clear flacfetch's GCS cache if we have artist/title
flacfetch_cache_cleared = False
flacfetch_error = None
artist = job.audio_search_artist
title = job.audio_search_title
if artist and title:
flacfetch_client = get_flacfetch_client()
if flacfetch_client:
try:
flacfetch_cache_cleared = await flacfetch_client.clear_search_cache(artist, title)
logger.info(
f"Cleared flacfetch cache for '{artist}' - '{title}': "
f"{'deleted' if flacfetch_cache_cleared else 'no entry found'}"
)
except FlacfetchServiceError as e:
flacfetch_error = str(e)
logger.warning(f"Failed to clear flacfetch cache: {e}")
else:
flacfetch_error = "flacfetch client not configured"
logger.debug("Skipping flacfetch cache clear - client not configured")
else:
flacfetch_error = "missing artist or title"
logger.debug(f"Skipping flacfetch cache clear - missing artist ({artist}) or title ({title})")
logger.info(
f"Admin {auth_data[0]} cleared audio search cache for job {job_id}. "
f"Cleared {results_count} results. Status changed from {previous_status} to pending."
f"Cleared {results_count} results. Status changed from {previous_status} to pending. "
f"Flacfetch cache cleared: {flacfetch_cache_cleared}"
)
message = f"Cleared {results_count} cached search results. Job reset to pending."
if flacfetch_cache_cleared:
message += " Flacfetch cache also cleared."
elif flacfetch_error:
message += f" Note: flacfetch cache not cleared ({flacfetch_error})."
return ClearSearchCacheResponse(
status="success",
job_id=job_id,
message=f"Cleared {results_count} cached search results. Job reset to pending.",
message=message,
previous_status=previous_status,
new_status="pending",
results_cleared=results_count,
flacfetch_cache_cleared=flacfetch_cache_cleared,
flacfetch_error=flacfetch_error,
)
@router.delete("/cache", response_model=ClearAllCacheResponse)
async def clear_all_flacfetch_cache(
auth_data: Tuple[str, UserType, int] = Depends(require_admin),
):
"""
Clear the entire flacfetch search cache.
This will delete all cached search results from flacfetch's GCS cache.
Use with caution - this will cause all subsequent searches to hit
the trackers fresh.
Note: This does NOT clear Firestore job.state_data caches, only the
flacfetch-side GCS cache.
"""
flacfetch_client = get_flacfetch_client()
if not flacfetch_client:
raise HTTPException(
status_code=503,
detail="flacfetch client not configured"
)
try:
deleted_count = await flacfetch_client.clear_all_cache()
logger.info(
f"Admin {auth_data[0]} cleared all flacfetch cache. "
f"Deleted {deleted_count} entries."
)
return ClearAllCacheResponse(
status="success",
message=f"Cleared {deleted_count} cache entries from flacfetch.",
deleted_count=deleted_count,
)
except FlacfetchServiceError as e:
logger.error(f"Failed to clear all flacfetch cache: {e}")
raise HTTPException(
status_code=502,
detail=f"Failed to clear flacfetch cache: {e}"
)
@router.get("/cache/stats", response_model=CacheStatsResponse)
async def get_flacfetch_cache_stats(
auth_data: Tuple[str, UserType, int] = Depends(require_admin),
):
"""
Get statistics about the flacfetch search cache.
Returns information about:
- Number of cached entries
- Total size in bytes
- Oldest and newest cache entries
- Whether cache is configured
"""
flacfetch_client = get_flacfetch_client()
if not flacfetch_client:
raise HTTPException(
status_code=503,
detail="flacfetch client not configured"
)
try:
stats = await flacfetch_client.get_cache_stats()
return CacheStatsResponse(
count=stats.get("count", 0),
total_size_bytes=stats.get("total_size_bytes", 0),
oldest_entry=stats.get("oldest_entry"),
newest_entry=stats.get("newest_entry"),
configured=stats.get("configured", True),
)
except FlacfetchServiceError as e:
logger.error(f"Failed to get flacfetch cache stats: {e}")
raise HTTPException(
status_code=502,
detail=f"Failed to get cache stats: {e}"
)
+103 -3
View File
@@ -408,11 +408,11 @@ class FlacfetchClient:
) -> Dict[str, Any]:
"""
Trigger disk cleanup.
Args:
strategy: Cleanup strategy (oldest, largest, lowest_ratio)
target_free_gb: Target free space to achieve
Returns:
Cleanup response with removed count and freed space
"""
@@ -429,12 +429,112 @@ class FlacfetchClient:
)
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Cleanup failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"Cleanup failed: {e.response.status_code}")
# =========================================================================
# Cache Management
# =========================================================================
async def clear_search_cache(self, artist: str, title: str) -> bool:
"""
Clear a specific cached search result by artist and title.
Args:
artist: Artist name
title: Track title
Returns:
True if a cache entry was deleted, False if no entry existed
Raises:
FlacfetchServiceError: On request failure
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.request(
"DELETE",
f"{self.base_url}/cache/search",
headers=self._headers(),
json={"artist": artist, "title": title},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
deleted = data.get("deleted", False)
logger.info(
f"Cleared flacfetch cache for '{artist}' - '{title}': "
f"{'deleted' if deleted else 'no entry found'}"
)
return deleted
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Clear search cache request failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(
f"Clear search cache failed: {e.response.status_code} - {e.response.text}"
)
async def clear_all_cache(self) -> int:
"""
Clear all cached search results.
Returns:
Number of cache entries deleted
Raises:
FlacfetchServiceError: On request failure
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.delete(
f"{self.base_url}/cache",
headers=self._headers(),
timeout=60,
)
resp.raise_for_status()
data = resp.json()
deleted_count = data.get("deleted_count", 0)
logger.info(f"Cleared all flacfetch cache: {deleted_count} entries deleted")
return deleted_count
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Clear all cache request failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(
f"Clear all cache failed: {e.response.status_code} - {e.response.text}"
)
async def get_cache_stats(self) -> Dict[str, Any]:
"""
Get statistics about the cache.
Returns:
Dict with count, total_size_bytes, oldest_entry, newest_entry, configured
Raises:
FlacfetchServiceError: On request failure
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{self.base_url}/cache/stats",
headers=self._headers(),
timeout=30,
)
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Get cache stats request failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(
f"Get cache stats failed: {e.response.status_code} - {e.response.text}"
)
# Singleton client instance
_client: Optional[FlacfetchClient] = None
+44 -2
View File
@@ -438,10 +438,52 @@ Response:
{
"status": "success",
"job_id": "abc123",
"message": "Cleared 5 cached search results. Job reset to pending.",
"message": "Cleared 5 cached search results. Job reset to pending. Flacfetch cache also cleared.",
"previous_status": "awaiting_audio_selection",
"new_status": "pending",
"results_cleared": 5
"results_cleared": 5,
"flacfetch_cache_cleared": true,
"flacfetch_error": null
}
```
Note: This endpoint now also clears the flacfetch GCS cache for the artist/title combination, ensuring the next search hits trackers fresh.
#### Clear All Flacfetch Cache
```http
DELETE /api/admin/cache
Authorization: Bearer ADMIN_TOKEN
```
Clears the entire flacfetch search cache (GCS-backed). Use when flacfetch has been updated and you want all subsequent searches to use fresh tracker results.
Response:
```json
{
"status": "success",
"message": "Cleared 15 cache entries from flacfetch.",
"deleted_count": 15
}
```
#### Get Flacfetch Cache Stats
```http
GET /api/admin/cache/stats
Authorization: Bearer ADMIN_TOKEN
```
Returns statistics about the flacfetch search cache.
Response:
```json
{
"count": 42,
"total_size_bytes": 128000,
"oldest_entry": "2025-12-15T10:30:00Z",
"newest_entry": "2026-01-03T15:45:00Z",
"configured": true
}
```
+11 -3
View File
@@ -802,15 +802,23 @@ result = job.state_data['search_results'][selection_index]
**Solution**: Added admin endpoints and UI to manage cached searches:
- `GET /api/admin/audio-searches` - List jobs with cached results
- `POST /api/admin/audio-searches/{job_id}/clear-cache` - Clear cache and reset to pending
- Admin UI at `/admin/searches` shows which jobs have YouTube-only results vs lossless
- `POST /api/admin/audio-searches/{job_id}/clear-cache` - Clear both Firestore and flacfetch GCS cache
- `DELETE /api/admin/cache` - Clear entire flacfetch cache
- `GET /api/admin/cache/stats` - View cache statistics
- Admin UI at `/admin/searches` shows cache stats and "Clear All Cache" button
**Key insight**: There are TWO caches involved:
1. **Firestore** (`job.state_data.audio_search_results`) - stores results for user selection UI
2. **Flacfetch GCS** - caches tracker search responses (30-day TTL)
Both must be cleared for a truly fresh search. The clear-cache endpoint now clears both automatically.
**Future consideration**: Add automatic cache invalidation based on:
- Time (TTL)
- flacfetch version change
- User-initiated "re-search" button
**Lesson**: When caching external API results, consider how/when the cache should be invalidated. Permanent caches need admin tooling to manage stale data.
**Lesson**: When caching external API results, consider how/when the cache should be invalidated. Permanent caches need admin tooling to manage stale data. When multiple layers cache the same data, clearing one layer isn't enough - you need to clear all of them.
### GCS Downloads Preserve Directory Structure
+2
View File
@@ -36,6 +36,8 @@
## Recent Changes
- **Flacfetch Cache API Integration** (2026-01-03): Enhanced audio search cache management to also clear flacfetch's GCS cache. When admins clear a job's cache, both Firestore (job state) and flacfetch (GCS search results) are cleared simultaneously. Added "Clear All Cache" button and cache stats display to admin UI. New endpoints: `DELETE /api/admin/cache`, `GET /api/admin/cache/stats`. See [API.md](API.md#audio-search-management-admin).
- **Audio Search Cache Management** (2026-01-03): Added admin UI at `/admin/searches` to view and manage cached audio search results. Admins can clear stale caches (e.g., when flacfetch is updated with new providers) and allow users to re-search. Fixes issue where jobs showed only YouTube results when lossless sources should be available. See [API.md](API.md#audio-search-management-admin) and [LESSONS-LEARNED.md](LESSONS-LEARNED.md#firestore-cache-has-no-automatic-invalidation).
- **User Database Separation** (2026-01-03): Migrated karaoke-gen users from shared `users` collection to dedicated `gen_users` collection. This separates karaoke-gen user data from karaoke-decide user data (both apps share the same GCP project/Firestore instance). Migration script in `scripts/migrate_users_to_gen_users.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#firestore-collections) for collection details.
+138 -8
View File
@@ -1,7 +1,7 @@
"use client"
import { useEffect, useState, useCallback } from "react"
import { adminApi, AudioSearchJobSummary } from "@/lib/api"
import { adminApi, AudioSearchJobSummary, CacheStatsResponse } from "@/lib/api"
import { Button } from "@/components/ui/button"
import { Badge } from "@/components/ui/badge"
import {
@@ -43,6 +43,8 @@ import {
CheckCircle,
XCircle,
ExternalLink,
Database,
HardDrive,
} from "lucide-react"
import { useToast } from "@/hooks/use-toast"
@@ -60,11 +62,19 @@ export default function AdminSearchesPage() {
const [loading, setLoading] = useState(true)
const [statusFilter, setStatusFilter] = useState("all")
// Clear cache dialog
// Clear cache dialog (single job)
const [clearDialogOpen, setClearDialogOpen] = useState(false)
const [jobToClear, setJobToClear] = useState<AudioSearchJobSummary | null>(null)
const [clearing, setClearing] = useState(false)
// Clear all cache dialog
const [clearAllDialogOpen, setClearAllDialogOpen] = useState(false)
const [clearingAll, setClearingAll] = useState(false)
// Cache stats
const [cacheStats, setCacheStats] = useState<CacheStatsResponse | null>(null)
const [statsLoading, setStatsLoading] = useState(false)
const loadSearches = useCallback(async () => {
try {
setLoading(true)
@@ -85,9 +95,24 @@ export default function AdminSearchesPage() {
}
}, [statusFilter, toast])
const loadCacheStats = useCallback(async () => {
try {
setStatsLoading(true)
const stats = await adminApi.getCacheStats()
setCacheStats(stats)
} catch (err: any) {
// Silently fail - stats are optional
console.error("Failed to load cache stats:", err)
setCacheStats(null)
} finally {
setStatsLoading(false)
}
}, [])
useEffect(() => {
loadSearches()
}, [loadSearches])
loadCacheStats()
}, [loadSearches, loadCacheStats])
const handleClearCache = async () => {
if (!jobToClear) return
@@ -102,6 +127,7 @@ export default function AdminSearchesPage() {
setClearDialogOpen(false)
setJobToClear(null)
loadSearches()
loadCacheStats()
} catch (err: any) {
toast({
title: "Error",
@@ -113,6 +139,35 @@ export default function AdminSearchesPage() {
}
}
const handleClearAllCache = async () => {
try {
setClearingAll(true)
const result = await adminApi.clearAllCache()
toast({
title: "All Cache Cleared",
description: result.message,
})
setClearAllDialogOpen(false)
loadCacheStats()
} catch (err: any) {
toast({
title: "Error",
description: err.message || "Failed to clear all cache",
variant: "destructive",
})
} finally {
setClearingAll(false)
}
}
const formatBytes = (bytes: number): string => {
if (bytes === 0) return "0 B"
const k = 1024
const sizes = ["B", "KB", "MB", "GB"]
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + " " + sizes[i]
}
const getStatusVariant = (status: string) => {
if (status === "complete" || status === "prep_complete") return "default"
if (status === "failed") return "destructive"
@@ -149,12 +204,53 @@ export default function AdminSearchesPage() {
View and manage cached audio search results
</p>
</div>
<Button variant="outline" size="sm" onClick={loadSearches} disabled={loading}>
<RefreshCw className={`w-4 h-4 mr-2 ${loading ? "animate-spin" : ""}`} />
Refresh
</Button>
<div className="flex items-center gap-2">
<Button variant="outline" size="sm" onClick={() => { loadSearches(); loadCacheStats() }} disabled={loading}>
<RefreshCw className={`w-4 h-4 mr-2 ${loading ? "animate-spin" : ""}`} />
Refresh
</Button>
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="outline"
size="sm"
className="text-red-500 hover:text-red-400 hover:bg-red-500/10 border-red-500/30"
onClick={() => setClearAllDialogOpen(true)}
>
<Database className="w-4 h-4 mr-2" />
Clear All Cache
</Button>
</TooltipTrigger>
<TooltipContent>
<p>Clear entire flacfetch search cache</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
</div>
{/* Cache Stats */}
{cacheStats && (
<div className="flex items-center gap-4 text-sm text-muted-foreground bg-muted/30 rounded-lg px-4 py-2">
<div className="flex items-center gap-2">
<HardDrive className="w-4 h-4" />
<span>Flacfetch Cache:</span>
</div>
<Badge variant="outline" className="font-mono">
{cacheStats.count} entries
</Badge>
<Badge variant="outline" className="font-mono">
{formatBytes(cacheStats.total_size_bytes)}
</Badge>
{cacheStats.newest_entry && (
<span className="text-xs">
Latest: {new Date(cacheStats.newest_entry).toLocaleDateString()}
</span>
)}
</div>
)}
{/* Filters */}
<div className="flex gap-4">
<Select value={statusFilter} onValueChange={setStatusFilter}>
@@ -305,7 +401,7 @@ export default function AdminSearchesPage() {
</div>
)}
{/* Clear Cache Dialog */}
{/* Clear Cache Dialog (single job) */}
<AlertDialog open={clearDialogOpen} onOpenChange={setClearDialogOpen}>
<AlertDialogContent>
<AlertDialogHeader>
@@ -338,6 +434,40 @@ export default function AdminSearchesPage() {
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
{/* Clear All Cache Dialog */}
<AlertDialog open={clearAllDialogOpen} onOpenChange={setClearAllDialogOpen}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Clear Entire Flacfetch Cache?</AlertDialogTitle>
<AlertDialogDescription>
This will delete all cached search results from flacfetch&apos;s GCS cache
{cacheStats && (
<span className="font-medium"> ({cacheStats.count} entries, {formatBytes(cacheStats.total_size_bytes)})</span>
)}.
<span className="block mt-2 text-orange-400">
All subsequent searches will hit the trackers fresh. This does NOT affect
the Firestore job caches - use individual &quot;Clear Cache&quot; buttons for those.
</span>
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleClearAllCache}
disabled={clearingAll}
className="bg-red-600 hover:bg-red-500"
>
{clearingAll ? (
<Loader2 className="w-4 h-4 animate-spin mr-2" />
) : (
<Database className="w-4 h-4 mr-2" />
)}
Clear All Cache
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
)
}
+38
View File
@@ -818,6 +818,22 @@ export interface ClearSearchCacheResponse {
previous_status: string;
new_status: string;
results_cleared: number;
flacfetch_cache_cleared: boolean;
flacfetch_error?: string;
}
export interface ClearAllCacheResponse {
status: string;
message: string;
deleted_count: number;
}
export interface CacheStatsResponse {
count: number;
total_size_bytes: number;
oldest_entry?: string;
newest_entry?: string;
configured: boolean;
}
// Admin API namespace
@@ -1000,6 +1016,28 @@ export const adminApi = {
);
return handleResponse(response);
},
/**
* Clear all flacfetch search cache
*/
async clearAllCache(): Promise<ClearAllCacheResponse> {
const response = await fetch(
`${API_BASE_URL}/api/admin/cache`,
{ method: 'DELETE', headers: getAuthHeaders() }
);
return handleResponse(response);
},
/**
* Get flacfetch cache statistics
*/
async getCacheStats(): Promise<CacheStatsResponse> {
const response = await fetch(
`${API_BASE_URL}/api/admin/cache/stats`,
{ headers: getAuthHeaders() }
);
return handleResponse(response);
},
};
export { ApiError };
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "karaoke-gen-frontend",
"version": "0.79.0",
"version": "0.80.0",
"private": true,
"scripts": {
"build": "next build",
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.87.1"
version = "0.88.0"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"