Files
karaoke-gen/tests/unit/test_lyrics_cache_service.py
Andrew Beveridge ☄️andClaude 2edd5ee7c1 Add GCS-backed cache for LyricsTranscriber to persist API responses (#158)
## Summary

- Add GCS-backed cache persistence for LyricsTranscriber to avoid
redundant API calls
- AudioShake transcription and lyrics provider responses now cached
across Cloud Run instances
- Significantly reduces API costs and speeds up repeated processing of
same songs

## Changes

- **New:** `backend/services/lyrics_cache_service.py` - Service to sync
cache files with GCS
  - Computes MD5 hashes matching LyricsTranscriber's algorithm
  - Downloads relevant cache files from GCS before processing
  - Uploads new cache files to GCS after processing
  - Best-effort with graceful error handling

- **Modified:** `backend/workers/lyrics_worker.py`
  - Sets `LYRICS_TRANSCRIBER_CACHE_DIR` environment variable
  - Syncs cache from GCS before transcription
  - Syncs cache to GCS after transcription
  - Error handling ensures cache failures don't fail jobs

- **New:** `tests/unit/test_lyrics_cache_service.py` - 16 unit tests
- **Updated:** `docs/README.md` and `docs/LESSONS-LEARNED.md`

## Cache Structure

| Type | Key | Files |
|------|-----|-------|
| Transcription | MD5(audio_file) | `audioshake_{hash}_raw.json`,
`audioshake_{hash}_converted.json` |
| Lyrics | MD5(artist_title) | `genius_{hash}_raw.json`,
`lrclib_{hash}_converted.json`, etc. |

Providers cached: `audioshake`, `whisper`, `genius`, `spotify`,
`lrclib`, `musixmatch`

## Testing

- [x] All 16 new unit tests pass
- [x] All 57 existing tests pass (`make test`)
- [x] CodeRabbit CLI review completed locally

## Review

- [x] Local CodeRabbit review completed
- [x] Review feedback addressed (added error handling for cache
operations)

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <noreply@anthropic.com>
2026-01-02 17:08:11 -05:00

360 lines
13 KiB
Python

"""
Unit tests for LyricsCacheService.
Tests the GCS cache synchronization service for LyricsTranscriber.
"""
import hashlib
import json
import os
import tempfile
from unittest.mock import MagicMock, patch
import pytest
from backend.services.lyrics_cache_service import (
LyricsCacheService,
TRANSCRIPTION_PROVIDERS,
LYRICS_PROVIDERS,
CACHE_SUFFIXES,
)
class TestHashComputation:
"""Tests for hash computation methods."""
def test_compute_audio_hash_matches_lyrics_transcriber(self, temp_dir):
"""Audio hash should match LyricsTranscriber's _get_file_hash implementation."""
# Create a test audio file
audio_path = os.path.join(temp_dir, "test_audio.wav")
test_content = b"fake audio content for testing hash computation"
with open(audio_path, "wb") as f:
f.write(test_content)
# Compute hash using our service
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
computed_hash = service.compute_audio_hash(audio_path)
# Compute expected hash (same algorithm as LyricsTranscriber)
md5_hash = hashlib.md5()
with open(audio_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
md5_hash.update(chunk)
expected_hash = md5_hash.hexdigest()
assert computed_hash == expected_hash
def test_compute_lyrics_hash_matches_lyrics_transcriber(self):
"""Lyrics hash should match LyricsTranscriber's _get_artist_title_hash implementation."""
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
artist = "Test Artist"
title = "Test Song"
computed_hash = service.compute_lyrics_hash(artist, title)
# Expected hash (same algorithm as LyricsTranscriber)
combined = f"{artist.lower()}_{title.lower()}"
expected_hash = hashlib.md5(combined.encode()).hexdigest()
assert computed_hash == expected_hash
def test_compute_lyrics_hash_case_insensitive(self):
"""Lyrics hash should be case-insensitive."""
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
hash1 = service.compute_lyrics_hash("Artist", "Title")
hash2 = service.compute_lyrics_hash("ARTIST", "TITLE")
hash3 = service.compute_lyrics_hash("artist", "title")
assert hash1 == hash2 == hash3
class TestCacheFilenames:
"""Tests for cache filename generation."""
def test_get_cache_filenames_transcription_providers(self):
"""Should generate correct filenames for transcription providers."""
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
filenames = service._get_cache_filenames(TRANSCRIPTION_PROVIDERS, "abc123")
expected = []
for provider in TRANSCRIPTION_PROVIDERS:
for suffix in CACHE_SUFFIXES:
expected.append(f"{provider}_abc123_{suffix}.json")
assert sorted(filenames) == sorted(expected)
def test_get_cache_filenames_lyrics_providers(self):
"""Should generate correct filenames for lyrics providers."""
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
filenames = service._get_cache_filenames(LYRICS_PROVIDERS, "def456")
expected = []
for provider in LYRICS_PROVIDERS:
for suffix in CACHE_SUFFIXES:
expected.append(f"{provider}_def456_{suffix}.json")
assert sorted(filenames) == sorted(expected)
class TestSyncCacheFromGCS:
"""Tests for downloading cache from GCS."""
def test_sync_cache_from_gcs_downloads_existing_files(self, temp_dir):
"""Should download files that exist in GCS."""
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
# Configure mock to simulate some files exist in GCS
def file_exists(path):
# Simulate audioshake_abc123_raw.json exists
return "audioshake_abc123_raw.json" in path
mock_storage.file_exists.side_effect = file_exists
stats = service.sync_cache_from_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
# Should have downloaded 1 file (audioshake_abc123_raw.json)
assert stats["downloaded"] == 1
# Should have not found the rest
assert stats["not_found"] > 0
assert stats["errors"] == 0
# Verify download_file was called for the existing file
mock_storage.download_file.assert_called_once()
def test_sync_cache_from_gcs_creates_directory(self, temp_dir):
"""Should create the local cache directory if it doesn't exist."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = False
service = LyricsCacheService(storage=mock_storage)
cache_dir = os.path.join(temp_dir, "new_cache_dir")
assert not os.path.exists(cache_dir)
service.sync_cache_from_gcs(
local_cache_dir=cache_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
assert os.path.exists(cache_dir)
def test_sync_cache_from_gcs_handles_errors(self, temp_dir):
"""Should handle download errors gracefully."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = True
mock_storage.download_file.side_effect = Exception("Network error")
service = LyricsCacheService(storage=mock_storage)
stats = service.sync_cache_from_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
# All files should have errors
assert stats["errors"] > 0
assert stats["downloaded"] == 0
class TestSyncCacheToGCS:
"""Tests for uploading cache to GCS."""
def test_sync_cache_to_gcs_uploads_new_files(self, temp_dir):
"""Should upload files that don't exist in GCS."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = False
service = LyricsCacheService(storage=mock_storage)
# Create a cache file in temp_dir
cache_file = os.path.join(temp_dir, "audioshake_abc123_raw.json")
with open(cache_file, "w") as f:
json.dump({"test": "data"}, f)
stats = service.sync_cache_to_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
assert stats["uploaded"] == 1
mock_storage.upload_file.assert_called_once()
def test_sync_cache_to_gcs_skips_existing_files(self, temp_dir):
"""Should skip files that already exist in GCS."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = True
service = LyricsCacheService(storage=mock_storage)
# Create a cache file in temp_dir
cache_file = os.path.join(temp_dir, "audioshake_abc123_raw.json")
with open(cache_file, "w") as f:
json.dump({"test": "data"}, f)
stats = service.sync_cache_to_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
assert stats["uploaded"] == 0
assert stats["skipped"] == 1
mock_storage.upload_file.assert_not_called()
def test_sync_cache_to_gcs_ignores_unexpected_files(self, temp_dir):
"""Should ignore files that don't match expected cache patterns."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = False
service = LyricsCacheService(storage=mock_storage)
# Create an unexpected file
unexpected_file = os.path.join(temp_dir, "random_file.txt")
with open(unexpected_file, "w") as f:
f.write("not a cache file")
# Create a file with wrong hash
wrong_hash_file = os.path.join(temp_dir, "audioshake_wrong_hash_raw.json")
with open(wrong_hash_file, "w") as f:
json.dump({"test": "data"}, f)
stats = service.sync_cache_to_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
# Should not upload any files
assert stats["uploaded"] == 0
mock_storage.upload_file.assert_not_called()
def test_sync_cache_to_gcs_handles_missing_directory(self, temp_dir):
"""Should handle missing cache directory gracefully."""
mock_storage = MagicMock()
service = LyricsCacheService(storage=mock_storage)
stats = service.sync_cache_to_gcs(
local_cache_dir=os.path.join(temp_dir, "nonexistent"),
audio_hash="abc123",
lyrics_hash="def456",
)
assert stats["uploaded"] == 0
assert stats["skipped"] == 0
assert stats["errors"] == 0
def test_sync_cache_to_gcs_handles_upload_errors(self, temp_dir):
"""Should handle upload errors gracefully."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = False
mock_storage.upload_file.side_effect = Exception("Upload failed")
service = LyricsCacheService(storage=mock_storage)
# Create a cache file
cache_file = os.path.join(temp_dir, "audioshake_abc123_raw.json")
with open(cache_file, "w") as f:
json.dump({"test": "data"}, f)
stats = service.sync_cache_to_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
assert stats["errors"] == 1
assert stats["uploaded"] == 0
class TestGCSPathStructure:
"""Tests for GCS path structure."""
def test_gcs_cache_prefix(self):
"""GCS cache prefix should be correct."""
assert LyricsCacheService.GCS_CACHE_PREFIX == "lyrics-transcriber-cache/"
def test_sync_uses_correct_gcs_paths(self, temp_dir):
"""Should use correct GCS paths when downloading/uploading."""
mock_storage = MagicMock()
mock_storage.file_exists.return_value = True
service = LyricsCacheService(storage=mock_storage)
service.sync_cache_from_gcs(
local_cache_dir=temp_dir,
audio_hash="abc123",
lyrics_hash="def456",
)
# Check that file_exists was called with correct GCS path prefix
calls = mock_storage.file_exists.call_args_list
for call in calls:
gcs_path = call[0][0]
assert gcs_path.startswith("lyrics-transcriber-cache/")
class TestEndToEnd:
"""End-to-end tests for cache sync workflow."""
def test_round_trip_cache_sync(self, temp_dir):
"""Test uploading and then downloading the same cache file."""
# We'll simulate this with two separate temp dirs
upload_dir = os.path.join(temp_dir, "upload")
download_dir = os.path.join(temp_dir, "download")
os.makedirs(upload_dir)
os.makedirs(download_dir)
# Create a test cache file
test_data = {"transcription": "test data", "words": [1, 2, 3]}
cache_filename = "audioshake_abc123_raw.json"
upload_path = os.path.join(upload_dir, cache_filename)
with open(upload_path, "w") as f:
json.dump(test_data, f)
# Create mock that tracks uploaded files
uploaded_files = {}
def mock_upload(local_path, gcs_path):
with open(local_path, "r") as f:
uploaded_files[gcs_path] = f.read()
return gcs_path
def mock_download(gcs_path, local_path):
if gcs_path in uploaded_files:
with open(local_path, "w") as f:
f.write(uploaded_files[gcs_path])
return local_path
def mock_exists(gcs_path):
return gcs_path in uploaded_files
mock_storage = MagicMock()
mock_storage.upload_file.side_effect = mock_upload
mock_storage.download_file.side_effect = mock_download
mock_storage.file_exists.side_effect = mock_exists
service = LyricsCacheService(storage=mock_storage)
# Upload
upload_stats = service.sync_cache_to_gcs(upload_dir, "abc123", "def456")
assert upload_stats["uploaded"] == 1
# Download to new location
download_stats = service.sync_cache_from_gcs(download_dir, "abc123", "def456")
assert download_stats["downloaded"] == 1
# Verify downloaded content matches
downloaded_path = os.path.join(download_dir, cache_filename)
with open(downloaded_path, "r") as f:
downloaded_data = json.load(f)
assert downloaded_data == test_data