Files
karaoke-gen/pyproject.toml
T
Andrew Beveridge ☄️andClaude ae85ec0a09 feat: Move worker logs to Firestore subcollection to avoid 1MB limit (#195)
## Summary

Move `worker_logs` from embedded array in job documents to a Firestore
subcollection (`jobs/{job_id}/logs`) to avoid the 1MB document size
limit. This fixes job failures when logs exceed the limit during
long-running operations like video encoding.

**Problem:** Job `501258e1` failed when logs reached 1.26MB (2,547 log
entries), representing 98.6% of the maximum document size.

## Changes

### New Files
- `backend/models/worker_log.py` - WorkerLogEntry dataclass with factory
methods, serialization (to_dict, to_legacy_dict, from_dict)
- `backend/tests/test_worker_log_subcollection.py` - 24 unit tests
- `backend/tests/emulator/test_worker_logs_subcollection.py` - 11
integration tests using Firestore emulator

### Modified Files
- `backend/services/firestore_service.py` - Added subcollection methods
(append, get, count, delete)
- `backend/services/job_manager.py` - Use subcollection with fallback to
legacy embedded arrays
- `backend/api/routes/jobs.py` - Updated logs endpoint to use new
methods
- `backend/config.py` - Added `USE_LOG_SUBCOLLECTION` feature flag
(default: true)
- `infrastructure/__main__.py` - Added TTL policy (30 days) and
composite index for logs subcollection
- `pyproject.toml` - Version bump 0.89.6 → 0.90.0

### Documentation
- `docs/README.md` - Added to Recent Changes
- `docs/ARCHITECTURE.md` - Added logs subcollection to Firestore
Collections table
- `docs/LESSONS-LEARNED.md` - Added "Firestore Document 1MB Limit with
Embedded Arrays" section

## Testing

- [x] 24 unit tests pass
(`backend/tests/test_worker_log_subcollection.py`)
- [x] 11 emulator integration tests pass
(`backend/tests/emulator/test_worker_logs_subcollection.py`)
- [x] All 68 existing emulator tests pass

## Review

- [x] CodeRabbit CLI review completed locally
- [x] Review feedback addressed (timestamp formatting fix)

## Rollback Plan

Set `USE_LOG_SUBCOLLECTION=false` environment variable and redeploy. New
logs will go back to embedded array; subcollection logs will remain but
won't be read.

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <noreply@anthropic.com>
2026-01-04 20:14:25 -05:00

161 lines
5.1 KiB
TOML

[tool.poetry]
name = "karaoke-gen"
version = "0.90.0"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
readme = "README.md"
packages = [
{ include = "karaoke_gen" },
{ include = "lyrics_transcriber", from = "lyrics_transcriber_temp" }
]
include = [
"karaoke_gen/instrumental_review/static/*",
"lyrics_transcriber_temp/lyrics_transcriber/frontend/web_assets",
"lyrics_transcriber_temp/lyrics_transcriber/frontend/web_assets/**/*",
"lyrics_transcriber_temp/lyrics_transcriber/output/fonts/*",
"lyrics_transcriber_temp/lyrics_transcriber/output/cdgmaker/images/*",
"lyrics_transcriber_temp/lyrics_transcriber/output/cdgmaker/transitions/*"
]
homepage = "https://github.com/nomadkaraoke/karaoke-gen"
repository = "https://github.com/nomadkaraoke/karaoke-gen"
documentation = "https://github.com/nomadkaraoke/karaoke-gen/blob/main/README.md"
[tool.poetry.dependencies]
python = ">=3.10,<3.14"
torch = ">=2.7"
requests = ">=2"
beautifulsoup4 = ">=4"
flacfetch = ">=0.9.0"
lyricsgenius = ">=3"
fetch-lyrics-from-genius = ">=0.1"
pillow = ">=10.1"
pyinstaller = ">=6.3"
google-api-python-client = "*"
google-auth = "*"
google-auth-oauthlib = "*"
google-auth-httplib2 = "*"
thefuzz = ">=0.22"
numpy = ">=2"
audio-separator = { version = ">=0.34.0", extras = ["cpu"] }
lyrics-converter = ">=0.2.1"
# lyrics-transcriber is vendored in lyrics_transcriber_temp/ and included via packages
# Dependencies from vendored lyrics_transcriber:
mutagen = ">=1.47" # For LRCLIB provider to get track duration for better matching
python-slugify = ">=8"
syrics = ">=0"
karaoke-lyrics-processor = ">=0.6"
dropbox = ">=12"
spacy = ">=3.8.7"
srsly = ">=2.5.1"
tqdm = ">=4.67"
python-levenshtein = ">=0.26"
transformers = ">=4.47"
metaphone = ">=0.6"
nltk = ">=3.9"
spacy-syllables = ">=3"
syllables = ">=1"
fonttools = ">=4.55"
ollama = ">=0.4.7"
shortuuid = ">=1.0.13"
nest-asyncio = ">=1.5"
openai = ">=1.63.2"
langgraph = ">=0.2.0"
langchain = ">=0.3.0"
langchain-core = ">=0.3.0"
langchain-openai = ">=0.2.0"
langchain-anthropic = ">=0.2.0"
langchain-ollama = ">=0.2.0"
langchain-google-genai = ">=2.0.0"
langfuse = ">=3.0.0"
jiwer = ">=3.0.0"
kbputils = "^0.0.16"
attrs = ">=24.2.0"
cattrs = ">=24.1.2"
toml = ">=0.10"
argparse = ">=1.4.0"
psutil = "^7.0.0"
pyperclip = "*"
pytest-asyncio = "*"
ffmpeg-python = "^0.2.0"
python-multipart = "^0.0.20"
yt-dlp = ">=2024.0.0"
matplotlib = ">=3"
# Backend API dependencies
fastapi = ">=0.104.0"
uvicorn = { version = ">=0.24.0", extras = ["standard"] }
pydantic = ">=2.5.0"
pydantic-settings = ">=2.1.0"
httpx = ">=0.25.0"
google-cloud-firestore = ">=2.14.0"
google-cloud-storage = ">=2.14.0"
google-cloud-secret-manager = ">=2.18.0"
google-cloud-tasks = ">=2.16.0" # Cloud Tasks for worker coordination (Phase 1 scalability)
google-cloud-run = ">=0.10.0" # Cloud Run Jobs for long-running video encoding (Phase 2 scalability)
python-dotenv = ">=1.0.0"
pydub = ">=0.25.1"
# Payment and email services
stripe = ">=7.0.0"
sendgrid = ">=6.10.0"
email-validator = ">=2.0.0" # For Pydantic EmailStr validation
# OpenTelemetry for distributed tracing
opentelemetry-api = ">=1.20.0"
opentelemetry-sdk = ">=1.20.0"
opentelemetry-instrumentation-fastapi = ">=0.41b0"
opentelemetry-instrumentation-httpx = ">=0.41b0"
opentelemetry-instrumentation-logging = ">=0.41b0"
opentelemetry-exporter-gcp-trace = ">=1.6.0"
opentelemetry-resourcedetector-gcp = ">=1.6.0a0"
# Optional: Local Whisper transcription (install with: pip install karaoke-gen[local-whisper])
whisper-timestamped = { version = ">=1.15.0", optional = true }
aiohttp = "^3.13.2"
[tool.poetry.group.dev.dependencies]
black = ">=23"
poetry = "*"
pytest = ">=7.0"
pytest-cov = ">=4.0"
pytest-mock = ">=3.10"
pytest-asyncio = "*"
[tool.poetry.extras]
# Local Whisper transcription support - optional, heavy dependencies
# Install with: pip install karaoke-gen[local-whisper]
# For CPU-only installation (recommended for systems without GPU):
# pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu
# pip install karaoke-gen[local-whisper]
local-whisper = ["whisper-timestamped"]
[tool.poetry.scripts]
karaoke-gen = 'karaoke_gen.utils.gen_cli:main'
karaoke-gen-remote = 'karaoke_gen.utils.remote_cli:main'
karaoke-bulk = 'karaoke_gen.utils.bulk_cli:main'
[tool.black]
line-length = 140
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.pytest.ini_options]
testpaths = ["tests/unit", "tests/integration", "backend/tests"]
python_files = ["test_*.py"]
addopts = [
"--asyncio-mode=auto",
"-v",
"--tb=short",
]
markers = [
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"integration: marks tests as integration tests",
"asyncio: mark test as async",
"emulator: marks tests requiring GCP emulators",
]
filterwarnings = [
"ignore::DeprecationWarning",
"ignore:'audioop' is deprecated:DeprecationWarning",
"ignore::RuntimeWarning",
"ignore::UserWarning",
]