mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary
Move `worker_logs` from embedded array in job documents to a Firestore
subcollection (`jobs/{job_id}/logs`) to avoid the 1MB document size
limit. This fixes job failures when logs exceed the limit during
long-running operations like video encoding.
**Problem:** Job `501258e1` failed when logs reached 1.26MB (2,547 log
entries), representing 98.6% of the maximum document size.
## Changes
### New Files
- `backend/models/worker_log.py` - WorkerLogEntry dataclass with factory
methods, serialization (to_dict, to_legacy_dict, from_dict)
- `backend/tests/test_worker_log_subcollection.py` - 24 unit tests
- `backend/tests/emulator/test_worker_logs_subcollection.py` - 11
integration tests using Firestore emulator
### Modified Files
- `backend/services/firestore_service.py` - Added subcollection methods
(append, get, count, delete)
- `backend/services/job_manager.py` - Use subcollection with fallback to
legacy embedded arrays
- `backend/api/routes/jobs.py` - Updated logs endpoint to use new
methods
- `backend/config.py` - Added `USE_LOG_SUBCOLLECTION` feature flag
(default: true)
- `infrastructure/__main__.py` - Added TTL policy (30 days) and
composite index for logs subcollection
- `pyproject.toml` - Version bump 0.89.6 → 0.90.0
### Documentation
- `docs/README.md` - Added to Recent Changes
- `docs/ARCHITECTURE.md` - Added logs subcollection to Firestore
Collections table
- `docs/LESSONS-LEARNED.md` - Added "Firestore Document 1MB Limit with
Embedded Arrays" section
## Testing
- [x] 24 unit tests pass
(`backend/tests/test_worker_log_subcollection.py`)
- [x] 11 emulator integration tests pass
(`backend/tests/emulator/test_worker_logs_subcollection.py`)
- [x] All 68 existing emulator tests pass
## Review
- [x] CodeRabbit CLI review completed locally
- [x] Review feedback addressed (timestamp formatting fix)
## Rollback Plan
Set `USE_LOG_SUBCOLLECTION=false` environment variable and redeploy. New
logs will go back to embedded array; subcollection logs will remain but
won't be read.
@coderabbitai ignore
---
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude <noreply@anthropic.com>
167 lines
7.5 KiB
Python
167 lines
7.5 KiB
Python
"""
|
|
Configuration management for the karaoke generation backend.
|
|
"""
|
|
import os
|
|
import logging
|
|
from typing import Optional, Dict
|
|
from pydantic_settings import BaseSettings
|
|
from google.cloud import secretmanager
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
"""Application settings."""
|
|
|
|
# Google Cloud
|
|
google_cloud_project: str = os.getenv("GOOGLE_CLOUD_PROJECT", "")
|
|
gcs_bucket_name: str = os.getenv("GCS_BUCKET_NAME", "karaoke-gen-storage")
|
|
gcs_temp_bucket: str = os.getenv("GCS_TEMP_BUCKET", "karaoke-gen-temp")
|
|
gcs_output_bucket: str = os.getenv("GCS_OUTPUT_BUCKET", "karaoke-gen-outputs")
|
|
firestore_collection: str = os.getenv("FIRESTORE_COLLECTION", "jobs")
|
|
|
|
# Audio Separator API (for GPU processing)
|
|
audio_separator_api_url: Optional[str] = os.getenv("AUDIO_SEPARATOR_API_URL")
|
|
|
|
# External APIs (can be set via env or Secret Manager)
|
|
audioshake_api_key: Optional[str] = os.getenv("AUDIOSHAKE_API_KEY")
|
|
genius_api_key: Optional[str] = os.getenv("GENIUS_API_KEY")
|
|
spotify_cookie: Optional[str] = os.getenv("SPOTIFY_COOKIE_SP_DC")
|
|
rapidapi_key: Optional[str] = os.getenv("RAPIDAPI_KEY")
|
|
|
|
# Authentication
|
|
admin_tokens: Optional[str] = os.getenv("ADMIN_TOKENS") # Comma-separated list
|
|
|
|
# Application
|
|
environment: str = os.getenv("ENVIRONMENT", "development")
|
|
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
|
|
|
# Processing
|
|
max_concurrent_jobs: int = int(os.getenv("MAX_CONCURRENT_JOBS", "5"))
|
|
job_timeout_seconds: int = int(os.getenv("JOB_TIMEOUT_SECONDS", "3600"))
|
|
|
|
# Agentic AI Correction (for lyrics correction via LLM)
|
|
# When enabled, uses Gemini via Vertex AI for intelligent lyrics correction
|
|
use_agentic_ai: bool = os.getenv("USE_AGENTIC_AI", "true").lower() in ("true", "1", "yes")
|
|
agentic_ai_model: str = os.getenv("AGENTIC_AI_MODEL", "vertexai/gemini-3-flash-preview")
|
|
# Timeout for agentic correction in seconds. If correction takes longer, abort and
|
|
# use uncorrected transcription - human review will fix any issues.
|
|
agentic_correction_timeout_seconds: int = int(os.getenv("AGENTIC_CORRECTION_TIMEOUT_SECONDS", "180"))
|
|
|
|
# Cloud Tasks (for scalable worker coordination)
|
|
# When enabled, workers are triggered via Cloud Tasks for guaranteed delivery
|
|
# When disabled (default), workers are triggered via direct HTTP (for development)
|
|
enable_cloud_tasks: bool = os.getenv("ENABLE_CLOUD_TASKS", "false").lower() in ("true", "1", "yes")
|
|
gcp_region: str = os.getenv("GCP_REGION", "us-central1")
|
|
|
|
# Cloud Run Jobs (for long-running video encoding)
|
|
# When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs
|
|
# instead of Cloud Tasks. This supports encoding times >30 minutes (up to 24 hours).
|
|
# Default is false - Cloud Tasks is sufficient for most videos (15-20 min).
|
|
use_cloud_run_jobs_for_video: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_VIDEO", "false").lower() in ("true", "1", "yes")
|
|
|
|
# GCE Encoding Worker (for high-performance video encoding)
|
|
# When enabled, video encoding is offloaded to a dedicated C4 GCE instance
|
|
# with faster CPU (Intel Granite Rapids 3.9 GHz) instead of Cloud Run.
|
|
# This provides 2-3x faster encoding times for CPU-bound FFmpeg libx264 encoding.
|
|
use_gce_encoding: bool = os.getenv("USE_GCE_ENCODING", "false").lower() in ("true", "1", "yes")
|
|
encoding_worker_url: Optional[str] = os.getenv("ENCODING_WORKER_URL") # e.g., http://136.119.50.148:8080
|
|
encoding_worker_api_key: Optional[str] = os.getenv("ENCODING_WORKER_API_KEY")
|
|
|
|
# Storage paths
|
|
temp_dir: str = os.getenv("TEMP_DIR", "/tmp/karaoke-gen")
|
|
|
|
# Worker logs storage mode
|
|
# When enabled, worker logs are stored in a Firestore subcollection (jobs/{job_id}/logs)
|
|
# instead of an embedded array. This avoids the 1MB document size limit.
|
|
# Default is true for new deployments.
|
|
use_log_subcollection: bool = os.getenv("USE_LOG_SUBCOLLECTION", "true").lower() in ("true", "1", "yes")
|
|
|
|
# Flacfetch remote service (for torrent downloads)
|
|
# When configured, audio search uses the remote flacfetch HTTP API instead of local flacfetch.
|
|
# This is required for torrent downloads since Cloud Run doesn't support BitTorrent.
|
|
flacfetch_api_url: Optional[str] = os.getenv("FLACFETCH_API_URL") # e.g., http://10.0.0.5:8080
|
|
flacfetch_api_key: Optional[str] = os.getenv("FLACFETCH_API_KEY")
|
|
|
|
# Default distribution settings (can be overridden per-request)
|
|
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
|
|
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
|
|
# Strip whitespace/newlines from webhook URL - common issue when env vars are set with trailing newlines
|
|
default_discord_webhook_url: Optional[str] = (
|
|
os.getenv("DEFAULT_DISCORD_WEBHOOK_URL", "").strip() or None
|
|
)
|
|
|
|
# Default values for web service jobs (YouTube/Dropbox distribution)
|
|
default_enable_youtube_upload: bool = os.getenv("DEFAULT_ENABLE_YOUTUBE_UPLOAD", "false").lower() in ("true", "1", "yes")
|
|
default_brand_prefix: Optional[str] = os.getenv("DEFAULT_BRAND_PREFIX")
|
|
default_youtube_description: str = os.getenv(
|
|
"DEFAULT_YOUTUBE_DESCRIPTION",
|
|
"Karaoke video created with Nomad Karaoke (https://nomadkaraoke.com)\n\n"
|
|
"AI-powered vocal separation and synchronized lyrics.\n\n"
|
|
"#karaoke #music #singing #instrumental #lyrics"
|
|
)
|
|
|
|
# Secret Manager cache
|
|
_secret_cache: Dict[str, str] = {}
|
|
|
|
class Config:
|
|
env_file = ".env"
|
|
case_sensitive = False
|
|
|
|
def get_secret(self, secret_id: str) -> Optional[str]:
|
|
"""
|
|
Get a secret from Google Secret Manager.
|
|
|
|
Caches secrets in memory to avoid repeated API calls.
|
|
Falls back to environment variables if Secret Manager unavailable.
|
|
|
|
Args:
|
|
secret_id: Secret name (e.g., "audioshake-api-key")
|
|
|
|
Returns:
|
|
Secret value or None if not found
|
|
"""
|
|
# Check cache first
|
|
if secret_id in self._secret_cache:
|
|
return self._secret_cache[secret_id]
|
|
|
|
# Check environment variable (development mode)
|
|
env_var = secret_id.upper().replace('-', '_')
|
|
env_value = os.getenv(env_var)
|
|
if env_value:
|
|
logger.debug(f"Using {secret_id} from environment variable")
|
|
self._secret_cache[secret_id] = env_value
|
|
return env_value
|
|
|
|
# Try Secret Manager (production mode)
|
|
if not self.google_cloud_project:
|
|
logger.warning(f"Cannot fetch secret {secret_id}: No GCP project configured")
|
|
return None
|
|
|
|
try:
|
|
client = secretmanager.SecretManagerServiceClient()
|
|
name = f"projects/{self.google_cloud_project}/secrets/{secret_id}/versions/latest"
|
|
response = client.access_secret_version(request={"name": name})
|
|
# Strip whitespace/newlines - common issue when secrets are created with trailing newlines
|
|
secret_value = response.payload.data.decode('UTF-8').strip()
|
|
|
|
# Cache it
|
|
self._secret_cache[secret_id] = secret_value
|
|
logger.info(f"Loaded secret {secret_id} from Secret Manager")
|
|
return secret_value
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to load secret {secret_id} from Secret Manager: {e}")
|
|
return None
|
|
|
|
|
|
# Global settings instance
|
|
settings = Settings()
|
|
|
|
|
|
def get_settings() -> Settings:
|
|
"""Get the global settings instance."""
|
|
return settings
|
|
|