From 70ea964ca9674c36fb1bb2020762964b5cf80db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 27 Aug 2026 02:48:00 -0400 Subject: [PATCH] feat(dev): full local backend for external contributors (no GCP creds needed) (#946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Motivated by external contributor Martín (netux) asking "is there an easy way to run the full backend without GCS?" — he couldn't test PR #895's backend changes locally. This PR makes the whole backend runnable locally against emulators with **zero GCP credentials and zero paid API keys**. ### What was broken for contributors - Workers migrated to Cloud Run Jobs for prod robustness (audio-download, audio-separation, lyrics-transcription, bulk-search) silently bypassed the direct-HTTP dev mode → local jobs stalled at PENDING forever - Signed URL generation called `google.auth.default()` → crashed with no credentials (review audio, downloads, theme previews, upload URLs) - Magic-link login 503'd locally (console email provider counted as "not configured") - Fresh emulator had no default theme → every job create returned 422 - `docs/DEVELOPMENT.md` told people to `cp .env.example .env` but the file didn't exist — and nothing loaded `.env` anyway - `run-backend-local.sh` defaulted to the production GCP project ### Changes - **worker_service**: CRJ-based workers run as local subprocesses (`python -m backend.workers.X --job-id Y`, exact CRJ parity) when `ENABLE_CLOUD_TASKS=false` **and** not `is_production()` (regression-tested both ways) - **storage_service**: signed URLs → plain fake-gcs-server URLs when `STORAGE_EMULATOR_HOST` set (GET via JSON API download path; PUT via unvalidated-signature path, enabled by `-public-host` on the emulator) - **users route**: magic-link 503 only in production; dev prints the link to the console - **file_upload bugfix**: a theme with style params but no asset files never got `style_params_gcs_path` written (empty-dict falsiness in `elif theme_style_assets:`) — surfaced by the minimal seed theme, latent in prod - **scripts/seed-local-data.py** (new): creates bucket + default theme; wired into `run-backend-local.sh --with-emulators`, which now uses `test-project` - **config**: `load_dotenv()` so `.env` actually works; new root `.env.example` - **CONTRIBUTING.md** (new): no-credentials quickstart; README + DEVELOPMENT.md link to it ### Verified end-to-end locally (no GCP credentials) Upload job → local audio separation (real models) → transcription → screens → `awaiting_review`; review endpoints all working including `/api/review/{job}/audio/vocals` (the PR #895 endpoint) and 302 → emulator audio URLs. Magic-link login round-trip via console link. ### Tests - 224 tests across affected files pass; full backend unit suite: 3969 passed - `backend/tests/test_emulator_integration.py` has 8 pre-existing failures that reproduce identically on unmodified `main` locally (broken theme-service mock in that file) — unrelated to this PR - New tests: dev-mode subprocess dispatch (audio + bulk-search), prod-mode never-Popen guard, emulator URL generation (GET + PUT) @coderabbitai ignore --------- Co-authored-by: Claude Fable 5 --- .env.example | 42 ++++++++++ .gitleaks.toml | 3 + CONTRIBUTING.md | 109 ++++++++++++++++++++++++++ README.md | 4 +- backend/api/routes/file_upload.py | 6 +- backend/api/routes/users.py | 7 +- backend/config.py | 5 ++ backend/services/storage_service.py | 23 +++++- backend/services/worker_service.py | 38 ++++++++- backend/tests/test_services.py | 82 +++++++++++++++++-- backend/tests/test_storage_service.py | 48 +++++++++++- docs/DEVELOPMENT.md | 18 ++++- pyproject.toml | 2 +- scripts/run-backend-local.sh | 19 +++-- scripts/seed-local-data.py | 81 +++++++++++++++++++ scripts/start-emulators.sh | 6 +- 16 files changed, 466 insertions(+), 27 deletions(-) create mode 100644 .env.example create mode 100644 CONTRIBUTING.md create mode 100644 scripts/seed-local-data.py diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..3bdcc56e --- /dev/null +++ b/.env.example @@ -0,0 +1,42 @@ +# karaoke-gen backend environment — copy to .env and adjust. +# +# NOTE: scripts/run-backend-local.sh --with-emulators sets all the required +# variables itself; you only need a .env for optional API keys or overrides. +# None of the keys below are required to boot the backend locally. + +# --- Core (defaults shown are what run-backend-local.sh uses in emulator mode) --- +# ENVIRONMENT=development +# GOOGLE_CLOUD_PROJECT=test-project +# GCS_BUCKET_NAME=test-bucket +# FIRESTORE_COLLECTION=test-jobs +# ADMIN_TOKENS=local-dev-token +# PORT=8000 +# FRONTEND_URL=http://localhost:3000 + +# --- Emulators (set automatically by run-backend-local.sh --with-emulators) --- +# FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 +# STORAGE_EMULATOR_HOST=http://127.0.0.1:4443 + +# --- Audio separation (pick one; without either, jobs stall at separation) --- +# Local CPU/GPU separation via the audio-separator package. Models are +# downloaded into this directory on first use (~2GB). CPU works but is slow. +# MODEL_DIR=./models +# ...or point at a remote audio-separator API service: +# AUDIO_SEPARATOR_API_URL= + +# --- Lyrics transcription --- +# The lyrics worker reads AUDIOSHAKE_API_TOKEN (not AUDIOSHAKE_API_KEY). +# Without it, transcription falls back to local Whisper if installed +# (poetry install --extras local-whisper). +# AUDIOSHAKE_API_TOKEN= + +# --- Lyrics sources (optional; improves reference lyrics) --- +# GENIUS_API_KEY= +# SPOTIFY_COOKIE_SP_DC= + +# --- Email (optional; without it, emails are logged to the console) --- +# POSTMARK_SERVER_TOKEN= + +# --- Payments (optional; only needed to exercise payment endpoints) --- +# STRIPE_SECRET_KEY= +# STRIPE_WEBHOOK_SECRET= diff --git a/.gitleaks.toml b/.gitleaks.toml index e4adf169..a1c4e3e4 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -57,4 +57,7 @@ regexes = [ # Firebase / GCP client API keys — public by design (identify the project, # access is gated by Firebase Security Rules). '''AIzaSy[A-Za-z0-9_-]{33}''', + + # Documented local-development placeholder token (CONTRIBUTING.md, dev scripts) + '''local-dev-token''', ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..af47c83e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# Contributing to karaoke-gen + +Thanks for your interest in contributing! This guide focuses on getting a +working **local development environment with no Google Cloud credentials or +paid API keys** — everything runs against local emulators. + +## What runs locally (no credentials needed) + +| Component | Local story | +|-----------|-------------| +| FastAPI backend | Runs directly via uvicorn | +| Firestore | Google's Firestore emulator | +| Cloud Storage | [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) (Docker) | +| Background workers (audio, lyrics, screens, video…) | Run as local subprocesses / in-process tasks | +| Audio separation | Local CPU/GPU via `audio-separator` (set `MODEL_DIR`; models download on first use) | +| Lyrics transcription | Local Whisper (`poetry install --extras local-whisper`); AudioShake used only if you set a token | +| Login (magic links) | Printed to the backend console instead of emailed | +| Signed URLs (review audio, downloads, uploads) | Rewritten to plain emulator URLs | +| Frontend (Next.js) | `npm run dev`, pointed at the local backend | + +Things that do NOT run locally: payments (Stripe), production email +(Postmark), YouTube/Drive/Dropbox distribution, and the `flacfetch` +URL-download service — **jobs from URLs won't process locally; use file +upload instead.** + +## Prerequisites + +- Python 3.12/3.13 + [Poetry](https://python-poetry.org/) +- Node.js 20+ (for the frontend) +- Docker (for the GCS emulator) +- Java 21+ and the Google Cloud SDK with the Firestore emulator: + `gcloud components install cloud-firestore-emulator` + (no login/account needed — the emulator runs entirely offline) +- ffmpeg (`brew install ffmpeg` / `apt install ffmpeg`) + +## Backend quickstart + +```bash +poetry install # heavy: torch etc, ~10 min first time +./scripts/run-backend-local.sh --with-emulators +``` + +This starts the Firestore emulator (port 8080), fake-gcs-server (port 4443), +creates the bucket, seeds a default theme (`scripts/seed-local-data.py`), and +runs the backend at (API docs at `/docs`). + +Optional API keys go in `.env` (see `.env.example`) — none are required to boot. + +### Try it + +```bash +# Health +curl http://localhost:8000/api/health + +# Create a job from a local audio file (admin token is "local-dev-token") +curl -X POST http://localhost:8000/api/jobs/upload \ + -H 'Authorization: Bearer local-dev-token' \ + -F "file=@/path/to/song.flac" -F "artist=Some Artist" -F "title=Some Song" + +# Watch it process; when status hits awaiting_review, open the review UI +curl http://localhost:8000/api/jobs/ -H 'Authorization: Bearer local-dev-token' +``` + +For audio separation to run, set `MODEL_DIR` (e.g. `MODEL_DIR=./models`) — +separation models (~2 GB) are downloaded on first use. Without it the job +fails at the separation step with a clear error. + +## Frontend quickstart + +```bash +cd frontend +npm install +NEXT_PUBLIC_API_URL=http://localhost:8000 npx next dev +``` + +Then open . To log in, enter any email address — the +magic link is printed in the **backend** console (look for `EMAIL TO:`); open +that link in your browser. + +### Review UI without running the full pipeline + +If you're working on the lyrics review UI specifically, there's a fixtures +harness that serves real correction data through the review API shape without +processing any audio: + +```bash +python scripts/review_test_fixtures.py # serves on :8765 +``` + +## Tests + +```bash +make test # everything (backend + frontend) +make test-backend # backend only (unit + emulator tests) +``` + +Emulator-backed integration tests live in `backend/tests/emulator/` and use +the same two emulators as the local dev setup. + +## Pull requests + +- Keep PRs focused; include tests for behavior changes. +- Run `make test` before pushing — CI runs the same suite. +- Bump `tool.poetry.version` in `pyproject.toml` for code changes. +- CI must pass before merge; a maintainer will review from there. + +See `docs/DEVELOPMENT.md` for deeper docs (architecture, deployment, +observability) — most of that file concerns the production GCP deployment, +which contributors don't need. diff --git a/README.md b/README.md index 2be5f775..cd383f3a 100644 --- a/README.md +++ b/README.md @@ -675,4 +675,6 @@ MIT ## 🤝 Contributing -Contributions are welcome! Please see our contributing guidelines. +Contributions are welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) — the whole +backend runs locally against emulators with no Google Cloud credentials or +paid API keys required. diff --git a/backend/api/routes/file_upload.py b/backend/api/routes/file_upload.py index be2c8be9..151af240 100644 --- a/backend/api/routes/file_upload.py +++ b/backend/api/routes/file_upload.py @@ -814,8 +814,10 @@ async def upload_and_create_job( update_data['style_assets'] = style_assets if 'style_params' in style_assets: update_data['style_params_gcs_path'] = style_assets['style_params'] - elif theme_style_assets: - # Theme style assets (no custom uploads) + elif theme_style_assets or theme_style_params_path: + # Theme style assets (no custom uploads). A theme may have style + # params but no asset files (e.g. the minimal local-dev seed theme), + # so the params path must be written even when the assets dict is empty. update_data['style_assets'] = theme_style_assets if theme_style_params_path: update_data['style_params_gcs_path'] = theme_style_params_path diff --git a/backend/api/routes/users.py b/backend/api/routes/users.py index f5e89171..16d0275d 100644 --- a/backend/api/routes/users.py +++ b/backend/api/routes/users.py @@ -36,6 +36,7 @@ def _mask_email(email: str) -> str: from fastapi import APIRouter, HTTPException, Depends, Request, Header from pydantic import BaseModel, EmailStr +from backend.config import is_production from backend.i18n import t, get_locale_from_request, get_full_locale_from_request from backend.services.email_validation_service import get_email_validation_service from backend.models.user import ( @@ -274,8 +275,10 @@ async def send_magic_link( message=t(locale, "users.magicLinkSent") ) - # Check if email service is configured - if not email_service.is_configured(): + # Check if email service is configured. In development the console provider + # logs the magic link to the backend console instead of emailing it, so + # contributors can log in locally without a Postmark token. + if not email_service.is_configured() and is_production(): logger.error("Email service not configured - cannot send magic links") raise HTTPException( status_code=503, diff --git a/backend/config.py b/backend/config.py index bb8b4171..b946237e 100644 --- a/backend/config.py +++ b/backend/config.py @@ -4,9 +4,14 @@ Configuration management for the karaoke generation backend. import os import logging from typing import Optional, Dict +from dotenv import load_dotenv from pydantic_settings import BaseSettings from google.cloud import secretmanager +# Load .env for local development (no-op when absent, e.g. in Cloud Run). +# Must run before the Settings class body below, which reads os.getenv at +# class-definition time. Real environment variables take precedence. +load_dotenv() logger = logging.getLogger(__name__) diff --git a/backend/services/storage_service.py b/backend/services/storage_service.py index 08c42689..37e81077 100644 --- a/backend/services/storage_service.py +++ b/backend/services/storage_service.py @@ -155,7 +155,28 @@ class StorageService: """Internal method to generate signed URLs for GET or PUT operations.""" import google.auth from google.auth.transport import requests - + + # Local development against fake-gcs-server: real URL signing requires + # service-account credentials, which contributors don't have. The emulator + # serves objects unauthenticated, so return a plain emulator URL instead. + emulator_host = os.getenv("STORAGE_EMULATOR_HOST") + if emulator_host: + from urllib.parse import quote + + base = emulator_host.rstrip("/") + if method == "GET": + url = f"{base}/download/storage/v1/b/{self.bucket.name}/o/{quote(blob_path, safe='')}?alt=media" + else: + # fake-gcs-server treats a PUT with X-Goog signature params as a + # signed-URL upload without validating the signature. Requires the + # server to run with -public-host matching this host. + url = ( + f"{base}/{self.bucket.name}/{quote(blob_path)}" + "?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Signature=local-dev" + ) + logger.info(f"Generated emulator {method} URL for {blob_path} (STORAGE_EMULATOR_HOST set)") + return url + try: blob = self.bucket.blob(blob_path) diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 89c2a57a..cd338174 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -24,7 +24,7 @@ import httpx from google.api_core import exceptions as gcp_exceptions from google.protobuf import duration_pb2 -from backend.config import get_settings +from backend.config import get_settings, is_production from backend.services.tracing import inject_trace_context @@ -404,6 +404,10 @@ class WorkerService: in AWAITING_AUDIO_SELECTION. Uses a Cloud Run Job (not BackgroundTasks) so the batch survives API instance scale-down and the user can close the tab. """ + if not self._use_cloud_tasks and not is_production(): + return self._run_worker_module_locally( + "bulk_search_worker", ["--batch-id", batch_id], log_prefix=f"[batch:{batch_id}]" + ) try: from google.cloud import run_v2 @@ -584,6 +588,29 @@ class WorkerService: # Exhausted transient retries — re-raise so the caller logs + returns False. raise last_exc + def _run_worker_module_locally( + self, worker_module: str, args: list[str], log_prefix: str = "" + ) -> bool: + """ + Run a worker module as a detached local subprocess (development mode). + + Mirrors what the corresponding Cloud Run Job would execute + (``python -m backend.workers. ``), so contributors can run + the full pipeline locally against the emulators without GCP credentials. + Worker output is streamed to the backend console. + """ + import subprocess + import sys + + cmd = [sys.executable, "-m", f"backend.workers.{worker_module}", *args] + try: + subprocess.Popen(cmd, env=os.environ.copy()) + logger.info(f"{log_prefix} Started local worker subprocess: {' '.join(cmd)}") + return True + except Exception as e: + logger.error(f"{log_prefix} Failed to start local worker {worker_module}: {e}", exc_info=True) + return False + async def _trigger_worker_cloud_run_job( self, job_id: str, @@ -607,6 +634,15 @@ class WorkerService: Returns: True if job was triggered successfully, False otherwise """ + # Local development (ENABLE_CLOUD_TASKS=false): run the worker module as a + # local subprocess with the exact args the Cloud Run Job would receive. + # The is_production() guard ensures a deployed environment that omits + # ENABLE_CLOUD_TASKS still dispatches to Cloud Run Jobs, never Popen. + if not self._use_cloud_tasks and not is_production(): + return self._run_worker_module_locally( + worker_module, ["--job-id", job_id], log_prefix=f"[job:{job_id}]" + ) + try: from google.cloud import run_v2 diff --git a/backend/tests/test_services.py b/backend/tests/test_services.py index a49bc47d..7320e0f8 100644 --- a/backend/tests/test_services.py +++ b/backend/tests/test_services.py @@ -168,7 +168,7 @@ class TestWorkerService: with patch('backend.services.worker_service.get_settings') as mock_settings: mock_settings.return_value.admin_tokens = "test-token" mock_settings.return_value.google_cloud_project = "test-project" - mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.enable_cloud_tasks = True mock_settings.return_value.gcp_region = "us-central1" mock_settings.return_value.use_cloud_run_jobs_for_video = False @@ -211,7 +211,7 @@ class TestWorkerService: with patch('backend.services.worker_service.get_settings') as mock_settings: mock_settings.return_value.admin_tokens = "test-token" mock_settings.return_value.google_cloud_project = "test-project" - mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.enable_cloud_tasks = True mock_settings.return_value.gcp_region = "us-central1" mock_settings.return_value.use_cloud_run_jobs_for_video = False @@ -245,6 +245,78 @@ class TestWorkerService: assert call_args is not None assert "lyrics-transcription-job" in str(call_args) + @pytest.mark.asyncio + async def test_trigger_audio_worker_local_subprocess_in_dev_mode(self): + """With ENABLE_CLOUD_TASKS=false, CRJ-based workers run as local subprocesses.""" + from backend.services.worker_service import WorkerService, reset_worker_service + reset_worker_service() + + with patch('backend.services.worker_service.get_settings') as mock_settings: + mock_settings.return_value.admin_tokens = "test-token" + mock_settings.return_value.google_cloud_project = "test-project" + mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.gcp_region = "us-central1" + mock_settings.return_value.use_cloud_run_jobs_for_video = False + + with patch('subprocess.Popen') as mock_popen: + service = WorkerService() + result = await service.trigger_audio_worker("test123") + + assert result is True + mock_popen.assert_called_once() + cmd = mock_popen.call_args[0][0] + assert cmd[1:] == ["-m", "backend.workers.audio_worker", "--job-id", "test123"] + + @pytest.mark.asyncio + async def test_no_local_subprocess_in_production_even_without_cloud_tasks(self): + """A prod deploy that omits ENABLE_CLOUD_TASKS must still use Cloud Run Jobs.""" + from backend.services.worker_service import WorkerService, reset_worker_service + reset_worker_service() + + with patch('backend.services.worker_service.get_settings') as mock_settings, \ + patch('backend.services.worker_service.is_production', return_value=True): + mock_settings.return_value.admin_tokens = "test-token" + mock_settings.return_value.google_cloud_project = "test-project" + mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.gcp_region = "us-central1" + mock_settings.return_value.use_cloud_run_jobs_for_video = False + + mock_operation = MagicMock() + mock_operation.metadata = "test-metadata" + mock_run_v2, mock_jobs_client = self._mock_run_v2([mock_operation]) + + import google.cloud + with patch('subprocess.Popen') as mock_popen, \ + patch.dict('sys.modules', {'google.cloud.run_v2': mock_run_v2}), \ + patch.object(google.cloud, 'run_v2', mock_run_v2, create=True): + service = WorkerService() + result = await service.trigger_audio_worker("test123") + + assert result is True + mock_popen.assert_not_called() + mock_jobs_client.run_job.assert_called_once() + + @pytest.mark.asyncio + async def test_trigger_bulk_search_worker_local_subprocess_in_dev_mode(self): + """Bulk search worker also falls back to a local subprocess in dev mode.""" + from backend.services.worker_service import WorkerService, reset_worker_service + reset_worker_service() + + with patch('backend.services.worker_service.get_settings') as mock_settings: + mock_settings.return_value.admin_tokens = "test-token" + mock_settings.return_value.google_cloud_project = "test-project" + mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.gcp_region = "us-central1" + mock_settings.return_value.use_cloud_run_jobs_for_video = False + + with patch('subprocess.Popen') as mock_popen: + service = WorkerService() + result = await service.trigger_bulk_search_worker("batch-1") + + assert result is True + cmd = mock_popen.call_args[0][0] + assert cmd[1:] == ["-m", "backend.workers.bulk_search_worker", "--batch-id", "batch-1"] + def _mock_run_v2(self, run_job_side_effect): """Build a mocked run_v2 module whose JobsClient.run_job uses side_effect.""" mock_jobs_client = MagicMock() @@ -269,7 +341,7 @@ class TestWorkerService: patch('backend.services.worker_service.asyncio.sleep', new=AsyncMock()) as mock_sleep: mock_settings.return_value.admin_tokens = "test-token" mock_settings.return_value.google_cloud_project = "test-project" - mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.enable_cloud_tasks = True mock_settings.return_value.gcp_region = "us-central1" mock_settings.return_value.use_cloud_run_jobs_for_video = False @@ -304,7 +376,7 @@ class TestWorkerService: patch('backend.services.worker_service.asyncio.sleep', new=AsyncMock()): mock_settings.return_value.admin_tokens = "test-token" mock_settings.return_value.google_cloud_project = "test-project" - mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.enable_cloud_tasks = True mock_settings.return_value.gcp_region = "us-central1" mock_settings.return_value.use_cloud_run_jobs_for_video = False @@ -332,7 +404,7 @@ class TestWorkerService: patch('backend.services.worker_service.asyncio.sleep', new=AsyncMock()) as mock_sleep: mock_settings.return_value.admin_tokens = "test-token" mock_settings.return_value.google_cloud_project = "test-project" - mock_settings.return_value.enable_cloud_tasks = False + mock_settings.return_value.enable_cloud_tasks = True mock_settings.return_value.gcp_region = "us-central1" mock_settings.return_value.use_cloud_run_jobs_for_video = False diff --git a/backend/tests/test_storage_service.py b/backend/tests/test_storage_service.py index 73e4da4c..27c2ebce 100644 --- a/backend/tests/test_storage_service.py +++ b/backend/tests/test_storage_service.py @@ -448,7 +448,14 @@ class TestStorageServiceFileOperations: class TestStorageServiceSignedUrls: """Test signed URL generation.""" - + + @pytest.fixture(autouse=True) + def _no_emulator_env(self, monkeypatch): + # Other test modules (e.g. test_emulator_integration) export + # STORAGE_EMULATOR_HOST at import time, which would divert these tests + # into the emulator URL fallback instead of real signing. + monkeypatch.delenv("STORAGE_EMULATOR_HOST", raising=False) + @patch("backend.services.storage_service.storage.Client") @patch("backend.services.storage_service.settings") def test_generate_signed_url(self, mock_settings, mock_client_class): @@ -541,3 +548,42 @@ class TestStorageServiceSignedUrls: assert call_kwargs["service_account_email"] == "sa@project.iam.gserviceaccount.com" assert call_kwargs["access_token"] == "access-token-123" + + +class TestStorageServiceEmulatorUrls: + """Test emulator URL fallback when STORAGE_EMULATOR_HOST is set (local dev).""" + + @patch("backend.services.storage_service.storage.Client") + @patch("backend.services.storage_service.settings") + def test_get_url_uses_emulator_download_path(self, mock_settings, mock_client_class, monkeypatch): + mock_settings.google_cloud_project = "test-project" + mock_settings.gcs_bucket_name = "test-bucket" + mock_bucket = Mock() + mock_bucket.name = "test-bucket" + mock_client_class.return_value.bucket.return_value = mock_bucket + monkeypatch.setenv("STORAGE_EMULATOR_HOST", "http://127.0.0.1:4443") + + from backend.services.storage_service import StorageService + url = StorageService().generate_signed_url("jobs/abc/review-audio/song.ogg") + + assert url == ( + "http://127.0.0.1:4443/download/storage/v1/b/test-bucket/o/" + "jobs%2Fabc%2Freview-audio%2Fsong.ogg?alt=media" + ) + + @patch("backend.services.storage_service.storage.Client") + @patch("backend.services.storage_service.settings") + def test_put_url_uses_emulator_signed_style_path(self, mock_settings, mock_client_class, monkeypatch): + mock_settings.google_cloud_project = "test-project" + mock_settings.gcs_bucket_name = "test-bucket" + mock_bucket = Mock() + mock_bucket.name = "test-bucket" + mock_client_class.return_value.bucket.return_value = mock_bucket + monkeypatch.setenv("STORAGE_EMULATOR_HOST", "http://127.0.0.1:4443") + + from backend.services.storage_service import StorageService + url = StorageService().generate_signed_upload_url("uploads/x/file.bin") + + # fake-gcs-server accepts PUTs with (unvalidated) X-Goog signature params + assert url.startswith("http://127.0.0.1:4443/test-bucket/uploads/x/file.bin?") + assert "X-Goog-Signature=" in url diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 679160a1..d992691d 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -1,5 +1,10 @@ # Development Guide +> **External contributor?** See [CONTRIBUTING.md](../CONTRIBUTING.md) at the repo +> root — the backend runs fully locally against emulators with **no GCP +> credentials or paid API keys**. This file covers the full setup including +> production/deployment concerns. + ## Quick Start ### Prerequisites @@ -24,9 +29,9 @@ source venv/bin/activate # Install dependencies poetry install -# Copy environment template +# Optional: copy environment template for API keys / overrides +# (nothing in it is required to run locally with emulators) cp .env.example .env -# Edit .env with required API keys ``` ### Frontend Setup @@ -43,13 +48,20 @@ cp .env.example .env.local ### Backend with Emulators ```bash -# Start emulators and backend +# Start emulators, create the bucket, seed a default theme, run the backend ./scripts/run-backend-local.sh --with-emulators # Backend runs at http://localhost:8000 # API docs at http://localhost:8000/docs ``` +In emulator mode no GCP credentials are needed: workers run as local +subprocesses, signed URLs are rewritten to emulator URLs, and magic-link +login emails are printed to the console. Jobs must be created via **file +upload** (URL jobs require the flacfetch cloud service). Set `MODEL_DIR` to +enable local audio separation. See [CONTRIBUTING.md](../CONTRIBUTING.md) for +the full walkthrough. + ### Frontend ```bash diff --git a/pyproject.toml b/pyproject.toml index 20decbc5..3731354b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.200.1" +version = "0.201.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/run-backend-local.sh b/scripts/run-backend-local.sh index 19a68bc3..fdec1df7 100755 --- a/scripts/run-backend-local.sh +++ b/scripts/run-backend-local.sh @@ -21,19 +21,22 @@ export PORT="${PORT:-8000}" if [[ "$1" == "--with-emulators" ]]; then echo "📦 Starting GCP emulators..." "$SCRIPT_DIR/start-emulators.sh" - + export FIRESTORE_EMULATOR_HOST="127.0.0.1:8080" export STORAGE_EMULATOR_HOST="http://127.0.0.1:4443" + export GOOGLE_CLOUD_PROJECT="test-project" export GCS_BUCKET_NAME="test-bucket" export FIRESTORE_COLLECTION="test-jobs" - - # Create test bucket - curl -s -X POST "http://127.0.0.1:4443/storage/v1/b" \ - -H "Content-Type: application/json" \ - -d '{"name":"test-bucket"}' \ - --data-urlencode "project=test-project" || true - + export FRONTEND_URL="${FRONTEND_URL:-http://localhost:3000}" + + # Create the bucket and seed a default theme (idempotent) so jobs can be created + poetry run python "$SCRIPT_DIR/seed-local-data.py" + echo "✅ Emulators running" +else + echo "⚠️ Running WITHOUT emulators: this targets the real GCP project" + echo " ($GOOGLE_CLOUD_PROJECT) and requires GCP credentials." + echo " External contributors should use: $0 --with-emulators" fi # Change to project root diff --git a/scripts/seed-local-data.py b/scripts/seed-local-data.py new file mode 100644 index 00000000..d00e5d2a --- /dev/null +++ b/scripts/seed-local-data.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Seed the local GCS emulator (fake-gcs-server) with the minimum data the +backend needs to create jobs: the bucket itself plus a default theme. + +Run after starting the emulators (scripts/start-emulators.sh); it is invoked +automatically by scripts/run-backend-local.sh --with-emulators. Idempotent — +safe to run repeatedly. + +Usage: + STORAGE_EMULATOR_HOST=http://127.0.0.1:4443 python scripts/seed-local-data.py + +Environment: + STORAGE_EMULATOR_HOST Required — refuses to run against real GCS. + GCS_BUCKET_NAME Bucket to create/seed (default: test-bucket) + GOOGLE_CLOUD_PROJECT Project for the emulator client (default: test-project) +""" + +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +DEFAULT_THEME_ID = "local-default" + + +def main() -> int: + emulator_host = os.getenv("STORAGE_EMULATOR_HOST") + if not emulator_host: + print("ERROR: STORAGE_EMULATOR_HOST is not set. This script only seeds the") + print("local GCS emulator — it refuses to touch real GCS buckets.") + return 1 + + project = os.getenv("GOOGLE_CLOUD_PROJECT", "test-project") + bucket_name = os.getenv("GCS_BUCKET_NAME", "test-bucket") + + from google.cloud import storage + + client = storage.Client(project=project) + + bucket = client.bucket(bucket_name) + if not bucket.exists(): + client.create_bucket(bucket_name) + print(f"Created bucket {bucket_name}") + else: + print(f"Bucket {bucket_name} already exists") + + # Theme registry with one default theme (jobs can't be created without one) + registry = { + "version": 1, + "themes": [ + { + "id": DEFAULT_THEME_ID, + "name": "Local Default", + "description": "Default theme seeded for local development", + "is_default": True, + "created_at": "2026-01-01T00:00:00Z", + } + ], + } + bucket.blob("themes/_metadata.json").upload_from_string( + json.dumps(registry, indent=2), content_type="application/json" + ) + print("Seeded themes/_metadata.json") + + from karaoke_gen.style_loader import get_default_style_params + + style_params = get_default_style_params() + bucket.blob(f"themes/{DEFAULT_THEME_ID}/style_params.json").upload_from_string( + json.dumps(style_params, indent=2), content_type="application/json" + ) + print(f"Seeded themes/{DEFAULT_THEME_ID}/style_params.json") + + print("\nLocal seed data ready — the backend can now create jobs.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/start-emulators.sh b/scripts/start-emulators.sh index 26f7cd32..82706022 100755 --- a/scripts/start-emulators.sh +++ b/scripts/start-emulators.sh @@ -43,14 +43,16 @@ else # Stop any existing container docker rm -f test-gcs-emulator >/dev/null 2>&1 || true - # Start fake-gcs-server + # Start fake-gcs-server. -public-host makes it accept signed-URL-style + # PUT uploads on this host (see StorageService emulator URL fallback). docker run -d \ --name test-gcs-emulator \ -p 4443:4443 \ fsouza/fake-gcs-server:latest \ -scheme http \ -port 4443 \ - -external-url http://localhost:4443 \ + -external-url http://127.0.0.1:4443 \ + -public-host 127.0.0.1:4443 \ > "$EMULATOR_LOG_DIR/gcs.log" 2>&1 GCS_CONTAINER=$(docker ps -q -f name=test-gcs-emulator)