Files
karaoke-gen/backend/services/storage_service.py
T
Andrew Beveridge ☄️andClaude Fable 5 70ea964ca9 feat(dev): full local backend for external contributors (no GCP creds needed) (#946)
## Summary

Motivated by external contributor Martín (netux) asking "is there an
easy way to run the full backend without GCS?" — he couldn't test PR
#895's backend changes locally. This PR makes the whole backend runnable
locally against emulators with **zero GCP credentials and zero paid API
keys**.

### What was broken for contributors
- Workers migrated to Cloud Run Jobs for prod robustness
(audio-download, audio-separation, lyrics-transcription, bulk-search)
silently bypassed the direct-HTTP dev mode → local jobs stalled at
PENDING forever
- Signed URL generation called `google.auth.default()` → crashed with no
credentials (review audio, downloads, theme previews, upload URLs)
- Magic-link login 503'd locally (console email provider counted as "not
configured")
- Fresh emulator had no default theme → every job create returned 422
- `docs/DEVELOPMENT.md` told people to `cp .env.example .env` but the
file didn't exist — and nothing loaded `.env` anyway
- `run-backend-local.sh` defaulted to the production GCP project

### Changes
- **worker_service**: CRJ-based workers run as local subprocesses
(`python -m backend.workers.X --job-id Y`, exact CRJ parity) when
`ENABLE_CLOUD_TASKS=false` **and** not `is_production()`
(regression-tested both ways)
- **storage_service**: signed URLs → plain fake-gcs-server URLs when
`STORAGE_EMULATOR_HOST` set (GET via JSON API download path; PUT via
unvalidated-signature path, enabled by `-public-host` on the emulator)
- **users route**: magic-link 503 only in production; dev prints the
link to the console
- **file_upload bugfix**: a theme with style params but no asset files
never got `style_params_gcs_path` written (empty-dict falsiness in `elif
theme_style_assets:`) — surfaced by the minimal seed theme, latent in
prod
- **scripts/seed-local-data.py** (new): creates bucket + default theme;
wired into `run-backend-local.sh --with-emulators`, which now uses
`test-project`
- **config**: `load_dotenv()` so `.env` actually works; new root
`.env.example`
- **CONTRIBUTING.md** (new): no-credentials quickstart; README +
DEVELOPMENT.md link to it

### Verified end-to-end locally (no GCP credentials)
Upload job → local audio separation (real models) → transcription →
screens → `awaiting_review`; review endpoints all working including
`/api/review/{job}/audio/vocals` (the PR #895 endpoint) and 302 →
emulator audio URLs. Magic-link login round-trip via console link.

### Tests
- 224 tests across affected files pass; full backend unit suite: 3969
passed
- `backend/tests/test_emulator_integration.py` has 8 pre-existing
failures that reproduce identically on unmodified `main` locally (broken
theme-service mock in that file) — unrelated to this PR
- New tests: dev-mode subprocess dispatch (audio + bulk-search),
prod-mode never-Popen guard, emulator URL generation (GET + PUT)

@coderabbitai ignore

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-27 06:48:00 +00:00

318 lines
13 KiB
Python

"""
Google Cloud Storage operations for file management.
"""
import logging
import os
import json
from typing import Optional, BinaryIO, Any, Dict
from pathlib import Path
from google.cloud import storage
from google.cloud.exceptions import NotFound
from datetime import timedelta
from backend.config import settings
logger = logging.getLogger(__name__)
class StorageService:
"""Service for Google Cloud Storage operations."""
def __init__(self):
"""Initialize GCS client."""
self.client = storage.Client(project=settings.google_cloud_project)
self.bucket = self.client.bucket(settings.gcs_bucket_name)
def upload_file(self, local_path: str, destination_path: str) -> str:
"""Upload a file to GCS."""
try:
blob = self.bucket.blob(destination_path)
blob.upload_from_filename(local_path)
logger.info(f"Uploaded {local_path} to gs://{settings.gcs_bucket_name}/{destination_path}")
return destination_path
except Exception as e:
logger.error(f"Error uploading file {local_path}: {e}")
raise
def upload_fileobj(self, file_obj: BinaryIO, destination_path: str, content_type: Optional[str] = None) -> str:
"""Upload a file object to GCS."""
try:
blob = self.bucket.blob(destination_path)
if content_type:
blob.content_type = content_type
blob.upload_from_file(file_obj, rewind=True)
logger.info(f"Uploaded file object to gs://{settings.gcs_bucket_name}/{destination_path}")
return destination_path
except Exception as e:
logger.error(f"Error uploading file object: {e}")
raise
def download_file(self, source_path: str, destination_path: str) -> str:
"""Download a file from GCS."""
try:
blob = self.bucket.blob(source_path)
blob.download_to_filename(destination_path)
logger.info(f"Downloaded gs://{settings.gcs_bucket_name}/{source_path} to {destination_path}")
return destination_path
except NotFound:
# A missing object is a routine "not found", not a server-side failure.
# Log at warning so it doesn't trip the error-monitor, and let the caller
# decide how to surface it (e.g. the download route returns HTTP 404).
logger.warning(f"Object not found downloading {source_path}")
raise
except Exception as e:
logger.error(f"Error downloading file {source_path}: {e}")
raise
def download_bytes(self, source_path: str) -> bytes:
"""Download an object from GCS fully into memory. Only use for small
objects (e.g. transcoded review audio); large files should stream."""
try:
blob = self.bucket.blob(source_path)
return blob.download_as_bytes()
except NotFound:
logger.warning(f"Object not found downloading {source_path}")
raise
except Exception as e:
logger.error(f"Error downloading bytes {source_path}: {e}")
raise
def copy_blob(self, source_path: str, destination_path: str) -> str:
"""Server-side copy of a blob within the bucket (no download/upload)."""
try:
src = self.bucket.blob(source_path)
self.bucket.copy_blob(src, self.bucket, destination_path)
logger.info(f"Copied gs://{settings.gcs_bucket_name}/{source_path} -> {destination_path}")
return destination_path
except Exception as e:
logger.error(f"Error copying blob {source_path} -> {destination_path}: {e}")
raise
def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str:
"""Generate a signed URL for downloading a file.
In Cloud Run, this uses the IAM signBlob API since we don't have
a private key available. Requires the service account to have
roles/iam.serviceAccountTokenCreator on itself.
"""
return self._generate_signed_url_internal(blob_path, "GET", expiration_minutes)
def generate_signed_upload_url(self, blob_path: str, content_type: str = "application/octet-stream", expiration_minutes: int = 60) -> str:
"""Generate a signed URL for uploading a file directly to GCS.
This allows clients to upload files directly to GCS without going through
the backend, bypassing any request body size limits.
Args:
blob_path: The destination path in GCS
content_type: The expected content type of the upload
expiration_minutes: How long the URL is valid for
Returns:
A signed URL that accepts PUT requests with the file content
"""
return self._generate_signed_url_internal(blob_path, "PUT", expiration_minutes, content_type)
def create_resumable_upload_session(
self,
blob_path: str,
content_type: str = "application/octet-stream",
origin: Optional[str] = None,
) -> str:
"""Create a GCS resumable upload session and return its session URI.
Unlike a signed PUT URL (single-shot: any interruption restarts the whole
file), a resumable session accepts chunked uploads with Content-Range and
supports querying the persisted offset after a failure, so clients on
flaky connections can resume mid-file with no wasted bandwidth. Session
URIs are unauthenticated capability tokens valid for ~1 week.
Args:
blob_path: The destination path in GCS.
content_type: Content type fixed at session creation.
origin: Browser Origin that will upload. When set, GCS answers CORS
preflights on the session URI for that origin itself — no bucket
CORS config is needed for the chunk PUTs.
Returns:
The resumable session URI the client PUTs chunks to.
"""
try:
blob = self.bucket.blob(blob_path)
session_uri = blob.create_resumable_upload_session(
content_type=content_type,
origin=origin,
)
logger.info(f"Created resumable upload session for {blob_path} (origin={origin})")
return session_uri
except Exception as e:
logger.error(f"Error creating resumable upload session for {blob_path}: {e}")
raise
def _generate_signed_url_internal(self, blob_path: str, method: str, expiration_minutes: int = 60, content_type: Optional[str] = None) -> str:
"""Internal method to generate signed URLs for GET or PUT operations."""
import google.auth
from google.auth.transport import requests
# Local development against fake-gcs-server: real URL signing requires
# service-account credentials, which contributors don't have. The emulator
# serves objects unauthenticated, so return a plain emulator URL instead.
emulator_host = os.getenv("STORAGE_EMULATOR_HOST")
if emulator_host:
from urllib.parse import quote
base = emulator_host.rstrip("/")
if method == "GET":
url = f"{base}/download/storage/v1/b/{self.bucket.name}/o/{quote(blob_path, safe='')}?alt=media"
else:
# fake-gcs-server treats a PUT with X-Goog signature params as a
# signed-URL upload without validating the signature. Requires the
# server to run with -public-host matching this host.
url = (
f"{base}/{self.bucket.name}/{quote(blob_path)}"
"?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Signature=local-dev"
)
logger.info(f"Generated emulator {method} URL for {blob_path} (STORAGE_EMULATOR_HOST set)")
return url
try:
blob = self.bucket.blob(blob_path)
# Get default credentials and refresh to ensure we have a valid token
credentials, project = google.auth.default()
# Common kwargs for signed URL generation
kwargs = {
"version": "v4",
"expiration": timedelta(minutes=expiration_minutes),
"method": method,
}
# For PUT requests, we need to specify the content type in headers
if method == "PUT" and content_type:
kwargs["headers"] = {"Content-Type": content_type}
# Check if we're using compute credentials (Cloud Run/GCE)
# These need to use IAM signBlob via service_account_email + access_token
if hasattr(credentials, 'service_account_email'):
# Refresh credentials to get a valid access token
auth_request = requests.Request()
credentials.refresh(auth_request)
kwargs["service_account_email"] = credentials.service_account_email
kwargs["access_token"] = credentials.token
url = blob.generate_signed_url(**kwargs)
logger.info(f"Generated signed {method} URL for {blob_path}")
return url
except Exception as e:
logger.error(f"Error generating signed {method} URL for {blob_path}: {e}")
raise
def delete_file(self, blob_path: str, ignore_missing: bool = False) -> bool:
"""Delete a file from GCS.
Args:
blob_path: Path of the object to delete.
ignore_missing: If True, a missing object (404 NotFound) is treated as a
successful no-op and logged at debug level instead of error. Use for
best-effort deletes over a template of possible paths (e.g. job
restart/regen) where many files legitimately never existed.
Returns:
True if an object was actually deleted, False if it was already absent
(only possible when ignore_missing=True; otherwise a missing object raises).
"""
try:
blob = self.bucket.blob(blob_path)
blob.delete()
logger.info(f"Deleted gs://{settings.gcs_bucket_name}/{blob_path}")
return True
except NotFound:
if ignore_missing:
logger.debug(f"File already absent, skipping delete: {blob_path}")
return False
logger.error(f"Error deleting file {blob_path}: 404 object not found")
raise
except Exception as e:
logger.error(f"Error deleting file {blob_path}: {e}")
raise
def delete_folder(self, prefix: str) -> int:
"""
Delete all files in GCS with a given prefix (folder).
Args:
prefix: The folder prefix to delete (e.g., "uploads/abc123/")
Returns:
Number of files deleted
"""
try:
blobs = list(self.bucket.list_blobs(prefix=prefix))
deleted_count = 0
for blob in blobs:
try:
blob.delete()
deleted_count += 1
except Exception as e:
logger.warning(f"Error deleting blob {blob.name}: {e}")
if deleted_count > 0:
logger.info(f"Deleted {deleted_count} files from gs://{settings.gcs_bucket_name}/{prefix}")
return deleted_count
except Exception as e:
logger.error(f"Error deleting folder {prefix}: {e}")
return 0 # Don't raise - folder deletion shouldn't break operations
def list_files(self, prefix: str) -> list:
"""List files in GCS with a given prefix."""
try:
blobs = self.bucket.list_blobs(prefix=prefix)
return [blob.name for blob in blobs]
except Exception as e:
logger.error(f"Error listing files with prefix {prefix}: {e}")
raise
def file_exists(self, blob_path: str) -> bool:
"""Check if a file exists in GCS."""
try:
blob = self.bucket.blob(blob_path)
return blob.exists()
except Exception as e:
logger.error(f"Error checking file existence {blob_path}: {e}")
raise
def upload_json(self, destination_path: str, data: Dict[str, Any]) -> str:
"""Upload a JSON object to GCS."""
try:
blob = self.bucket.blob(destination_path)
blob.content_type = "application/json"
blob.upload_from_string(
json.dumps(data, indent=2, ensure_ascii=False),
content_type="application/json"
)
logger.info(f"Uploaded JSON to gs://{settings.gcs_bucket_name}/{destination_path}")
return destination_path
except Exception as e:
logger.error(f"Error uploading JSON to {destination_path}: {e}")
raise
def download_json(self, source_path: str) -> Dict[str, Any]:
"""Download and parse a JSON file from GCS."""
try:
blob = self.bucket.blob(source_path)
content = blob.download_as_text()
data = json.loads(content)
logger.info(f"Downloaded JSON from gs://{settings.gcs_bucket_name}/{source_path}")
return data
except Exception as e:
logger.error(f"Error downloading JSON from {source_path}: {e}")
raise