mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Motivated by external contributor Martín (netux) asking "is there an easy way to run the full backend without GCS?" — he couldn't test PR #895's backend changes locally. This PR makes the whole backend runnable locally against emulators with **zero GCP credentials and zero paid API keys**. ### What was broken for contributors - Workers migrated to Cloud Run Jobs for prod robustness (audio-download, audio-separation, lyrics-transcription, bulk-search) silently bypassed the direct-HTTP dev mode → local jobs stalled at PENDING forever - Signed URL generation called `google.auth.default()` → crashed with no credentials (review audio, downloads, theme previews, upload URLs) - Magic-link login 503'd locally (console email provider counted as "not configured") - Fresh emulator had no default theme → every job create returned 422 - `docs/DEVELOPMENT.md` told people to `cp .env.example .env` but the file didn't exist — and nothing loaded `.env` anyway - `run-backend-local.sh` defaulted to the production GCP project ### Changes - **worker_service**: CRJ-based workers run as local subprocesses (`python -m backend.workers.X --job-id Y`, exact CRJ parity) when `ENABLE_CLOUD_TASKS=false` **and** not `is_production()` (regression-tested both ways) - **storage_service**: signed URLs → plain fake-gcs-server URLs when `STORAGE_EMULATOR_HOST` set (GET via JSON API download path; PUT via unvalidated-signature path, enabled by `-public-host` on the emulator) - **users route**: magic-link 503 only in production; dev prints the link to the console - **file_upload bugfix**: a theme with style params but no asset files never got `style_params_gcs_path` written (empty-dict falsiness in `elif theme_style_assets:`) — surfaced by the minimal seed theme, latent in prod - **scripts/seed-local-data.py** (new): creates bucket + default theme; wired into `run-backend-local.sh --with-emulators`, which now uses `test-project` - **config**: `load_dotenv()` so `.env` actually works; new root `.env.example` - **CONTRIBUTING.md** (new): no-credentials quickstart; README + DEVELOPMENT.md link to it ### Verified end-to-end locally (no GCP credentials) Upload job → local audio separation (real models) → transcription → screens → `awaiting_review`; review endpoints all working including `/api/review/{job}/audio/vocals` (the PR #895 endpoint) and 302 → emulator audio URLs. Magic-link login round-trip via console link. ### Tests - 224 tests across affected files pass; full backend unit suite: 3969 passed - `backend/tests/test_emulator_integration.py` has 8 pre-existing failures that reproduce identically on unmodified `main` locally (broken theme-service mock in that file) — unrelated to this PR - New tests: dev-mode subprocess dispatch (audio + bulk-search), prod-mode never-Popen guard, emulator URL generation (GET + PUT) @coderabbitai ignore --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
318 lines
13 KiB
Python
318 lines
13 KiB
Python
"""
|
|
Google Cloud Storage operations for file management.
|
|
"""
|
|
import logging
|
|
import os
|
|
import json
|
|
from typing import Optional, BinaryIO, Any, Dict
|
|
from pathlib import Path
|
|
from google.cloud import storage
|
|
from google.cloud.exceptions import NotFound
|
|
from datetime import timedelta
|
|
|
|
from backend.config import settings
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class StorageService:
|
|
"""Service for Google Cloud Storage operations."""
|
|
|
|
def __init__(self):
|
|
"""Initialize GCS client."""
|
|
self.client = storage.Client(project=settings.google_cloud_project)
|
|
self.bucket = self.client.bucket(settings.gcs_bucket_name)
|
|
|
|
def upload_file(self, local_path: str, destination_path: str) -> str:
|
|
"""Upload a file to GCS."""
|
|
try:
|
|
blob = self.bucket.blob(destination_path)
|
|
blob.upload_from_filename(local_path)
|
|
logger.info(f"Uploaded {local_path} to gs://{settings.gcs_bucket_name}/{destination_path}")
|
|
return destination_path
|
|
except Exception as e:
|
|
logger.error(f"Error uploading file {local_path}: {e}")
|
|
raise
|
|
|
|
def upload_fileobj(self, file_obj: BinaryIO, destination_path: str, content_type: Optional[str] = None) -> str:
|
|
"""Upload a file object to GCS."""
|
|
try:
|
|
blob = self.bucket.blob(destination_path)
|
|
if content_type:
|
|
blob.content_type = content_type
|
|
blob.upload_from_file(file_obj, rewind=True)
|
|
logger.info(f"Uploaded file object to gs://{settings.gcs_bucket_name}/{destination_path}")
|
|
return destination_path
|
|
except Exception as e:
|
|
logger.error(f"Error uploading file object: {e}")
|
|
raise
|
|
|
|
def download_file(self, source_path: str, destination_path: str) -> str:
|
|
"""Download a file from GCS."""
|
|
try:
|
|
blob = self.bucket.blob(source_path)
|
|
blob.download_to_filename(destination_path)
|
|
logger.info(f"Downloaded gs://{settings.gcs_bucket_name}/{source_path} to {destination_path}")
|
|
return destination_path
|
|
except NotFound:
|
|
# A missing object is a routine "not found", not a server-side failure.
|
|
# Log at warning so it doesn't trip the error-monitor, and let the caller
|
|
# decide how to surface it (e.g. the download route returns HTTP 404).
|
|
logger.warning(f"Object not found downloading {source_path}")
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error downloading file {source_path}: {e}")
|
|
raise
|
|
|
|
def download_bytes(self, source_path: str) -> bytes:
|
|
"""Download an object from GCS fully into memory. Only use for small
|
|
objects (e.g. transcoded review audio); large files should stream."""
|
|
try:
|
|
blob = self.bucket.blob(source_path)
|
|
return blob.download_as_bytes()
|
|
except NotFound:
|
|
logger.warning(f"Object not found downloading {source_path}")
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error downloading bytes {source_path}: {e}")
|
|
raise
|
|
|
|
def copy_blob(self, source_path: str, destination_path: str) -> str:
|
|
"""Server-side copy of a blob within the bucket (no download/upload)."""
|
|
try:
|
|
src = self.bucket.blob(source_path)
|
|
self.bucket.copy_blob(src, self.bucket, destination_path)
|
|
logger.info(f"Copied gs://{settings.gcs_bucket_name}/{source_path} -> {destination_path}")
|
|
return destination_path
|
|
except Exception as e:
|
|
logger.error(f"Error copying blob {source_path} -> {destination_path}: {e}")
|
|
raise
|
|
|
|
def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str:
|
|
"""Generate a signed URL for downloading a file.
|
|
|
|
In Cloud Run, this uses the IAM signBlob API since we don't have
|
|
a private key available. Requires the service account to have
|
|
roles/iam.serviceAccountTokenCreator on itself.
|
|
"""
|
|
return self._generate_signed_url_internal(blob_path, "GET", expiration_minutes)
|
|
|
|
def generate_signed_upload_url(self, blob_path: str, content_type: str = "application/octet-stream", expiration_minutes: int = 60) -> str:
|
|
"""Generate a signed URL for uploading a file directly to GCS.
|
|
|
|
This allows clients to upload files directly to GCS without going through
|
|
the backend, bypassing any request body size limits.
|
|
|
|
Args:
|
|
blob_path: The destination path in GCS
|
|
content_type: The expected content type of the upload
|
|
expiration_minutes: How long the URL is valid for
|
|
|
|
Returns:
|
|
A signed URL that accepts PUT requests with the file content
|
|
"""
|
|
return self._generate_signed_url_internal(blob_path, "PUT", expiration_minutes, content_type)
|
|
|
|
def create_resumable_upload_session(
|
|
self,
|
|
blob_path: str,
|
|
content_type: str = "application/octet-stream",
|
|
origin: Optional[str] = None,
|
|
) -> str:
|
|
"""Create a GCS resumable upload session and return its session URI.
|
|
|
|
Unlike a signed PUT URL (single-shot: any interruption restarts the whole
|
|
file), a resumable session accepts chunked uploads with Content-Range and
|
|
supports querying the persisted offset after a failure, so clients on
|
|
flaky connections can resume mid-file with no wasted bandwidth. Session
|
|
URIs are unauthenticated capability tokens valid for ~1 week.
|
|
|
|
Args:
|
|
blob_path: The destination path in GCS.
|
|
content_type: Content type fixed at session creation.
|
|
origin: Browser Origin that will upload. When set, GCS answers CORS
|
|
preflights on the session URI for that origin itself — no bucket
|
|
CORS config is needed for the chunk PUTs.
|
|
|
|
Returns:
|
|
The resumable session URI the client PUTs chunks to.
|
|
"""
|
|
try:
|
|
blob = self.bucket.blob(blob_path)
|
|
session_uri = blob.create_resumable_upload_session(
|
|
content_type=content_type,
|
|
origin=origin,
|
|
)
|
|
logger.info(f"Created resumable upload session for {blob_path} (origin={origin})")
|
|
return session_uri
|
|
except Exception as e:
|
|
logger.error(f"Error creating resumable upload session for {blob_path}: {e}")
|
|
raise
|
|
|
|
|
|
def _generate_signed_url_internal(self, blob_path: str, method: str, expiration_minutes: int = 60, content_type: Optional[str] = None) -> str:
|
|
"""Internal method to generate signed URLs for GET or PUT operations."""
|
|
import google.auth
|
|
from google.auth.transport import requests
|
|
|
|
# Local development against fake-gcs-server: real URL signing requires
|
|
# service-account credentials, which contributors don't have. The emulator
|
|
# serves objects unauthenticated, so return a plain emulator URL instead.
|
|
emulator_host = os.getenv("STORAGE_EMULATOR_HOST")
|
|
if emulator_host:
|
|
from urllib.parse import quote
|
|
|
|
base = emulator_host.rstrip("/")
|
|
if method == "GET":
|
|
url = f"{base}/download/storage/v1/b/{self.bucket.name}/o/{quote(blob_path, safe='')}?alt=media"
|
|
else:
|
|
# fake-gcs-server treats a PUT with X-Goog signature params as a
|
|
# signed-URL upload without validating the signature. Requires the
|
|
# server to run with -public-host matching this host.
|
|
url = (
|
|
f"{base}/{self.bucket.name}/{quote(blob_path)}"
|
|
"?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Signature=local-dev"
|
|
)
|
|
logger.info(f"Generated emulator {method} URL for {blob_path} (STORAGE_EMULATOR_HOST set)")
|
|
return url
|
|
|
|
try:
|
|
blob = self.bucket.blob(blob_path)
|
|
|
|
# Get default credentials and refresh to ensure we have a valid token
|
|
credentials, project = google.auth.default()
|
|
|
|
# Common kwargs for signed URL generation
|
|
kwargs = {
|
|
"version": "v4",
|
|
"expiration": timedelta(minutes=expiration_minutes),
|
|
"method": method,
|
|
}
|
|
|
|
# For PUT requests, we need to specify the content type in headers
|
|
if method == "PUT" and content_type:
|
|
kwargs["headers"] = {"Content-Type": content_type}
|
|
|
|
# Check if we're using compute credentials (Cloud Run/GCE)
|
|
# These need to use IAM signBlob via service_account_email + access_token
|
|
if hasattr(credentials, 'service_account_email'):
|
|
# Refresh credentials to get a valid access token
|
|
auth_request = requests.Request()
|
|
credentials.refresh(auth_request)
|
|
|
|
kwargs["service_account_email"] = credentials.service_account_email
|
|
kwargs["access_token"] = credentials.token
|
|
|
|
url = blob.generate_signed_url(**kwargs)
|
|
|
|
logger.info(f"Generated signed {method} URL for {blob_path}")
|
|
return url
|
|
except Exception as e:
|
|
logger.error(f"Error generating signed {method} URL for {blob_path}: {e}")
|
|
raise
|
|
|
|
def delete_file(self, blob_path: str, ignore_missing: bool = False) -> bool:
|
|
"""Delete a file from GCS.
|
|
|
|
Args:
|
|
blob_path: Path of the object to delete.
|
|
ignore_missing: If True, a missing object (404 NotFound) is treated as a
|
|
successful no-op and logged at debug level instead of error. Use for
|
|
best-effort deletes over a template of possible paths (e.g. job
|
|
restart/regen) where many files legitimately never existed.
|
|
|
|
Returns:
|
|
True if an object was actually deleted, False if it was already absent
|
|
(only possible when ignore_missing=True; otherwise a missing object raises).
|
|
"""
|
|
try:
|
|
blob = self.bucket.blob(blob_path)
|
|
blob.delete()
|
|
logger.info(f"Deleted gs://{settings.gcs_bucket_name}/{blob_path}")
|
|
return True
|
|
except NotFound:
|
|
if ignore_missing:
|
|
logger.debug(f"File already absent, skipping delete: {blob_path}")
|
|
return False
|
|
logger.error(f"Error deleting file {blob_path}: 404 object not found")
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error deleting file {blob_path}: {e}")
|
|
raise
|
|
|
|
def delete_folder(self, prefix: str) -> int:
|
|
"""
|
|
Delete all files in GCS with a given prefix (folder).
|
|
|
|
Args:
|
|
prefix: The folder prefix to delete (e.g., "uploads/abc123/")
|
|
|
|
Returns:
|
|
Number of files deleted
|
|
"""
|
|
try:
|
|
blobs = list(self.bucket.list_blobs(prefix=prefix))
|
|
deleted_count = 0
|
|
|
|
for blob in blobs:
|
|
try:
|
|
blob.delete()
|
|
deleted_count += 1
|
|
except Exception as e:
|
|
logger.warning(f"Error deleting blob {blob.name}: {e}")
|
|
|
|
if deleted_count > 0:
|
|
logger.info(f"Deleted {deleted_count} files from gs://{settings.gcs_bucket_name}/{prefix}")
|
|
|
|
return deleted_count
|
|
except Exception as e:
|
|
logger.error(f"Error deleting folder {prefix}: {e}")
|
|
return 0 # Don't raise - folder deletion shouldn't break operations
|
|
|
|
def list_files(self, prefix: str) -> list:
|
|
"""List files in GCS with a given prefix."""
|
|
try:
|
|
blobs = self.bucket.list_blobs(prefix=prefix)
|
|
return [blob.name for blob in blobs]
|
|
except Exception as e:
|
|
logger.error(f"Error listing files with prefix {prefix}: {e}")
|
|
raise
|
|
|
|
def file_exists(self, blob_path: str) -> bool:
|
|
"""Check if a file exists in GCS."""
|
|
try:
|
|
blob = self.bucket.blob(blob_path)
|
|
return blob.exists()
|
|
except Exception as e:
|
|
logger.error(f"Error checking file existence {blob_path}: {e}")
|
|
raise
|
|
|
|
def upload_json(self, destination_path: str, data: Dict[str, Any]) -> str:
|
|
"""Upload a JSON object to GCS."""
|
|
try:
|
|
blob = self.bucket.blob(destination_path)
|
|
blob.content_type = "application/json"
|
|
blob.upload_from_string(
|
|
json.dumps(data, indent=2, ensure_ascii=False),
|
|
content_type="application/json"
|
|
)
|
|
logger.info(f"Uploaded JSON to gs://{settings.gcs_bucket_name}/{destination_path}")
|
|
return destination_path
|
|
except Exception as e:
|
|
logger.error(f"Error uploading JSON to {destination_path}: {e}")
|
|
raise
|
|
|
|
def download_json(self, source_path: str) -> Dict[str, Any]:
|
|
"""Download and parse a JSON file from GCS."""
|
|
try:
|
|
blob = self.bucket.blob(source_path)
|
|
content = blob.download_as_text()
|
|
data = json.loads(content)
|
|
logger.info(f"Downloaded JSON from gs://{settings.gcs_bucket_name}/{source_path}")
|
|
return data
|
|
except Exception as e:
|
|
logger.error(f"Error downloading JSON from {source_path}: {e}")
|
|
raise
|
|
|