mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-22 05:47:55 +09:00
Compare commits
55
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e46cf57fb | ||
|
|
ed3e6ae039 | ||
|
|
4fe7d84789 | ||
|
|
fdd84bad36 | ||
|
|
f52964441a | ||
|
|
92affebded | ||
|
|
3221c2626b | ||
|
|
2ea21b0e51 | ||
|
|
76fa4befd8 | ||
|
|
583dd323dc | ||
|
|
ef2ecb4c55 | ||
|
|
6d28e09fce | ||
|
|
565586aca4 | ||
|
|
9147e2f21b | ||
|
|
9ad20e3a36 | ||
|
|
85a7f62b01 | ||
|
|
09f9e15ff0 | ||
|
|
dd62d30fa9 | ||
|
|
adfd3de651 | ||
|
|
871e97cae7 | ||
|
|
03e1aec641 | ||
|
|
40948c2ada | ||
|
|
2182bf7e58 | ||
|
|
ed723dd6d0 | ||
|
|
ddc4460065 | ||
|
|
b0f3d15112 | ||
|
|
213d3b4311 | ||
|
|
753499db01 | ||
|
|
1b6e682860 | ||
|
|
9baba12280 | ||
|
|
24b7b75ab2 | ||
|
|
0b59b7abfc | ||
|
|
942cbb4f66 | ||
|
|
96383a60d4 | ||
|
|
0cf14be531 | ||
|
|
95edd82555 | ||
|
|
6e02ca3056 | ||
|
|
966e8cc963 | ||
|
|
04b1c8fd3e | ||
|
|
f47b126c08 | ||
|
|
4438293b6d | ||
|
|
de4e92d9bf | ||
|
|
e7f88b60d0 | ||
|
|
fbb1cd7299 | ||
|
|
77820a44c9 | ||
|
|
e92ddd8609 | ||
|
|
a2bf2230ab | ||
|
|
75c6879b7e | ||
|
|
f51ef68179 | ||
|
|
8a5420f5f8 | ||
|
|
c9b7f1bb1a | ||
|
|
d9da1dd360 | ||
|
|
a8fa6b5f46 | ||
|
|
ebe0121938 | ||
|
|
01db9a764a |
No files matched your search
@@ -397,8 +397,34 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: false
|
||||
fetch-depth: 0 # Need full history for tag check
|
||||
|
||||
- name: Set up Python (for version extraction)
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
|
||||
- name: Extract version and check if deployment needed
|
||||
id: version_check
|
||||
run: |
|
||||
VERSION=$(poetry version --short)
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "📋 Current version: $VERSION"
|
||||
|
||||
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
|
||||
echo "needs_deploy=false" >> $GITHUB_OUTPUT
|
||||
echo "⏭️ Tag v$VERSION already exists - skipping backend deployment"
|
||||
echo " (Bump version in pyproject.toml to trigger a new deployment)"
|
||||
else
|
||||
echo "needs_deploy=true" >> $GITHUB_OUTPUT
|
||||
echo "✨ Tag v$VERSION does not exist - proceeding with deployment"
|
||||
fi
|
||||
|
||||
- name: Authenticate to Google Cloud (Workload Identity)
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
@@ -406,17 +432,20 @@ jobs:
|
||||
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
||||
|
||||
- name: Set up Cloud SDK
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
version: "latest"
|
||||
|
||||
- name: Check if base image needs rebuild
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
id: check-base
|
||||
run: |
|
||||
echo "🔍 Checking if base image needs rebuild..."
|
||||
|
||||
CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1)
|
||||
echo "Current poetry.lock hash: $CURRENT_HASH"
|
||||
# Hash both poetry.lock AND Dockerfile.base - rebuild if either changes
|
||||
CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
|
||||
echo "Current combined hash (poetry.lock + Dockerfile.base): $CURRENT_HASH"
|
||||
|
||||
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
|
||||
|
||||
@@ -424,14 +453,18 @@ jobs:
|
||||
EXISTING_HASH=""
|
||||
|
||||
if docker pull $BASE_IMAGE 2>/dev/null; then
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "base.content.hash"}}' 2>/dev/null || echo "")
|
||||
# Fallback to old label name for backwards compatibility
|
||||
if [ -z "$EXISTING_HASH" ]; then
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
|
||||
fi
|
||||
echo "Existing base image hash: $EXISTING_HASH"
|
||||
else
|
||||
echo "⚠️ No existing base image found"
|
||||
fi
|
||||
|
||||
if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then
|
||||
echo "🔄 Base image needs rebuild (poetry.lock changed)"
|
||||
echo "🔄 Base image needs rebuild (poetry.lock or Dockerfile.base changed)"
|
||||
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "✅ Base image is up to date"
|
||||
@@ -439,7 +472,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Rebuild base image (if needed)
|
||||
if: steps.check-base.outputs.needs_rebuild == 'true'
|
||||
if: steps.version_check.outputs.needs_deploy == 'true' && steps.check-base.outputs.needs_rebuild == 'true'
|
||||
run: |
|
||||
echo "🏗️ Rebuilding base image with updated dependencies..."
|
||||
|
||||
@@ -479,9 +512,11 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Trigger Cloud Build
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
run: |
|
||||
echo "🚀 Triggering Cloud Build for deployment..."
|
||||
echo "Commit: ${{ github.sha }}"
|
||||
echo "Version: ${{ steps.version_check.outputs.version }}"
|
||||
|
||||
BUILD_ID=$(gcloud builds submit \
|
||||
--config=cloudbuild.yaml \
|
||||
@@ -519,21 +554,61 @@ jobs:
|
||||
sleep 30
|
||||
done
|
||||
|
||||
- name: Verify deployment
|
||||
- name: Verify deployment and version
|
||||
if: steps.version_check.outputs.needs_deploy == 'true'
|
||||
run: |
|
||||
EXPECTED_VERSION="${{ steps.version_check.outputs.version }}"
|
||||
SERVICE_URL=$(gcloud run services describe karaoke-backend \
|
||||
--region us-central1 \
|
||||
--format 'value(status.url)')
|
||||
|
||||
echo "Service URL: $SERVICE_URL"
|
||||
sleep 10
|
||||
echo "Expected version: $EXPECTED_VERSION"
|
||||
|
||||
# Wait for new revision to be ready
|
||||
echo "Waiting for new revision to be ready..."
|
||||
sleep 15
|
||||
|
||||
# Basic health check
|
||||
curl -f "$SERVICE_URL/api/health" || exit 1
|
||||
echo "✅ Deployment verified - service is healthy"
|
||||
echo "✅ Service is healthy"
|
||||
|
||||
# Version verification with retries (Cloud Run may take time to route to new revision)
|
||||
MAX_RETRIES=6
|
||||
RETRY_DELAY=10
|
||||
for i in $(seq 1 $MAX_RETRIES); do
|
||||
echo "Checking deployed version (attempt $i/$MAX_RETRIES)..."
|
||||
DEPLOYED_VERSION=$(curl -sf "$SERVICE_URL/api/health/detailed" | jq -r '.version // empty')
|
||||
|
||||
if [ "$DEPLOYED_VERSION" = "$EXPECTED_VERSION" ]; then
|
||||
echo "✅ Version verified: $DEPLOYED_VERSION"
|
||||
break
|
||||
else
|
||||
echo "⚠️ Version mismatch: deployed=$DEPLOYED_VERSION, expected=$EXPECTED_VERSION"
|
||||
if [ $i -lt $MAX_RETRIES ]; then
|
||||
echo " Retrying in ${RETRY_DELAY}s..."
|
||||
sleep $RETRY_DELAY
|
||||
else
|
||||
echo "❌ Version verification failed after $MAX_RETRIES attempts"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Summary
|
||||
if: success()
|
||||
if: steps.version_check.outputs.needs_deploy == 'true' && success()
|
||||
run: |
|
||||
echo "🎉 Deployment successful!"
|
||||
echo "Version: ${{ steps.version_check.outputs.version }}"
|
||||
echo "Commit: ${{ github.sha }}"
|
||||
echo "Branch: ${{ github.ref_name }}"
|
||||
echo "Run: ${{ github.run_number }}"
|
||||
|
||||
- name: Skipped deployment summary
|
||||
if: steps.version_check.outputs.needs_deploy == 'false'
|
||||
run: |
|
||||
echo "⏭️ Backend deployment skipped - version ${{ steps.version_check.outputs.version }} already deployed"
|
||||
echo ""
|
||||
echo "To deploy a new version:"
|
||||
echo " 1. Update version in pyproject.toml"
|
||||
echo " 2. Commit and push to main"
|
||||
+8
-3
@@ -11,7 +11,9 @@
|
||||
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
|
||||
|
||||
# Use pre-built base image with all dependencies
|
||||
FROM us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
|
||||
# Override with --build-arg BASE_IMAGE=karaoke-backend-base:local for local testing
|
||||
ARG BASE_IMAGE=us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
# Set working directory (should already exist from base)
|
||||
WORKDIR /app
|
||||
@@ -34,5 +36,8 @@ ENV PORT=8080
|
||||
# Expose port
|
||||
EXPOSE 8080
|
||||
|
||||
# Run the application
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
# Run uvicorn directly
|
||||
# Note: Transmission daemon removed - torrent downloads now handled by
|
||||
# dedicated flacfetch VM service (see docs/00-current-plan/TORRENT-SERVICE-PLAN.md)
|
||||
# Using --loop asyncio (instead of uvloop) to support nest_asyncio for remote flacfetch calls
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080", "--loop", "asyncio"]
|
||||
@@ -7,12 +7,15 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies required by karaoke-gen
|
||||
# Note: Transmission daemon removed - torrent downloads now handled by
|
||||
# dedicated flacfetch VM service (see docs/00-current-plan/TORRENT-SERVICE-PLAN.md)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ffmpeg \
|
||||
libsndfile1 \
|
||||
libsox-dev \
|
||||
sox \
|
||||
build-essential \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set working directory
|
||||
@@ -34,9 +37,9 @@ RUN python -m spacy download en_core_web_sm
|
||||
# Label with build info
|
||||
ARG BUILD_DATE
|
||||
ARG BUILD_ID
|
||||
ARG POETRY_LOCK_HASH
|
||||
ARG BASE_CONTENT_HASH
|
||||
LABEL org.opencontainers.image.created="${BUILD_DATE}"
|
||||
LABEL org.opencontainers.image.revision="${BUILD_ID}"
|
||||
LABEL org.opencontainers.image.title="karaoke-backend-base"
|
||||
LABEL org.opencontainers.image.description="Base image with Python dependencies for karaoke-backend"
|
||||
LABEL poetry.lock.hash="${POETRY_LOCK_HASH}"
|
||||
LABEL org.opencontainers.image.description="Base image with Python dependencies and system tools for karaoke-backend"
|
||||
LABEL base.content.hash="${BASE_CONTENT_HASH}"
|
||||
@@ -70,7 +70,15 @@ async def require_auth(
|
||||
is_valid, user_type, remaining_uses, message = auth_service.validate_token(token_str)
|
||||
|
||||
if not is_valid:
|
||||
logger.warning(f"Authentication failed: {message}")
|
||||
# Log more details for debugging token issues
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
logger.warning(
|
||||
f"Authentication failed: {message}. "
|
||||
f"Token provided: {bool(token_str)}, "
|
||||
f"Token length: {len(token_str) if token_str else 0}, "
|
||||
f"Auth header present: {bool(auth_header)}, "
|
||||
f"Header prefix: {auth_header[:20] if auth_header else 'none'}..."
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail=f"Authentication failed: {message}"
|
||||
|
||||
@@ -82,15 +82,40 @@ class AudioSearchRequest(BaseModel):
|
||||
|
||||
|
||||
class AudioSearchResultResponse(BaseModel):
|
||||
"""A single audio search result."""
|
||||
"""A single audio search result.
|
||||
|
||||
Contains all fields needed for rich display using flacfetch's
|
||||
shared formatting functions.
|
||||
"""
|
||||
index: int
|
||||
title: str
|
||||
artist: str
|
||||
provider: str
|
||||
url: str
|
||||
duration: Optional[int] = None
|
||||
quality: Optional[str] = None
|
||||
source_id: Optional[str] = None
|
||||
provider: str # Maps to source_name in Release
|
||||
url: Optional[str] = None # Maps to download_url in Release (may be None for remote)
|
||||
duration: Optional[int] = None # Maps to duration_seconds in Release
|
||||
quality: Optional[str] = None # Stringified quality
|
||||
source_id: Optional[str] = None # Maps to info_hash in Release
|
||||
seeders: Optional[int] = None
|
||||
target_file: Optional[str] = None
|
||||
# Additional fields for rich display (from Release.to_dict())
|
||||
year: Optional[int] = None
|
||||
label: Optional[str] = None
|
||||
edition_info: Optional[str] = None
|
||||
release_type: Optional[str] = None
|
||||
channel: Optional[str] = None # For YouTube
|
||||
view_count: Optional[int] = None # For YouTube
|
||||
size_bytes: Optional[int] = None
|
||||
target_file_size: Optional[int] = None
|
||||
track_pattern: Optional[str] = None
|
||||
match_score: Optional[float] = None
|
||||
# Pre-computed display fields
|
||||
formatted_size: Optional[str] = None
|
||||
formatted_duration: Optional[str] = None
|
||||
formatted_views: Optional[str] = None
|
||||
is_lossless: Optional[bool] = None
|
||||
quality_str: Optional[str] = None
|
||||
# Full quality object for Release.from_dict()
|
||||
quality_data: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class AudioSearchResponse(BaseModel):
|
||||
@@ -168,6 +193,10 @@ async def _download_and_start_processing(
|
||||
This is called either:
|
||||
- Immediately after search if auto_download=True
|
||||
- When user calls the select endpoint
|
||||
|
||||
For remote flacfetch downloads (torrent sources), the file is downloaded
|
||||
on the flacfetch VM and uploaded directly to GCS. For local downloads
|
||||
(YouTube), the file is downloaded locally and then uploaded to GCS.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
@@ -199,33 +228,71 @@ async def _download_and_start_processing(
|
||||
)
|
||||
|
||||
try:
|
||||
# Download to temp directory
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"audio_download_{job_id}_")
|
||||
# Determine if this is a remote torrent download
|
||||
is_torrent_source = selected.get('provider') in ['Redacted', 'OPS']
|
||||
is_remote_enabled = audio_search_service.is_remote_enabled()
|
||||
|
||||
result = audio_search_service.download(
|
||||
result_index=selection_index,
|
||||
output_dir=temp_dir,
|
||||
)
|
||||
|
||||
# Upload to GCS
|
||||
filename = os.path.basename(result.filepath)
|
||||
audio_gcs_path = f"uploads/{job_id}/audio/{filename}"
|
||||
|
||||
with open(result.filepath, 'rb') as f:
|
||||
storage_service.upload_fileobj(
|
||||
f,
|
||||
audio_gcs_path,
|
||||
content_type='audio/flac' # flacfetch typically returns FLAC
|
||||
# For remote torrent downloads, have flacfetch VM upload directly to GCS
|
||||
if is_torrent_source and is_remote_enabled:
|
||||
# Generate GCS path for remote upload
|
||||
gcs_destination = f"uploads/{job_id}/audio/"
|
||||
|
||||
logger.info(f"Using remote download with GCS upload to: {gcs_destination}")
|
||||
|
||||
result = audio_search_service.download(
|
||||
result_index=selection_index,
|
||||
output_dir="", # Not used for remote
|
||||
gcs_path=gcs_destination,
|
||||
)
|
||||
|
||||
logger.info(f"Uploaded audio to GCS: {audio_gcs_path}")
|
||||
|
||||
# Clean up temp file
|
||||
try:
|
||||
os.remove(result.filepath)
|
||||
os.rmdir(temp_dir)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp files: {e}")
|
||||
|
||||
# For remote downloads, filepath is already the GCS path
|
||||
if result.filepath.startswith("gs://"):
|
||||
# Extract the path portion after the bucket name
|
||||
# Format: gs://bucket/uploads/job_id/audio/filename.flac
|
||||
parts = result.filepath.replace("gs://", "").split("/", 1)
|
||||
if len(parts) == 2:
|
||||
audio_gcs_path = parts[1]
|
||||
else:
|
||||
audio_gcs_path = result.filepath
|
||||
filename = os.path.basename(result.filepath)
|
||||
else:
|
||||
# Fallback: treat as local path (shouldn't happen for remote)
|
||||
filename = os.path.basename(result.filepath)
|
||||
audio_gcs_path = f"uploads/{job_id}/audio/{filename}"
|
||||
|
||||
logger.warning(f"Remote download returned local path: {result.filepath}, uploading manually")
|
||||
with open(result.filepath, 'rb') as f:
|
||||
storage_service.upload_fileobj(f, audio_gcs_path, content_type='audio/flac')
|
||||
|
||||
logger.info(f"Remote download complete, GCS path: {audio_gcs_path}")
|
||||
else:
|
||||
# Local download (YouTube or fallback)
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"audio_download_{job_id}_")
|
||||
|
||||
result = audio_search_service.download(
|
||||
result_index=selection_index,
|
||||
output_dir=temp_dir,
|
||||
)
|
||||
|
||||
# Upload to GCS
|
||||
filename = os.path.basename(result.filepath)
|
||||
audio_gcs_path = f"uploads/{job_id}/audio/{filename}"
|
||||
|
||||
with open(result.filepath, 'rb') as f:
|
||||
storage_service.upload_fileobj(
|
||||
f,
|
||||
audio_gcs_path,
|
||||
content_type='audio/flac' # flacfetch typically returns FLAC
|
||||
)
|
||||
|
||||
logger.info(f"Uploaded audio to GCS: {audio_gcs_path}")
|
||||
|
||||
# Clean up temp file
|
||||
try:
|
||||
os.remove(result.filepath)
|
||||
os.rmdir(temp_dir)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp files: {e}")
|
||||
|
||||
# Update job with GCS path and transition to DOWNLOADING
|
||||
job_manager.update_job(job_id, {
|
||||
@@ -418,20 +485,50 @@ async def search_audio(
|
||||
message=f"Found {len(search_results)} audio sources. Waiting for selection."
|
||||
)
|
||||
|
||||
# Convert to response format
|
||||
result_responses = [
|
||||
AudioSearchResultResponse(
|
||||
index=r.index,
|
||||
title=r.title,
|
||||
artist=r.artist,
|
||||
provider=r.provider,
|
||||
url=r.url,
|
||||
duration=r.duration,
|
||||
quality=r.quality,
|
||||
source_id=r.source_id,
|
||||
# Convert to response format with full Release data for rich display
|
||||
result_responses = []
|
||||
for r in search_results:
|
||||
# Get full serialized data from raw_result if available
|
||||
raw_dict = {}
|
||||
if r.raw_result:
|
||||
try:
|
||||
raw_dict = r.raw_result.to_dict()
|
||||
except AttributeError:
|
||||
pass # Not a Release object
|
||||
|
||||
result_responses.append(
|
||||
AudioSearchResultResponse(
|
||||
index=r.index,
|
||||
title=r.title,
|
||||
artist=r.artist,
|
||||
provider=r.provider,
|
||||
url=r.url,
|
||||
duration=r.duration,
|
||||
quality=r.quality,
|
||||
source_id=r.source_id,
|
||||
seeders=raw_dict.get('seeders') or r.seeders,
|
||||
target_file=raw_dict.get('target_file') or r.target_file,
|
||||
# Additional fields for rich display
|
||||
year=raw_dict.get('year'),
|
||||
label=raw_dict.get('label'),
|
||||
edition_info=raw_dict.get('edition_info'),
|
||||
release_type=raw_dict.get('release_type'),
|
||||
channel=raw_dict.get('channel'),
|
||||
view_count=raw_dict.get('view_count'),
|
||||
size_bytes=raw_dict.get('size_bytes'),
|
||||
target_file_size=raw_dict.get('target_file_size'),
|
||||
track_pattern=raw_dict.get('track_pattern'),
|
||||
match_score=raw_dict.get('match_score'),
|
||||
# Pre-computed display fields
|
||||
formatted_size=raw_dict.get('formatted_size'),
|
||||
formatted_duration=raw_dict.get('formatted_duration'),
|
||||
formatted_views=raw_dict.get('formatted_views'),
|
||||
is_lossless=raw_dict.get('is_lossless'),
|
||||
quality_str=raw_dict.get('quality_str'),
|
||||
# Full quality object for Release.from_dict()
|
||||
quality_data=raw_dict.get('quality'),
|
||||
)
|
||||
)
|
||||
for r in search_results
|
||||
]
|
||||
|
||||
return AudioSearchResponse(
|
||||
status="awaiting_selection",
|
||||
@@ -473,6 +570,8 @@ async def get_audio_search_results(job_id: str):
|
||||
"status": "success",
|
||||
"job_id": job_id,
|
||||
"job_status": job.status,
|
||||
"artist": job.audio_search_artist or job.artist,
|
||||
"title": job.audio_search_title or job.title,
|
||||
"results": search_results,
|
||||
"results_count": len(search_results),
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ from backend.services.worker_service import get_worker_service
|
||||
from backend.services.credential_manager import get_credential_manager, CredentialStatus
|
||||
from backend.config import get_settings
|
||||
from backend.version import VERSION
|
||||
from backend.services.metrics import metrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(tags=["jobs"])
|
||||
@@ -428,6 +429,9 @@ async def upload_and_create_job(
|
||||
job = job_manager.create_job(job_create)
|
||||
job_id = job.job_id
|
||||
|
||||
# Record job creation metric
|
||||
metrics.record_job_created(job_id, source="upload")
|
||||
|
||||
logger.info(f"Created job {job_id} for {artist} - {title}")
|
||||
|
||||
# Upload main audio file to GCS
|
||||
@@ -882,6 +886,9 @@ async def create_job_with_upload_urls(
|
||||
job = job_manager.create_job(job_create)
|
||||
job_id = job.job_id
|
||||
|
||||
# Record job creation metric
|
||||
metrics.record_job_created(job_id, source="upload")
|
||||
|
||||
logger.info(f"Created job {job_id} for {body.artist} - {body.title} (signed URL upload flow)")
|
||||
|
||||
# Generate signed upload URLs for each file
|
||||
@@ -1265,6 +1272,9 @@ async def create_job_from_url(
|
||||
job = job_manager.create_job(job_create)
|
||||
job_id = job.job_id
|
||||
|
||||
# Record job creation metric
|
||||
metrics.record_job_created(job_id, source="url")
|
||||
|
||||
logger.info(f"Created URL-based job {job_id} for URL: {body.url}")
|
||||
if artist:
|
||||
logger.info(f" Artist: {artist}")
|
||||
@@ -1466,6 +1476,9 @@ async def create_finalise_only_job(
|
||||
job = job_manager.create_job(job_create)
|
||||
job_id = job.job_id
|
||||
|
||||
# Record job creation metric
|
||||
metrics.record_job_created(job_id, source="finalise")
|
||||
|
||||
logger.info(f"Created finalise-only job {job_id} for {body.artist} - {body.title}")
|
||||
|
||||
# Generate signed upload URLs for each file
|
||||
|
||||
@@ -1,10 +1,64 @@
|
||||
"""
|
||||
Health check routes.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from fastapi import APIRouter
|
||||
from typing import Dict
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from backend.version import VERSION
|
||||
from backend.services.flacfetch_client import get_flacfetch_client
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_transmission_status() -> Dict[str, Any]:
|
||||
"""Check if Transmission daemon is available and responsive."""
|
||||
try:
|
||||
import transmission_rpc
|
||||
host = os.environ.get("TRANSMISSION_HOST", "localhost")
|
||||
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
|
||||
|
||||
client = transmission_rpc.Client(host=host, port=port, timeout=5)
|
||||
stats = client.session_stats()
|
||||
|
||||
# Get detailed torrent info
|
||||
torrents = client.get_torrents()
|
||||
torrent_details = []
|
||||
for t in torrents:
|
||||
torrent_info = {
|
||||
"name": t.name,
|
||||
"progress": round(t.progress, 1),
|
||||
"status": str(t.status),
|
||||
"peers": t.peers_connected if hasattr(t, 'peers_connected') else 0,
|
||||
"download_speed": round(t.rate_download / 1024, 1) if hasattr(t, 'rate_download') else 0, # KB/s
|
||||
}
|
||||
# Check if stalled (downloading but no progress and no peers)
|
||||
if t.status.downloading and t.progress < 100 and torrent_info['peers'] == 0:
|
||||
torrent_info['stalled'] = True
|
||||
torrent_details.append(torrent_info)
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"host": host,
|
||||
"port": port,
|
||||
"download_dir": stats.download_dir if hasattr(stats, 'download_dir') else None,
|
||||
"active_torrent_count": stats.active_torrent_count if hasattr(stats, 'active_torrent_count') else 0,
|
||||
"torrents": torrent_details,
|
||||
}
|
||||
except ImportError as e:
|
||||
return {
|
||||
"available": False,
|
||||
"error": f"transmission_rpc not installed: {e}",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"available": False,
|
||||
"host": os.environ.get("TRANSMISSION_HOST", "localhost"),
|
||||
"port": int(os.environ.get("TRANSMISSION_PORT", "9091")),
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
@@ -16,6 +70,67 @@ async def health_check() -> Dict[str, str]:
|
||||
}
|
||||
|
||||
|
||||
async def check_flacfetch_service_status() -> Dict[str, Any]:
|
||||
"""Check if remote flacfetch service is available and healthy."""
|
||||
client = get_flacfetch_client()
|
||||
|
||||
if not client:
|
||||
return {
|
||||
"configured": False,
|
||||
"message": "Remote flacfetch service not configured (FLACFETCH_API_URL not set)",
|
||||
}
|
||||
|
||||
try:
|
||||
health = await client.health_check()
|
||||
return {
|
||||
"configured": True,
|
||||
"available": True,
|
||||
"status": health.get("status"),
|
||||
"version": health.get("version"),
|
||||
"transmission": health.get("transmission", {}),
|
||||
"disk": health.get("disk", {}),
|
||||
"providers": health.get("providers", {}),
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"configured": True,
|
||||
"available": False,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health/detailed")
|
||||
async def detailed_health_check() -> Dict[str, Any]:
|
||||
"""
|
||||
Detailed health check including dependencies.
|
||||
|
||||
Use this to debug issues with Transmission, flacfetch service, etc.
|
||||
"""
|
||||
transmission_status = check_transmission_status()
|
||||
flacfetch_status = await check_flacfetch_service_status()
|
||||
|
||||
# Log for debugging
|
||||
if not transmission_status.get("available"):
|
||||
logger.warning(f"Local Transmission not available: {transmission_status.get('error')}")
|
||||
else:
|
||||
logger.info(f"Local Transmission available at {transmission_status.get('host')}:{transmission_status.get('port')}")
|
||||
|
||||
if flacfetch_status.get("configured") and not flacfetch_status.get("available"):
|
||||
logger.warning(f"Remote flacfetch service not available: {flacfetch_status.get('error')}")
|
||||
elif flacfetch_status.get("available"):
|
||||
logger.info(f"Remote flacfetch service healthy: {flacfetch_status.get('status')}")
|
||||
|
||||
return {
|
||||
"status": "healthy",
|
||||
"service": "karaoke-gen-backend",
|
||||
"version": VERSION,
|
||||
"dependencies": {
|
||||
"transmission_local": transmission_status,
|
||||
"flacfetch_remote": flacfetch_status,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@router.get("/readiness")
|
||||
async def readiness_check() -> Dict[str, str]:
|
||||
"""Readiness check endpoint for Cloud Run."""
|
||||
|
||||
@@ -6,11 +6,17 @@ They are protected by admin authentication.
|
||||
|
||||
With Cloud Tasks integration, these endpoints may be called multiple times
|
||||
(retry on failure). Idempotency checks prevent duplicate processing.
|
||||
|
||||
Observability:
|
||||
- Extracts trace context from incoming requests (propagated via Cloud Tasks)
|
||||
- Creates worker spans linked to the original request trace
|
||||
- All logs include job_id for easy filtering in Cloud Logging
|
||||
"""
|
||||
import logging
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Tuple, Optional
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.workers.audio_worker import process_audio_separation
|
||||
@@ -21,6 +27,12 @@ from backend.workers.render_video_worker import process_render_video
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.auth_service import UserType
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.tracing import (
|
||||
extract_trace_context,
|
||||
start_span_with_context,
|
||||
add_span_attribute,
|
||||
add_span_event,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -57,7 +69,7 @@ def _check_worker_idempotency(job_id: str, worker_name: str) -> Optional[WorkerR
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
if not job:
|
||||
logger.warning(f"Job {job_id} not found for {worker_name} worker")
|
||||
logger.warning(f"[job:{job_id}] Job not found for {worker_name} worker")
|
||||
return WorkerResponse(
|
||||
status="not_found",
|
||||
job_id=job_id,
|
||||
@@ -70,7 +82,7 @@ def _check_worker_idempotency(job_id: str, worker_name: str) -> Optional[WorkerR
|
||||
stage = worker_progress.get('stage')
|
||||
|
||||
if stage == 'running':
|
||||
logger.info(f"{worker_name.capitalize()} worker already running for job {job_id}, skipping")
|
||||
logger.info(f"[job:{job_id}] {worker_name.capitalize()} worker already running, skipping")
|
||||
return WorkerResponse(
|
||||
status="already_running",
|
||||
job_id=job_id,
|
||||
@@ -78,7 +90,7 @@ def _check_worker_idempotency(job_id: str, worker_name: str) -> Optional[WorkerR
|
||||
)
|
||||
|
||||
if stage == 'complete':
|
||||
logger.info(f"{worker_name.capitalize()} worker already complete for job {job_id}, skipping")
|
||||
logger.info(f"[job:{job_id}] {worker_name.capitalize()} worker already complete, skipping")
|
||||
return WorkerResponse(
|
||||
status="already_complete",
|
||||
job_id=job_id,
|
||||
@@ -93,6 +105,7 @@ def _check_worker_idempotency(job_id: str, worker_name: str) -> Optional[WorkerR
|
||||
@router.post("/workers/audio", response_model=WorkerResponse)
|
||||
async def trigger_audio_worker(
|
||||
request: WorkerRequest,
|
||||
http_request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
@@ -107,11 +120,18 @@ async def trigger_audio_worker(
|
||||
The worker runs in the background and updates job state as it progresses.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering audio worker for job {job_id}")
|
||||
|
||||
# Extract trace context from incoming request (propagated via Cloud Tasks)
|
||||
trace_context = extract_trace_context(dict(http_request.headers))
|
||||
|
||||
logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=audio")
|
||||
add_span_attribute("job_id", job_id)
|
||||
add_span_attribute("worker", "audio")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "audio")
|
||||
if skip_response:
|
||||
add_span_event("worker_skipped", {"reason": skip_response.status})
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
@@ -119,6 +139,7 @@ async def trigger_audio_worker(
|
||||
# while the worker continues processing
|
||||
background_tasks.add_task(process_audio_separation, job_id)
|
||||
|
||||
add_span_event("worker_started")
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
@@ -129,6 +150,7 @@ async def trigger_audio_worker(
|
||||
@router.post("/workers/lyrics", response_model=WorkerResponse)
|
||||
async def trigger_lyrics_worker(
|
||||
request: WorkerRequest,
|
||||
http_request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
@@ -143,16 +165,24 @@ async def trigger_lyrics_worker(
|
||||
The worker runs in the background and updates job state as it progresses.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering lyrics worker for job {job_id}")
|
||||
|
||||
# Extract trace context from incoming request
|
||||
trace_context = extract_trace_context(dict(http_request.headers))
|
||||
|
||||
logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=lyrics")
|
||||
add_span_attribute("job_id", job_id)
|
||||
add_span_attribute("worker", "lyrics")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "lyrics")
|
||||
if skip_response:
|
||||
add_span_event("worker_skipped", {"reason": skip_response.status})
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(process_lyrics_transcription, job_id)
|
||||
|
||||
add_span_event("worker_started")
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
@@ -163,6 +193,7 @@ async def trigger_lyrics_worker(
|
||||
@router.post("/workers/screens", response_model=WorkerResponse)
|
||||
async def trigger_screens_worker(
|
||||
request: WorkerRequest,
|
||||
http_request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
@@ -174,16 +205,24 @@ async def trigger_screens_worker(
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering screens worker for job {job_id}")
|
||||
|
||||
# Extract trace context from incoming request
|
||||
trace_context = extract_trace_context(dict(http_request.headers))
|
||||
|
||||
logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=screens")
|
||||
add_span_attribute("job_id", job_id)
|
||||
add_span_attribute("worker", "screens")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "screens")
|
||||
if skip_response:
|
||||
add_span_event("worker_skipped", {"reason": skip_response.status})
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(generate_screens, job_id)
|
||||
|
||||
add_span_event("worker_started")
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
@@ -194,6 +233,7 @@ async def trigger_screens_worker(
|
||||
@router.post("/workers/video", response_model=WorkerResponse)
|
||||
async def trigger_video_worker(
|
||||
request: WorkerRequest,
|
||||
http_request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
@@ -206,16 +246,24 @@ async def trigger_video_worker(
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering video worker for job {job_id}")
|
||||
|
||||
# Extract trace context from incoming request
|
||||
trace_context = extract_trace_context(dict(http_request.headers))
|
||||
|
||||
logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=video")
|
||||
add_span_attribute("job_id", job_id)
|
||||
add_span_attribute("worker", "video")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "video")
|
||||
if skip_response:
|
||||
add_span_event("worker_skipped", {"reason": skip_response.status})
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(generate_video, job_id)
|
||||
|
||||
add_span_event("worker_started")
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
@@ -226,6 +274,7 @@ async def trigger_video_worker(
|
||||
@router.post("/workers/render-video", response_model=WorkerResponse)
|
||||
async def trigger_render_video_worker(
|
||||
request: WorkerRequest,
|
||||
http_request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
@@ -242,16 +291,24 @@ async def trigger_render_video_worker(
|
||||
Next state: AWAITING_INSTRUMENTAL_SELECTION
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering render-video worker for job {job_id}")
|
||||
|
||||
# Extract trace context from incoming request
|
||||
trace_context = extract_trace_context(dict(http_request.headers))
|
||||
|
||||
logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=render-video")
|
||||
add_span_attribute("job_id", job_id)
|
||||
add_span_attribute("worker", "render-video")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "render")
|
||||
if skip_response:
|
||||
add_span_event("worker_skipped", {"reason": skip_response.status})
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(process_render_video, job_id)
|
||||
|
||||
add_span_event("worker_started")
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
|
||||
@@ -28,6 +28,7 @@ from backend.services.storage_service import StorageService
|
||||
from backend.config import get_settings
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.auth_service import UserType
|
||||
from backend.services.metrics import metrics
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -81,6 +82,9 @@ async def create_job(
|
||||
)
|
||||
job = job_manager.create_job(job_create)
|
||||
|
||||
# Record job creation metric
|
||||
metrics.record_job_created(job.job_id, source="url")
|
||||
|
||||
# Trigger both workers in parallel using asyncio.gather
|
||||
# (FastAPI's BackgroundTasks runs async tasks sequentially)
|
||||
background_tasks.add_task(_trigger_workers_parallel, job.job_id)
|
||||
|
||||
@@ -56,6 +56,12 @@ class Settings(BaseSettings):
|
||||
# Storage paths
|
||||
temp_dir: str = os.getenv("TEMP_DIR", "/tmp/karaoke-gen")
|
||||
|
||||
# Flacfetch remote service (for torrent downloads)
|
||||
# When configured, audio search uses the remote flacfetch HTTP API instead of local flacfetch.
|
||||
# This is required for torrent downloads since Cloud Run doesn't support BitTorrent.
|
||||
flacfetch_api_url: Optional[str] = os.getenv("FLACFETCH_API_URL") # e.g., http://10.0.0.5:8080
|
||||
flacfetch_api_key: Optional[str] = os.getenv("FLACFETCH_API_KEY")
|
||||
|
||||
# Default distribution settings (can be overridden per-request)
|
||||
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
|
||||
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
|
||||
|
||||
+4
-5
@@ -9,16 +9,15 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from backend.config import settings
|
||||
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search
|
||||
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
|
||||
from backend.services.structured_logging import setup_structured_logging
|
||||
|
||||
|
||||
from backend.version import VERSION
|
||||
|
||||
|
||||
# Configure logging with trace ID support
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, settings.log_level.upper()),
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
# Configure structured logging (JSON in Cloud Run, human-readable locally)
|
||||
# This must happen before any logging calls
|
||||
setup_structured_logging()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Initialize OpenTelemetry tracing (must happen before app creation)
|
||||
|
||||
@@ -1,79 +1,69 @@
|
||||
"""
|
||||
Audio search service for finding and downloading audio files.
|
||||
|
||||
This service integrates with flacfetch to search for audio from various sources
|
||||
(YouTube, music trackers, etc.) and download the selected audio file.
|
||||
This service integrates with karaoke_gen.audio_fetcher (which wraps flacfetch)
|
||||
to search for audio from various sources (YouTube, music trackers, etc.)
|
||||
and download the selected audio file.
|
||||
|
||||
This is a thin wrapper that adds backend-specific functionality:
|
||||
- Caching raw results for API-based selection flow
|
||||
- Firestore-compatible serialization
|
||||
- Singleton pattern for service lifecycle
|
||||
- Remote flacfetch service integration for torrent downloads
|
||||
|
||||
When FLACFETCH_API_URL is configured, this service uses the remote flacfetch
|
||||
HTTP API for search and download operations. This is required for torrent
|
||||
downloads since Cloud Run doesn't support BitTorrent peer connections.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, asdict
|
||||
from typing import List, Optional, Dict, Any
|
||||
from typing import List, Optional
|
||||
|
||||
import nest_asyncio
|
||||
|
||||
# Import shared classes from karaoke_gen.audio_fetcher - single source of truth
|
||||
# Note: Import directly from audio_fetcher module, not karaoke_gen.__init__
|
||||
# to avoid pulling in heavier dependencies like lyrics_transcriber
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
FlacFetcher,
|
||||
AudioSearchResult,
|
||||
AudioFetchResult,
|
||||
AudioFetcherError,
|
||||
NoResultsError,
|
||||
DownloadError,
|
||||
)
|
||||
|
||||
from .flacfetch_client import get_flacfetch_client, FlacfetchServiceError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioSearchResult:
|
||||
"""
|
||||
Represents a single search result for audio.
|
||||
|
||||
This is a serializable representation of a search result that can be
|
||||
stored in Firestore and sent to clients.
|
||||
"""
|
||||
title: str
|
||||
artist: str
|
||||
provider: str
|
||||
url: str
|
||||
duration: Optional[int] = None # Duration in seconds
|
||||
quality: Optional[str] = None # e.g., "FLAC", "320kbps", etc.
|
||||
source_id: Optional[str] = None # Unique ID from the source
|
||||
index: int = 0 # Index in the results list (for selection)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict for JSON serialization."""
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AudioSearchResult":
|
||||
"""Create from dict."""
|
||||
return cls(**data)
|
||||
# Re-export exception under backend's naming convention for compatibility
|
||||
AudioSearchError = AudioFetcherError
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioDownloadResult:
|
||||
"""Result of downloading an audio file."""
|
||||
filepath: str
|
||||
artist: str
|
||||
title: str
|
||||
provider: str
|
||||
duration: Optional[int] = None
|
||||
quality: Optional[str] = None
|
||||
|
||||
|
||||
class AudioSearchError(Exception):
|
||||
"""Base exception for audio search errors."""
|
||||
pass
|
||||
|
||||
|
||||
class NoResultsError(AudioSearchError):
|
||||
"""Raised when no search results are found."""
|
||||
pass
|
||||
|
||||
|
||||
class DownloadError(AudioSearchError):
|
||||
"""Raised when download fails."""
|
||||
pass
|
||||
# Also alias AudioFetchResult as AudioDownloadResult for backwards compatibility
|
||||
AudioDownloadResult = AudioFetchResult
|
||||
|
||||
|
||||
class AudioSearchService:
|
||||
"""
|
||||
Service for searching and downloading audio files using flacfetch.
|
||||
Service for searching and downloading audio files.
|
||||
|
||||
This service is used by the backend to:
|
||||
1. Search for audio by artist/title
|
||||
2. Return results for user selection
|
||||
3. Download the selected audio to a local file
|
||||
This is a thin wrapper around karaoke_gen.audio_fetcher.FlacFetcher that adds
|
||||
backend-specific functionality:
|
||||
- Caches raw results for API-based selection (client picks by index)
|
||||
- Provides Firestore-compatible serialization
|
||||
- Manages service lifecycle as singleton
|
||||
- Remote flacfetch service integration for torrent downloads
|
||||
|
||||
When FLACFETCH_API_URL is configured, this service uses the remote flacfetch
|
||||
HTTP API for search and download operations. This enables torrent downloads
|
||||
which are not possible in Cloud Run due to network restrictions.
|
||||
|
||||
The actual flacfetch integration is in karaoke_gen.audio_fetcher.FlacFetcher,
|
||||
which is shared between local CLI and cloud backend.
|
||||
"""
|
||||
|
||||
# Sentinel value to indicate "use environment variable"
|
||||
@@ -93,60 +83,42 @@ class AudioSearchService:
|
||||
"""
|
||||
# Use environment variables if not explicitly provided
|
||||
if redacted_api_key is self._USE_ENV:
|
||||
self._redacted_api_key = os.environ.get("REDACTED_API_KEY")
|
||||
else:
|
||||
self._redacted_api_key = redacted_api_key
|
||||
redacted_api_key = os.environ.get("REDACTED_API_KEY")
|
||||
|
||||
if ops_api_key is self._USE_ENV:
|
||||
self._ops_api_key = os.environ.get("OPS_API_KEY")
|
||||
else:
|
||||
self._ops_api_key = ops_api_key
|
||||
|
||||
self._manager = None
|
||||
self._raw_results = {} # Cache raw results for download
|
||||
|
||||
def _get_manager(self):
|
||||
"""Lazily initialize and return the FetchManager."""
|
||||
if self._manager is None:
|
||||
try:
|
||||
from flacfetch.core.manager import FetchManager
|
||||
from flacfetch.providers.youtube import YouTubeProvider
|
||||
except ImportError as e:
|
||||
logger.error(f"flacfetch not installed: {e}")
|
||||
raise AudioSearchError(
|
||||
"flacfetch library is not installed. "
|
||||
"Install with: pip install flacfetch"
|
||||
) from e
|
||||
|
||||
self._manager = FetchManager()
|
||||
|
||||
# Add providers based on available API keys
|
||||
if self._redacted_api_key:
|
||||
try:
|
||||
from flacfetch.providers.redacted import RedactedProvider
|
||||
self._manager.add_provider(RedactedProvider(api_key=self._redacted_api_key))
|
||||
logger.debug("Added Redacted provider")
|
||||
except ImportError:
|
||||
logger.warning("Redacted provider not available")
|
||||
|
||||
if self._ops_api_key:
|
||||
try:
|
||||
from flacfetch.providers.ops import OPSProvider
|
||||
self._manager.add_provider(OPSProvider(api_key=self._ops_api_key))
|
||||
logger.debug("Added OPS provider")
|
||||
except ImportError:
|
||||
logger.warning("OPS provider not available")
|
||||
|
||||
# Always add YouTube as a fallback provider
|
||||
self._manager.add_provider(YouTubeProvider())
|
||||
logger.debug("Added YouTube provider")
|
||||
ops_api_key = os.environ.get("OPS_API_KEY")
|
||||
|
||||
return self._manager
|
||||
# Check for remote flacfetch client
|
||||
self._remote_client = get_flacfetch_client()
|
||||
|
||||
# Log which mode we're using
|
||||
if self._remote_client:
|
||||
logger.info("AudioSearchService using REMOTE flacfetch service (torrent downloads enabled)")
|
||||
else:
|
||||
logger.info("AudioSearchService using LOCAL flacfetch (YouTube only in Cloud Run)")
|
||||
|
||||
# Delegate to shared FlacFetcher implementation (for local mode or fallback)
|
||||
self._fetcher = FlacFetcher(
|
||||
redacted_api_key=redacted_api_key,
|
||||
ops_api_key=ops_api_key,
|
||||
)
|
||||
|
||||
# Cache search results for API-based selection flow
|
||||
# Key: index, Value: AudioSearchResult (with raw_result)
|
||||
self._cached_results: List[AudioSearchResult] = []
|
||||
|
||||
# Cache for remote search results (search_id -> results)
|
||||
self._remote_search_id: Optional[str] = None
|
||||
|
||||
def search(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""
|
||||
Search for audio matching the given artist and title.
|
||||
|
||||
Results are cached internally for later download via download().
|
||||
|
||||
If a remote flacfetch service is configured (FLACFETCH_API_URL), uses that
|
||||
for better torrent download support. Otherwise falls back to local flacfetch.
|
||||
|
||||
Args:
|
||||
artist: The artist name to search for
|
||||
title: The track title to search for
|
||||
@@ -156,49 +128,94 @@ class AudioSearchService:
|
||||
|
||||
Raises:
|
||||
NoResultsError: If no results are found
|
||||
AudioSearchError: For other errors
|
||||
AudioSearchError: For other errors (e.g., flacfetch not installed)
|
||||
"""
|
||||
try:
|
||||
from flacfetch.core.models import TrackQuery
|
||||
except ImportError as e:
|
||||
raise AudioSearchError("flacfetch not installed") from e
|
||||
|
||||
manager = self._get_manager()
|
||||
query = TrackQuery(artist=artist, title=title)
|
||||
|
||||
logger.info(f"Searching for audio: {artist} - {title}")
|
||||
|
||||
try:
|
||||
results = manager.search(query)
|
||||
# Try remote flacfetch service first if configured
|
||||
if self._remote_client:
|
||||
return self._search_remote(artist, title)
|
||||
|
||||
# Fallback to local flacfetch
|
||||
results = self._fetcher.search(artist, title)
|
||||
|
||||
# Cache results for later download
|
||||
self._cached_results = results
|
||||
self._remote_search_id = None
|
||||
|
||||
logger.info(f"Found {len(results)} results for: {artist} - {title}")
|
||||
return results
|
||||
|
||||
except (NoResultsError, AudioFetcherError):
|
||||
# Re-raise these as-is
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}")
|
||||
raise AudioSearchError(f"Search failed: {e}") from e
|
||||
|
||||
def _search_remote(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""
|
||||
Search using the remote flacfetch service.
|
||||
|
||||
if not results:
|
||||
raise NoResultsError(f"No results found for: {artist} - {title}")
|
||||
|
||||
# Convert to our AudioSearchResult format and cache raw results
|
||||
search_results = []
|
||||
self._raw_results.clear()
|
||||
|
||||
for i, result in enumerate(results):
|
||||
search_result = AudioSearchResult(
|
||||
title=getattr(result, "title", title),
|
||||
artist=getattr(result, "artist", artist),
|
||||
provider=getattr(result, "provider", "Unknown"),
|
||||
url=getattr(result, "url", ""),
|
||||
duration=getattr(result, "duration", None),
|
||||
quality=getattr(result, "quality", None),
|
||||
source_id=getattr(result, "id", str(i)),
|
||||
index=i,
|
||||
)
|
||||
search_results.append(search_result)
|
||||
Runs async code in a sync context for compatibility with existing API.
|
||||
"""
|
||||
try:
|
||||
# Enable nested event loops (needed when called from FastAPI async context)
|
||||
# Apply to the current loop if one exists, otherwise it's a no-op
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
nest_asyncio.apply(loop)
|
||||
except RuntimeError:
|
||||
pass # No running loop, nest_asyncio not needed
|
||||
|
||||
# Cache raw result for later download
|
||||
self._raw_results[i] = result
|
||||
|
||||
logger.info(f"Found {len(search_results)} results for: {artist} - {title}")
|
||||
return search_results
|
||||
# Run async search in sync context
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
response = loop.run_until_complete(
|
||||
self._remote_client.search(artist, title)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# Check for empty results
|
||||
if not response.get("results"):
|
||||
raise NoResultsError(f"No results found for: {artist} - {title}")
|
||||
|
||||
# Store search_id for download
|
||||
self._remote_search_id = response.get("search_id")
|
||||
|
||||
# Convert remote results to AudioSearchResult objects
|
||||
results = []
|
||||
for item in response.get("results", []):
|
||||
result = AudioSearchResult(
|
||||
title=item.get("title", "Unknown"),
|
||||
artist=item.get("artist", "Unknown"),
|
||||
url=item.get("download_url", ""), # May be empty for remote
|
||||
provider=item.get("provider", "Unknown"),
|
||||
duration=item.get("duration_seconds"),
|
||||
quality=item.get("quality", ""),
|
||||
source_id=item.get("info_hash"),
|
||||
index=item.get("index", 0),
|
||||
seeders=item.get("seeders"),
|
||||
target_file=item.get("target_file"),
|
||||
# Store full remote data in raw_result for rich display
|
||||
raw_result=item,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
# Cache results
|
||||
self._cached_results = results
|
||||
|
||||
logger.info(f"Found {len(results)} results from remote flacfetch for: {artist} - {title}")
|
||||
return results
|
||||
|
||||
except FlacfetchServiceError as e:
|
||||
logger.warning(f"Remote flacfetch search failed, falling back to local: {e}")
|
||||
# Fallback to local search
|
||||
self._remote_search_id = None
|
||||
results = self._fetcher.search(artist, title)
|
||||
self._cached_results = results
|
||||
return results
|
||||
|
||||
def select_best(self, results: List[AudioSearchResult]) -> int:
|
||||
"""
|
||||
@@ -212,90 +229,142 @@ class AudioSearchService:
|
||||
Returns:
|
||||
Index of the best result
|
||||
"""
|
||||
if not results:
|
||||
return 0
|
||||
|
||||
manager = self._get_manager()
|
||||
|
||||
# Get raw results for the indices we have
|
||||
raw_results = [self._raw_results.get(r.index) for r in results if r.index in self._raw_results]
|
||||
|
||||
if raw_results:
|
||||
try:
|
||||
best = manager.select_best(raw_results)
|
||||
# Find index of best result
|
||||
for i, raw in enumerate(raw_results):
|
||||
if raw == best:
|
||||
return results[i].index
|
||||
except Exception as e:
|
||||
logger.warning(f"select_best failed, using first result: {e}")
|
||||
|
||||
# Fallback: return first result
|
||||
return results[0].index if results else 0
|
||||
return self._fetcher.select_best(results)
|
||||
|
||||
def download(
|
||||
self,
|
||||
result_index: int,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Download audio from a search result.
|
||||
Download audio from a cached search result.
|
||||
|
||||
This method uses the cached results from the last search() call.
|
||||
The API flow is:
|
||||
1. Client calls search() -> gets list of results
|
||||
2. Client picks an index
|
||||
3. Client calls download(index) -> gets downloaded file
|
||||
|
||||
If a remote flacfetch service is configured and the search was performed
|
||||
remotely (for torrent sources), uses the remote service for download.
|
||||
|
||||
Args:
|
||||
result_index: Index of the result to download (from search results)
|
||||
output_dir: Directory to save the downloaded file
|
||||
output_filename: Optional filename (without extension)
|
||||
gcs_path: Optional GCS path for remote uploads (e.g., "uploads/job123/audio/")
|
||||
|
||||
Returns:
|
||||
AudioDownloadResult with the downloaded file path
|
||||
|
||||
Raises:
|
||||
DownloadError: If download fails
|
||||
DownloadError: If download fails or no cached result for index
|
||||
"""
|
||||
if result_index not in self._raw_results:
|
||||
raise DownloadError(f"No cached result for index {result_index}. Run search() first.")
|
||||
if result_index < 0 or result_index >= len(self._cached_results):
|
||||
raise DownloadError(
|
||||
f"No cached result for index {result_index}. "
|
||||
f"Available indices: 0-{len(self._cached_results) - 1}. "
|
||||
"Run search() first."
|
||||
)
|
||||
|
||||
raw_result = self._raw_results[result_index]
|
||||
manager = self._get_manager()
|
||||
result = self._cached_results[result_index]
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
logger.info(f"Downloading: {result.artist} - {result.title} from {result.provider}")
|
||||
|
||||
# Get metadata from raw result
|
||||
artist = getattr(raw_result, "artist", "Unknown")
|
||||
title = getattr(raw_result, "title", "Unknown")
|
||||
provider = getattr(raw_result, "provider", "Unknown")
|
||||
# Check if this was a remote search (torrent sources need remote download)
|
||||
if self._remote_search_id and self._remote_client:
|
||||
# Check if this is a torrent source that needs remote handling
|
||||
is_torrent = result.provider in ["Redacted", "OPS"]
|
||||
|
||||
if is_torrent:
|
||||
return self._download_remote(result_index, output_dir, output_filename, gcs_path)
|
||||
|
||||
# Generate filename if not provided
|
||||
if output_filename is None:
|
||||
output_filename = f"{artist} - {title}"
|
||||
# Delegate to shared FlacFetcher (local download)
|
||||
fetch_result = self._fetcher.download(result, output_dir, output_filename)
|
||||
|
||||
logger.info(f"Downloading: {artist} - {title} from {provider}")
|
||||
logger.info(f"Downloaded to: {fetch_result.filepath}")
|
||||
|
||||
return fetch_result
|
||||
|
||||
def _download_remote(
|
||||
self,
|
||||
result_index: int,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Download using the remote flacfetch service.
|
||||
|
||||
The remote service downloads via torrent and optionally uploads to GCS.
|
||||
"""
|
||||
if not self._remote_search_id:
|
||||
raise DownloadError("No remote search ID - run search() first")
|
||||
|
||||
result = self._cached_results[result_index]
|
||||
|
||||
try:
|
||||
filepath = manager.download(
|
||||
raw_result,
|
||||
output_path=output_dir,
|
||||
output_filename=output_filename,
|
||||
)
|
||||
# Enable nested event loops (needed when called from FastAPI async context)
|
||||
try:
|
||||
running_loop = asyncio.get_running_loop()
|
||||
nest_asyncio.apply(running_loop)
|
||||
except RuntimeError:
|
||||
pass # No running loop, nest_asyncio not needed
|
||||
|
||||
if filepath is None:
|
||||
raise DownloadError(f"Download returned no file path for: {artist} - {title}")
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
# Start download
|
||||
download_id = loop.run_until_complete(
|
||||
self._remote_client.download(
|
||||
search_id=self._remote_search_id,
|
||||
result_index=result_index,
|
||||
output_filename=output_filename,
|
||||
gcs_path=gcs_path,
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"Remote download started: {download_id}")
|
||||
|
||||
# Wait for completion with progress logging
|
||||
def log_progress(status):
|
||||
progress = status.get("progress", 0)
|
||||
speed = status.get("download_speed_kbps", 0)
|
||||
logger.debug(f"Download progress: {progress:.1f}% ({speed:.1f} KB/s)")
|
||||
|
||||
final_status = loop.run_until_complete(
|
||||
self._remote_client.wait_for_download(
|
||||
download_id,
|
||||
timeout=600, # 10 minute timeout
|
||||
progress_callback=log_progress,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
logger.info(f"Downloaded to: {filepath}")
|
||||
# Determine file path
|
||||
filepath = final_status.get("gcs_path") or final_status.get("output_path")
|
||||
|
||||
if not filepath:
|
||||
raise DownloadError("Remote download completed but no file path returned")
|
||||
|
||||
logger.info(f"Remote download complete: {filepath}")
|
||||
|
||||
return AudioDownloadResult(
|
||||
filepath=filepath,
|
||||
artist=artist,
|
||||
title=title,
|
||||
provider=provider,
|
||||
duration=getattr(raw_result, "duration", None),
|
||||
quality=getattr(raw_result, "quality", None),
|
||||
artist=result.artist,
|
||||
title=result.title,
|
||||
provider=result.provider,
|
||||
quality=result.quality,
|
||||
)
|
||||
|
||||
except FlacfetchServiceError as e:
|
||||
raise DownloadError(f"Remote download failed: {e}") from e
|
||||
except Exception as e:
|
||||
raise DownloadError(f"Failed to download {artist} - {title}: {e}") from e
|
||||
logger.error(f"Remote download error: {e}")
|
||||
raise DownloadError(f"Remote download failed: {e}") from e
|
||||
|
||||
def search_and_download_auto(
|
||||
self,
|
||||
@@ -303,6 +372,7 @@ class AudioSearchService:
|
||||
title: str,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Search for audio and automatically download the best result.
|
||||
@@ -315,6 +385,7 @@ class AudioSearchService:
|
||||
title: The track title to search for
|
||||
output_dir: Directory to save the downloaded file
|
||||
output_filename: Optional filename (without extension)
|
||||
gcs_path: Optional GCS path for remote uploads
|
||||
|
||||
Returns:
|
||||
AudioDownloadResult with the downloaded file path
|
||||
@@ -328,10 +399,64 @@ class AudioSearchService:
|
||||
|
||||
# Select best
|
||||
best_index = self.select_best(results)
|
||||
logger.info(f"Auto-selected result {best_index}: {results[best_index].artist} - {results[best_index].title} ({results[best_index].provider})")
|
||||
best_result = results[best_index]
|
||||
logger.info(
|
||||
f"Auto-selected result {best_index}: "
|
||||
f"{best_result.artist} - {best_result.title} ({best_result.provider})"
|
||||
)
|
||||
|
||||
# Download
|
||||
return self.download(best_index, output_dir, output_filename)
|
||||
return self.download(best_index, output_dir, output_filename, gcs_path)
|
||||
|
||||
async def search_async(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""
|
||||
Async version of search for use in async routes.
|
||||
|
||||
Note: Currently wraps sync search in executor. Future optimization
|
||||
could make this fully async.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, self.search, artist, title)
|
||||
|
||||
async def download_async(
|
||||
self,
|
||||
result_index: int,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Async version of download for use in async routes.
|
||||
|
||||
Note: Currently wraps sync download in executor. Future optimization
|
||||
could make this fully async.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self.download(result_index, output_dir, output_filename, gcs_path)
|
||||
)
|
||||
|
||||
def is_remote_enabled(self) -> bool:
|
||||
"""Check if remote flacfetch service is configured."""
|
||||
return self._remote_client is not None
|
||||
|
||||
async def check_remote_health(self) -> Optional[dict]:
|
||||
"""
|
||||
Check health of remote flacfetch service.
|
||||
|
||||
Returns:
|
||||
Health status dict if remote service is configured and healthy,
|
||||
None if not configured or unhealthy.
|
||||
"""
|
||||
if not self._remote_client:
|
||||
return None
|
||||
|
||||
try:
|
||||
return await self._remote_client.health_check()
|
||||
except Exception as e:
|
||||
logger.warning(f"Remote flacfetch health check failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# Singleton instance
|
||||
@@ -344,5 +469,3 @@ def get_audio_search_service() -> AudioSearchService:
|
||||
if _audio_search_service is None:
|
||||
_audio_search_service = AudioSearchService()
|
||||
return _audio_search_service
|
||||
|
||||
|
||||
@@ -68,6 +68,16 @@ class AuthService:
|
||||
logger.info("Admin token validated")
|
||||
return True, UserType.ADMIN, -1, "Admin access granted"
|
||||
|
||||
# Debug: Log token comparison info (only first/last few chars for security)
|
||||
if self.admin_tokens:
|
||||
expected_prefix = self.admin_tokens[0][:8] if len(self.admin_tokens[0]) >= 8 else self.admin_tokens[0]
|
||||
provided_prefix = token[:8] if len(token) >= 8 else token
|
||||
logger.debug(
|
||||
f"Token mismatch: expected prefix '{expected_prefix}...', "
|
||||
f"got '{provided_prefix}...', "
|
||||
f"expected len={len(self.admin_tokens[0])}, got len={len(token)}"
|
||||
)
|
||||
|
||||
# Check stored tokens in Firestore
|
||||
token_data = self.firestore.get_token(token)
|
||||
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
Client for the remote flacfetch HTTP API service.
|
||||
|
||||
This client communicates with a dedicated flacfetch VM that handles:
|
||||
- BitTorrent downloads from private trackers (Redacted, OPS)
|
||||
- YouTube downloads
|
||||
- GCS uploads of downloaded files
|
||||
|
||||
The flacfetch service provides:
|
||||
- Full peer connectivity for torrents (not possible in Cloud Run)
|
||||
- Indefinite seeding of completed torrents
|
||||
- Automatic disk cleanup
|
||||
|
||||
Usage:
|
||||
client = get_flacfetch_client()
|
||||
if client:
|
||||
# Search for audio
|
||||
search_result = await client.search("ABBA", "Waterloo")
|
||||
|
||||
# Download the best result
|
||||
download_id = await client.download(
|
||||
search_id=search_result["search_id"],
|
||||
result_index=0,
|
||||
gcs_path="uploads/job123/audio/",
|
||||
)
|
||||
|
||||
# Wait for completion
|
||||
result = await client.wait_for_download(download_id)
|
||||
print(f"Downloaded to: {result['gcs_path']}")
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FlacfetchServiceError(Exception):
|
||||
"""Error communicating with or returned from flacfetch service."""
|
||||
pass
|
||||
|
||||
|
||||
class FlacfetchClient:
|
||||
"""
|
||||
Client for remote flacfetch HTTP API.
|
||||
|
||||
All endpoints require authentication via X-API-Key header.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
timeout: int = 60,
|
||||
):
|
||||
"""
|
||||
Initialize flacfetch client.
|
||||
|
||||
Args:
|
||||
base_url: Base URL of flacfetch service (e.g., http://10.0.0.5:8080)
|
||||
api_key: API key for authentication
|
||||
timeout: Default request timeout in seconds
|
||||
"""
|
||||
self.base_url = base_url.rstrip('/')
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
logger.info(f"FlacfetchClient initialized with base_url={self.base_url}")
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
"""Get request headers with authentication."""
|
||||
return {
|
||||
"X-API-Key": self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
async def health_check(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Check if flacfetch service is healthy.
|
||||
|
||||
Returns:
|
||||
Health status dict with transmission, disk, and provider info
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: If service is unhealthy or unreachable
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/health",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if data.get("status") not in ["healthy", "degraded"]:
|
||||
raise FlacfetchServiceError(f"Service unhealthy: {data}")
|
||||
|
||||
return data
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Cannot reach flacfetch service: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Health check failed: {e.response.status_code}")
|
||||
|
||||
async def search(
|
||||
self,
|
||||
artist: str,
|
||||
title: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Search for audio matching artist and title.
|
||||
|
||||
Args:
|
||||
artist: Artist name
|
||||
title: Track title
|
||||
|
||||
Returns:
|
||||
Search response dict with search_id and results list
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On search failure
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/search",
|
||||
headers=self._headers(),
|
||||
json={"artist": artist, "title": title},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
if resp.status_code == 404:
|
||||
# No results found - return empty results
|
||||
return {
|
||||
"search_id": None,
|
||||
"artist": artist,
|
||||
"title": title,
|
||||
"results": [],
|
||||
"results_count": 0,
|
||||
}
|
||||
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Search request failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Search failed: {e.response.status_code} - {e.response.text}")
|
||||
|
||||
async def download(
|
||||
self,
|
||||
search_id: str,
|
||||
result_index: int,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Start downloading an audio file.
|
||||
|
||||
Args:
|
||||
search_id: Search ID from previous search
|
||||
result_index: Index of result to download
|
||||
output_filename: Optional custom filename (without extension)
|
||||
gcs_path: GCS path for upload (e.g., "uploads/job123/audio/")
|
||||
|
||||
Returns:
|
||||
Download ID for tracking progress
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On download start failure
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
payload = {
|
||||
"search_id": search_id,
|
||||
"result_index": result_index,
|
||||
}
|
||||
if output_filename:
|
||||
payload["output_filename"] = output_filename
|
||||
if gcs_path:
|
||||
payload["upload_to_gcs"] = True
|
||||
payload["gcs_path"] = gcs_path
|
||||
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/download",
|
||||
headers=self._headers(),
|
||||
json=payload,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
data = resp.json()
|
||||
return data["download_id"]
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Download request failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Download start failed: {e.response.status_code} - {e.response.text}")
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get the current status of a download.
|
||||
|
||||
Args:
|
||||
download_id: Download ID from start_download
|
||||
|
||||
Returns:
|
||||
Status dict with progress, speed, path info
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On status check failure
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/download/{download_id}/status",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Status request failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 404:
|
||||
raise FlacfetchServiceError(f"Download not found: {download_id}")
|
||||
raise FlacfetchServiceError(f"Status check failed: {e.response.status_code}")
|
||||
|
||||
async def wait_for_download(
|
||||
self,
|
||||
download_id: str,
|
||||
timeout: int = 600,
|
||||
poll_interval: float = 2.0,
|
||||
progress_callback: Optional[callable] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Wait for a download to complete.
|
||||
|
||||
Args:
|
||||
download_id: Download ID to wait for
|
||||
timeout: Maximum wait time in seconds
|
||||
poll_interval: Time between status checks
|
||||
progress_callback: Optional callback(status_dict) for progress updates
|
||||
|
||||
Returns:
|
||||
Final status dict with output_path or gcs_path
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On download failure or timeout
|
||||
"""
|
||||
elapsed = 0.0
|
||||
|
||||
while elapsed < timeout:
|
||||
status = await self.get_download_status(download_id)
|
||||
|
||||
if progress_callback:
|
||||
try:
|
||||
progress_callback(status)
|
||||
except Exception as e:
|
||||
logger.warning(f"Progress callback error: {e}")
|
||||
|
||||
download_status = status.get("status")
|
||||
|
||||
if download_status in ["complete", "seeding"]:
|
||||
logger.info(f"Download {download_id} complete: {status.get('gcs_path') or status.get('output_path')}")
|
||||
return status
|
||||
elif download_status == "failed":
|
||||
error = status.get("error", "Unknown error")
|
||||
raise FlacfetchServiceError(f"Download failed: {error}")
|
||||
elif download_status == "cancelled":
|
||||
raise FlacfetchServiceError("Download was cancelled")
|
||||
|
||||
# Log progress
|
||||
progress = status.get("progress", 0)
|
||||
speed = status.get("download_speed_kbps", 0)
|
||||
peers = status.get("peers", 0)
|
||||
logger.debug(
|
||||
f"Download {download_id}: {progress:.1f}% "
|
||||
f"({speed:.1f} KB/s, {peers} peers)"
|
||||
)
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
raise FlacfetchServiceError(f"Download timed out after {timeout}s")
|
||||
|
||||
async def list_torrents(self) -> Dict[str, Any]:
|
||||
"""
|
||||
List all torrents in Transmission.
|
||||
|
||||
Returns:
|
||||
Torrent list response with torrent info and totals
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/torrents",
|
||||
headers=self._headers(),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"List torrents failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"List torrents failed: {e.response.status_code}")
|
||||
|
||||
async def delete_torrent(
|
||||
self,
|
||||
torrent_id: int,
|
||||
delete_data: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete a torrent from Transmission.
|
||||
|
||||
Args:
|
||||
torrent_id: Transmission torrent ID
|
||||
delete_data: Also delete downloaded files
|
||||
|
||||
Returns:
|
||||
Delete response
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.delete(
|
||||
f"{self.base_url}/torrents/{torrent_id}",
|
||||
headers=self._headers(),
|
||||
params={"delete_data": str(delete_data).lower()},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Delete torrent failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Delete torrent failed: {e.response.status_code}")
|
||||
|
||||
async def cleanup_torrents(
|
||||
self,
|
||||
strategy: str = "oldest",
|
||||
target_free_gb: float = 10.0,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Trigger disk cleanup.
|
||||
|
||||
Args:
|
||||
strategy: Cleanup strategy (oldest, largest, lowest_ratio)
|
||||
target_free_gb: Target free space to achieve
|
||||
|
||||
Returns:
|
||||
Cleanup response with removed count and freed space
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/torrents/cleanup",
|
||||
headers=self._headers(),
|
||||
json={
|
||||
"strategy": strategy,
|
||||
"target_free_gb": target_free_gb,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Cleanup failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Cleanup failed: {e.response.status_code}")
|
||||
|
||||
|
||||
# Singleton client instance
|
||||
_client: Optional[FlacfetchClient] = None
|
||||
|
||||
|
||||
def get_flacfetch_client() -> Optional[FlacfetchClient]:
|
||||
"""
|
||||
Get the flacfetch client if configured.
|
||||
|
||||
Returns:
|
||||
FlacfetchClient instance if FLACFETCH_API_URL and FLACFETCH_API_KEY are set,
|
||||
None otherwise (indicating local-only mode).
|
||||
"""
|
||||
global _client
|
||||
|
||||
if _client is None:
|
||||
from backend.config import get_settings
|
||||
settings = get_settings()
|
||||
|
||||
if settings.flacfetch_api_url and settings.flacfetch_api_key:
|
||||
_client = FlacfetchClient(
|
||||
base_url=settings.flacfetch_api_url,
|
||||
api_key=settings.flacfetch_api_key,
|
||||
)
|
||||
logger.info("FlacfetchClient initialized for remote service")
|
||||
else:
|
||||
logger.debug("FlacfetchClient not configured (missing URL or API key)")
|
||||
|
||||
return _client
|
||||
|
||||
|
||||
def reset_flacfetch_client():
|
||||
"""Reset the singleton client (for testing)."""
|
||||
global _client
|
||||
_client = None
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
"""
|
||||
Custom metrics for Cloud Monitoring.
|
||||
|
||||
This module provides application-level metrics for tracking job processing,
|
||||
worker performance, and external API usage. Metrics can be viewed in:
|
||||
1. Cloud Logging (via structured log entries)
|
||||
2. Cloud Monitoring (via log-based metrics or OpenTelemetry)
|
||||
|
||||
The metrics service uses a pragmatic approach:
|
||||
- Always emits metrics as structured log entries (works immediately)
|
||||
- Uses the same JSON format as Cloud Logging
|
||||
- Can be enhanced with OpenTelemetry metrics exporters when available
|
||||
|
||||
Usage:
|
||||
from backend.services.metrics import metrics
|
||||
|
||||
# Record a job completion
|
||||
metrics.record_job_completed("abc123", source="upload")
|
||||
|
||||
# Record worker duration
|
||||
metrics.record_worker_duration("audio", 45.2, success=True)
|
||||
|
||||
# Record external API call
|
||||
with metrics.time_external_api("modal"):
|
||||
response = await modal_client.separate_audio(...)
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from backend.services.tracing import get_current_trace_id, get_current_span_id
|
||||
|
||||
|
||||
logger = logging.getLogger("metrics")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricLabels:
|
||||
"""Common metric labels."""
|
||||
job_id: Optional[str] = None
|
||||
worker: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
api: Optional[str] = None
|
||||
operation: Optional[str] = None
|
||||
bucket: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict, excluding None values."""
|
||||
return {k: v for k, v in self.__dict__.items() if v is not None}
|
||||
|
||||
|
||||
class MetricsService:
|
||||
"""
|
||||
Application metrics service.
|
||||
|
||||
Emits metrics as structured log entries that can be:
|
||||
1. Queried directly in Cloud Logging
|
||||
2. Converted to Cloud Monitoring metrics via log-based metrics
|
||||
3. Exported via OpenTelemetry (when configured)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the metrics service."""
|
||||
self._logger = logging.getLogger("metrics")
|
||||
# Ensure metrics logger outputs at INFO level
|
||||
self._logger.setLevel(logging.INFO)
|
||||
|
||||
def _emit_metric(self, metric_name: str, metric_type: str, value: float, labels: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Emit a metric as a structured log entry.
|
||||
|
||||
The log format is designed to be easily parsed by Cloud Logging
|
||||
and converted to log-based metrics.
|
||||
|
||||
Args:
|
||||
metric_name: Name of the metric (e.g., "jobs_total")
|
||||
metric_type: Type of metric (counter, histogram, gauge)
|
||||
value: Metric value
|
||||
labels: Metric labels/dimensions
|
||||
"""
|
||||
# Add trace context if available
|
||||
trace_id = get_current_trace_id()
|
||||
span_id = get_current_span_id()
|
||||
|
||||
# Build metric entry
|
||||
metric_entry = {
|
||||
"metric_name": metric_name,
|
||||
"metric_type": metric_type,
|
||||
"metric_value": value,
|
||||
**labels,
|
||||
}
|
||||
|
||||
if trace_id:
|
||||
metric_entry["trace_id"] = trace_id
|
||||
if span_id:
|
||||
metric_entry["span_id"] = span_id
|
||||
|
||||
# Emit as structured log entry
|
||||
# Use INFO level so metrics always show up
|
||||
self._logger.info(
|
||||
f"METRIC {metric_name}={value}",
|
||||
extra=metric_entry
|
||||
)
|
||||
|
||||
# =========================================
|
||||
# Job Metrics
|
||||
# =========================================
|
||||
|
||||
def record_job_created(self, job_id: str, source: str = "unknown") -> None:
|
||||
"""
|
||||
Record a new job creation.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
source: Job source (upload, url, search)
|
||||
"""
|
||||
self._emit_metric(
|
||||
metric_name="jobs_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels={"job_id": job_id, "status": "created", "source": source}
|
||||
)
|
||||
|
||||
def record_job_completed(self, job_id: str, source: str = "unknown") -> None:
|
||||
"""
|
||||
Record a job completion.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
source: Job source (upload, url, search)
|
||||
"""
|
||||
self._emit_metric(
|
||||
metric_name="jobs_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels={"job_id": job_id, "status": "completed", "source": source}
|
||||
)
|
||||
|
||||
def record_job_failed(self, job_id: str, source: str = "unknown", error: Optional[str] = None) -> None:
|
||||
"""
|
||||
Record a job failure.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
source: Job source (upload, url, search)
|
||||
error: Optional error message
|
||||
"""
|
||||
labels = {"job_id": job_id, "status": "failed", "source": source}
|
||||
if error:
|
||||
labels["error"] = error[:200] # Truncate long errors
|
||||
self._emit_metric(
|
||||
metric_name="jobs_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels=labels
|
||||
)
|
||||
|
||||
def record_job_duration(self, job_id: str, duration_seconds: float, source: str = "unknown") -> None:
|
||||
"""
|
||||
Record total job processing duration.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
duration_seconds: Total processing time in seconds
|
||||
source: Job source (upload, url, search)
|
||||
"""
|
||||
self._emit_metric(
|
||||
metric_name="job_duration_seconds",
|
||||
metric_type="histogram",
|
||||
value=duration_seconds,
|
||||
labels={"job_id": job_id, "source": source}
|
||||
)
|
||||
|
||||
# =========================================
|
||||
# Worker Metrics
|
||||
# =========================================
|
||||
|
||||
def record_worker_started(self, worker: str, job_id: str) -> None:
|
||||
"""
|
||||
Record a worker invocation start.
|
||||
|
||||
Args:
|
||||
worker: Worker name (audio, lyrics, screens, video, render_video)
|
||||
job_id: Job ID
|
||||
"""
|
||||
self._emit_metric(
|
||||
metric_name="worker_invocations_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels={"worker": worker, "job_id": job_id, "status": "started"}
|
||||
)
|
||||
|
||||
def record_worker_duration(self, worker: str, duration_seconds: float, success: bool, job_id: Optional[str] = None) -> None:
|
||||
"""
|
||||
Record worker processing duration.
|
||||
|
||||
Args:
|
||||
worker: Worker name
|
||||
duration_seconds: Processing time in seconds
|
||||
success: Whether worker completed successfully
|
||||
job_id: Optional job ID
|
||||
"""
|
||||
labels = {
|
||||
"worker": worker,
|
||||
"success": str(success).lower(),
|
||||
}
|
||||
if job_id:
|
||||
labels["job_id"] = job_id
|
||||
|
||||
self._emit_metric(
|
||||
metric_name="job_stage_duration_seconds",
|
||||
metric_type="histogram",
|
||||
value=duration_seconds,
|
||||
labels=labels
|
||||
)
|
||||
|
||||
# Also emit a counter for success/failure tracking
|
||||
self._emit_metric(
|
||||
metric_name="worker_invocations_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels={"worker": worker, "success": str(success).lower(), "job_id": job_id or "unknown"}
|
||||
)
|
||||
|
||||
# =========================================
|
||||
# GCS Metrics
|
||||
# =========================================
|
||||
|
||||
def record_gcs_operation(
|
||||
self,
|
||||
operation: str,
|
||||
bucket: str,
|
||||
success: bool,
|
||||
size_bytes: Optional[int] = None,
|
||||
duration_seconds: Optional[float] = None,
|
||||
job_id: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record a GCS operation.
|
||||
|
||||
Args:
|
||||
operation: Operation type (upload, download, delete)
|
||||
bucket: GCS bucket name
|
||||
success: Whether operation succeeded
|
||||
size_bytes: Optional file size in bytes
|
||||
duration_seconds: Optional operation duration
|
||||
job_id: Optional job ID
|
||||
"""
|
||||
labels = {
|
||||
"operation": operation,
|
||||
"bucket": bucket,
|
||||
"success": str(success).lower(),
|
||||
}
|
||||
if job_id:
|
||||
labels["job_id"] = job_id
|
||||
|
||||
self._emit_metric(
|
||||
metric_name="gcs_operations_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels=labels
|
||||
)
|
||||
|
||||
if size_bytes is not None:
|
||||
self._emit_metric(
|
||||
metric_name="gcs_operation_bytes",
|
||||
metric_type="histogram",
|
||||
value=size_bytes,
|
||||
labels={**labels, "operation": operation}
|
||||
)
|
||||
|
||||
if duration_seconds is not None:
|
||||
self._emit_metric(
|
||||
metric_name="gcs_operation_duration_seconds",
|
||||
metric_type="histogram",
|
||||
value=duration_seconds,
|
||||
labels={**labels, "operation": operation}
|
||||
)
|
||||
|
||||
# =========================================
|
||||
# External API Metrics
|
||||
# =========================================
|
||||
|
||||
def record_external_api_call(
|
||||
self,
|
||||
api: str,
|
||||
success: bool,
|
||||
duration_seconds: float,
|
||||
job_id: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record an external API call.
|
||||
|
||||
Args:
|
||||
api: API name (modal, audioshake, genius, spotify)
|
||||
success: Whether call succeeded
|
||||
duration_seconds: API call duration
|
||||
job_id: Optional job ID
|
||||
error: Optional error message
|
||||
"""
|
||||
labels = {
|
||||
"api": api,
|
||||
"success": str(success).lower(),
|
||||
}
|
||||
if job_id:
|
||||
labels["job_id"] = job_id
|
||||
if error:
|
||||
labels["error"] = error[:100] # Truncate
|
||||
|
||||
self._emit_metric(
|
||||
metric_name="external_api_calls_total",
|
||||
metric_type="counter",
|
||||
value=1,
|
||||
labels=labels
|
||||
)
|
||||
|
||||
self._emit_metric(
|
||||
metric_name="external_api_duration_seconds",
|
||||
metric_type="histogram",
|
||||
value=duration_seconds,
|
||||
labels={"api": api, "success": str(success).lower()}
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def time_external_api(self, api: str, job_id: Optional[str] = None):
|
||||
"""
|
||||
Context manager to time an external API call.
|
||||
|
||||
Usage:
|
||||
with metrics.time_external_api("modal", job_id) as timer:
|
||||
response = await client.call_api()
|
||||
timer.set_success(True)
|
||||
|
||||
Args:
|
||||
api: API name
|
||||
job_id: Optional job ID
|
||||
|
||||
Yields:
|
||||
Timer object with set_success() method
|
||||
"""
|
||||
timer = _ApiTimer()
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
yield timer
|
||||
except Exception as e:
|
||||
timer.set_success(False)
|
||||
timer.error = str(e)
|
||||
raise
|
||||
finally:
|
||||
duration = time.time() - start_time
|
||||
self.record_external_api_call(
|
||||
api=api,
|
||||
success=timer.success,
|
||||
duration_seconds=duration,
|
||||
job_id=job_id,
|
||||
error=timer.error,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def time_worker(self, worker: str, job_id: str):
|
||||
"""
|
||||
Context manager to time a worker execution.
|
||||
|
||||
Usage:
|
||||
with metrics.time_worker("audio", job_id) as timer:
|
||||
await process_audio()
|
||||
timer.set_success(True)
|
||||
|
||||
Args:
|
||||
worker: Worker name
|
||||
job_id: Job ID
|
||||
|
||||
Yields:
|
||||
Timer object with set_success() method
|
||||
"""
|
||||
timer = _ApiTimer()
|
||||
self.record_worker_started(worker, job_id)
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
yield timer
|
||||
except Exception as e:
|
||||
timer.set_success(False)
|
||||
raise
|
||||
finally:
|
||||
duration = time.time() - start_time
|
||||
self.record_worker_duration(
|
||||
worker=worker,
|
||||
duration_seconds=duration,
|
||||
success=timer.success,
|
||||
job_id=job_id,
|
||||
)
|
||||
|
||||
|
||||
class _ApiTimer:
|
||||
"""Helper class for tracking API call success state."""
|
||||
|
||||
def __init__(self):
|
||||
self.success = True # Assume success unless set otherwise
|
||||
self.error: Optional[str] = None
|
||||
|
||||
def set_success(self, success: bool) -> None:
|
||||
self.success = success
|
||||
|
||||
|
||||
# Global metrics instance
|
||||
metrics = MetricsService()
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
"""
|
||||
Structured logging with trace correlation for Cloud Logging.
|
||||
|
||||
This module provides JSON-formatted logging that integrates with Google Cloud Logging
|
||||
and correlates logs with OpenTelemetry traces in Cloud Trace.
|
||||
|
||||
When running in Cloud Run:
|
||||
- Logs are output as JSON for Cloud Logging to parse
|
||||
- Each log entry includes trace ID and span ID for correlation
|
||||
- Custom fields (job_id, worker) are preserved in the log structure
|
||||
|
||||
Usage:
|
||||
# In main.py, before any logging:
|
||||
from backend.services.structured_logging import setup_structured_logging
|
||||
setup_structured_logging()
|
||||
|
||||
# Then use standard logging:
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info("Processing started", extra={"job_id": "abc123"})
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
# Cloud Logging severity mapping (compatible with GCP)
|
||||
# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
|
||||
SEVERITY_MAP = {
|
||||
"DEBUG": "DEBUG",
|
||||
"INFO": "INFO",
|
||||
"WARNING": "WARNING",
|
||||
"ERROR": "ERROR",
|
||||
"CRITICAL": "CRITICAL",
|
||||
}
|
||||
|
||||
|
||||
class StructuredFormatter(logging.Formatter):
|
||||
"""
|
||||
JSON formatter with trace correlation for Google Cloud Logging.
|
||||
|
||||
Produces log entries in the format expected by Cloud Logging, including:
|
||||
- logging.googleapis.com/trace: Links to Cloud Trace
|
||||
- logging.googleapis.com/spanId: Current span ID
|
||||
- severity: Cloud Logging severity level
|
||||
- Custom fields passed via 'extra' dict
|
||||
"""
|
||||
|
||||
def __init__(self, project_id: Optional[str] = None):
|
||||
"""
|
||||
Initialize the structured formatter.
|
||||
|
||||
Args:
|
||||
project_id: GCP project ID for trace URLs (auto-detected if not provided)
|
||||
"""
|
||||
super().__init__()
|
||||
self.project_id = project_id or os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCP_PROJECT")
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
"""
|
||||
Format a log record as JSON.
|
||||
|
||||
Args:
|
||||
record: The log record to format
|
||||
|
||||
Returns:
|
||||
JSON string for Cloud Logging
|
||||
"""
|
||||
from backend.services.tracing import get_current_trace_id, get_current_span_id
|
||||
|
||||
# Build base log entry
|
||||
log_entry: Dict[str, Any] = {
|
||||
"timestamp": datetime.utcfromtimestamp(record.created).isoformat() + "Z",
|
||||
"severity": SEVERITY_MAP.get(record.levelname, record.levelname),
|
||||
"message": record.getMessage(),
|
||||
"logger": record.name,
|
||||
}
|
||||
|
||||
# Add trace correlation if available
|
||||
trace_id = get_current_trace_id()
|
||||
span_id = get_current_span_id()
|
||||
|
||||
if trace_id and self.project_id:
|
||||
# Format for Cloud Logging trace correlation
|
||||
log_entry["logging.googleapis.com/trace"] = f"projects/{self.project_id}/traces/{trace_id}"
|
||||
|
||||
if span_id:
|
||||
log_entry["logging.googleapis.com/spanId"] = span_id
|
||||
|
||||
# Add custom fields from 'extra' dict
|
||||
# Common fields we want to extract from log records
|
||||
custom_fields = ["job_id", "worker", "operation", "duration", "status", "error"]
|
||||
for field in custom_fields:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None:
|
||||
log_entry[field] = value
|
||||
|
||||
# Add source location for debugging
|
||||
if record.pathname and record.lineno:
|
||||
log_entry["logging.googleapis.com/sourceLocation"] = {
|
||||
"file": record.pathname,
|
||||
"line": record.lineno,
|
||||
"function": record.funcName,
|
||||
}
|
||||
|
||||
# Add exception info if present
|
||||
if record.exc_info:
|
||||
log_entry["exception"] = self.formatException(record.exc_info)
|
||||
|
||||
# Remove None values for cleaner output
|
||||
return json.dumps({k: v for k, v in log_entry.items() if v is not None})
|
||||
|
||||
|
||||
class HumanReadableFormatter(logging.Formatter):
|
||||
"""
|
||||
Human-readable formatter for local development.
|
||||
|
||||
Includes trace context when available but outputs in traditional format.
|
||||
"""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
"""Format a log record for human readability."""
|
||||
from backend.services.tracing import get_current_trace_id
|
||||
|
||||
# Build base message
|
||||
timestamp = datetime.utcfromtimestamp(record.created).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# Add job_id prefix if available
|
||||
job_id = getattr(record, "job_id", None)
|
||||
job_prefix = f"[job:{job_id[:8]}] " if job_id else ""
|
||||
|
||||
# Add trace ID suffix if available
|
||||
trace_id = get_current_trace_id()
|
||||
trace_suffix = f" [trace:{trace_id[:8]}]" if trace_id else ""
|
||||
|
||||
message = f"{timestamp} - {record.name} - {record.levelname} - {job_prefix}{record.getMessage()}{trace_suffix}"
|
||||
|
||||
# Add exception info if present
|
||||
if record.exc_info:
|
||||
message += "\n" + self.formatException(record.exc_info)
|
||||
|
||||
return message
|
||||
|
||||
|
||||
def is_cloud_run() -> bool:
|
||||
"""Check if we're running in Cloud Run."""
|
||||
return os.environ.get("K_SERVICE") is not None
|
||||
|
||||
|
||||
def setup_structured_logging(force_json: bool = False, log_level: Optional[str] = None) -> None:
|
||||
"""
|
||||
Configure structured logging for the application.
|
||||
|
||||
In Cloud Run: Uses JSON format with trace correlation for Cloud Logging
|
||||
In development: Uses human-readable format with optional trace context
|
||||
|
||||
Args:
|
||||
force_json: Force JSON output even in development
|
||||
log_level: Override log level (default from settings or INFO)
|
||||
"""
|
||||
from backend.config import settings
|
||||
|
||||
# Determine log level
|
||||
level = log_level or getattr(settings, "log_level", "INFO")
|
||||
level = getattr(logging, level.upper(), logging.INFO)
|
||||
|
||||
# Choose formatter based on environment
|
||||
if is_cloud_run() or force_json:
|
||||
formatter = StructuredFormatter()
|
||||
else:
|
||||
formatter = HumanReadableFormatter()
|
||||
|
||||
# Create handler
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(formatter)
|
||||
handler.setLevel(level)
|
||||
|
||||
# Configure root logger
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.setLevel(level)
|
||||
|
||||
# Remove existing handlers and add our handler
|
||||
root_logger.handlers = [handler]
|
||||
|
||||
# Also configure uvicorn loggers to use our format
|
||||
for logger_name in ["uvicorn", "uvicorn.access", "uvicorn.error"]:
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.handlers = [handler]
|
||||
logger.propagate = False
|
||||
|
||||
|
||||
class JobLogAdapter(logging.LoggerAdapter):
|
||||
"""
|
||||
Logger adapter that automatically adds job context to log records.
|
||||
|
||||
Usage:
|
||||
logger = logging.getLogger(__name__)
|
||||
job_logger = JobLogAdapter(logger, job_id="abc123", worker="audio")
|
||||
job_logger.info("Processing started") # Automatically includes job_id and worker
|
||||
"""
|
||||
|
||||
def __init__(self, logger: logging.Logger, job_id: str, worker: Optional[str] = None, **extra):
|
||||
"""
|
||||
Initialize the adapter.
|
||||
|
||||
Args:
|
||||
logger: Base logger instance
|
||||
job_id: Job ID to include in all log records
|
||||
worker: Optional worker name
|
||||
**extra: Additional fields to include in all log records
|
||||
"""
|
||||
super().__init__(logger, {"job_id": job_id, "worker": worker, **extra})
|
||||
|
||||
def process(self, msg: str, kwargs: Dict[str, Any]) -> tuple:
|
||||
"""Process a logging call to add extra context."""
|
||||
# Merge extra dict from adapter with any extra passed to the log call
|
||||
extra = kwargs.get("extra", {})
|
||||
extra.update(self.extra)
|
||||
kwargs["extra"] = extra
|
||||
return msg, kwargs
|
||||
|
||||
|
||||
def get_job_logger(job_id: str, worker: Optional[str] = None, name: Optional[str] = None) -> JobLogAdapter:
|
||||
"""
|
||||
Get a logger adapter configured for a specific job.
|
||||
|
||||
This is a convenience function for creating job-specific loggers.
|
||||
|
||||
Args:
|
||||
job_id: Job ID to include in all log records
|
||||
worker: Optional worker name
|
||||
name: Logger name (defaults to "job")
|
||||
|
||||
Returns:
|
||||
JobLogAdapter configured for the job
|
||||
|
||||
Usage:
|
||||
logger = get_job_logger("abc123", "audio")
|
||||
logger.info("Starting audio separation")
|
||||
logger.error("Failed to process", extra={"error": str(e)})
|
||||
"""
|
||||
base_logger = logging.getLogger(name or "job")
|
||||
return JobLogAdapter(base_logger, job_id=job_id, worker=worker)
|
||||
|
||||
@@ -19,6 +19,25 @@ Usage:
|
||||
with tracer.start_as_current_span("my-operation") as span:
|
||||
span.set_attribute("job_id", job_id)
|
||||
# ... do work ...
|
||||
|
||||
# For job-specific spans with automatic job_id attribute
|
||||
from backend.services.tracing import job_span
|
||||
|
||||
with job_span("audio-worker", job_id) as span:
|
||||
# job_id is automatically set as an attribute
|
||||
span.set_attribute("stage", "separation")
|
||||
# ... do work ...
|
||||
|
||||
# For propagating trace context through Cloud Tasks
|
||||
from backend.services.tracing import inject_trace_context, extract_trace_context
|
||||
|
||||
# In worker_service.py (when creating task):
|
||||
headers = inject_trace_context({})
|
||||
|
||||
# In internal.py (when receiving task):
|
||||
context = extract_trace_context(request.headers)
|
||||
with tracer.start_as_current_span("worker", context=context):
|
||||
...
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
@@ -380,3 +399,145 @@ def get_current_span_id() -> Optional[str]:
|
||||
if ctx.is_valid:
|
||||
return format(ctx.span_id, '016x')
|
||||
return None
|
||||
|
||||
|
||||
# =====================================================
|
||||
# Job-Aware Tracing Utilities
|
||||
# =====================================================
|
||||
|
||||
@contextmanager
|
||||
def job_span(
|
||||
name: str,
|
||||
job_id: str,
|
||||
attributes: Optional[Dict[str, Any]] = None,
|
||||
kind: trace.SpanKind = trace.SpanKind.INTERNAL,
|
||||
):
|
||||
"""
|
||||
Create a traced span with job_id automatically set as an attribute.
|
||||
|
||||
This is the preferred way to create spans in job processing workers.
|
||||
The job_id attribute enables filtering spans by job in Cloud Trace.
|
||||
|
||||
Usage:
|
||||
with job_span("audio-separation", job_id) as span:
|
||||
span.set_attribute("stage", "clean_instrumental")
|
||||
# ... do work ...
|
||||
|
||||
Args:
|
||||
name: Span name (appears in Cloud Trace)
|
||||
job_id: Job ID to attach to the span
|
||||
attributes: Additional span attributes
|
||||
kind: Span kind (INTERNAL, SERVER, CLIENT, PRODUCER, CONSUMER)
|
||||
|
||||
Yields:
|
||||
The active span
|
||||
"""
|
||||
all_attributes = {
|
||||
"job_id": job_id,
|
||||
"service.operation": name,
|
||||
}
|
||||
if attributes:
|
||||
all_attributes.update(attributes)
|
||||
|
||||
with create_span(name, all_attributes, kind) as span:
|
||||
yield span
|
||||
|
||||
|
||||
def inject_trace_context(headers: Dict[str, str]) -> Dict[str, str]:
|
||||
"""
|
||||
Inject current trace context into HTTP headers.
|
||||
|
||||
Use this when creating Cloud Tasks or making HTTP calls to propagate
|
||||
the trace context so child spans link to the parent trace.
|
||||
|
||||
Usage:
|
||||
headers = inject_trace_context({
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer token",
|
||||
})
|
||||
# Use headers in HTTP request or Cloud Task
|
||||
|
||||
Args:
|
||||
headers: Existing headers dict to add trace context to
|
||||
|
||||
Returns:
|
||||
Headers dict with traceparent/tracestate headers added
|
||||
"""
|
||||
propagator = TraceContextTextMapPropagator()
|
||||
carrier = dict(headers) # Make a copy
|
||||
propagator.inject(carrier)
|
||||
return carrier
|
||||
|
||||
|
||||
def extract_trace_context(headers: Dict[str, str]) -> Optional[trace.Context]:
|
||||
"""
|
||||
Extract trace context from HTTP headers.
|
||||
|
||||
Use this when receiving Cloud Tasks or HTTP requests to continue
|
||||
the trace from the parent span.
|
||||
|
||||
Usage:
|
||||
context = extract_trace_context(request.headers)
|
||||
with tracer.start_as_current_span("worker", context=context):
|
||||
# ... processing ...
|
||||
|
||||
Args:
|
||||
headers: HTTP headers containing traceparent/tracestate
|
||||
|
||||
Returns:
|
||||
Context object for use with start_as_current_span, or None
|
||||
"""
|
||||
propagator = TraceContextTextMapPropagator()
|
||||
return propagator.extract(headers)
|
||||
|
||||
|
||||
def start_span_with_context(
|
||||
name: str,
|
||||
context: Optional[trace.Context],
|
||||
job_id: Optional[str] = None,
|
||||
attributes: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Start a span as child of the given context (or current context if None).
|
||||
|
||||
This is useful for starting worker spans that should link to the
|
||||
trace from the original API request.
|
||||
|
||||
Usage:
|
||||
context = extract_trace_context(request.headers)
|
||||
with start_span_with_context("audio-worker", context, job_id) as span:
|
||||
# ... processing ...
|
||||
|
||||
Args:
|
||||
name: Span name
|
||||
context: Parent context (from extract_trace_context) or None
|
||||
job_id: Optional job ID to set as attribute
|
||||
attributes: Additional span attributes
|
||||
|
||||
Returns:
|
||||
Context manager yielding the span
|
||||
"""
|
||||
tracer = get_tracer()
|
||||
|
||||
all_attributes = attributes or {}
|
||||
if job_id:
|
||||
all_attributes["job_id"] = job_id
|
||||
all_attributes["service.operation"] = name
|
||||
|
||||
@contextmanager
|
||||
def _span():
|
||||
kwargs = {"kind": trace.SpanKind.INTERNAL}
|
||||
if context is not None:
|
||||
kwargs["context"] = context
|
||||
|
||||
with tracer.start_as_current_span(name, **kwargs) as span:
|
||||
for key, value in all_attributes.items():
|
||||
span.set_attribute(key, value)
|
||||
try:
|
||||
yield span
|
||||
except Exception as e:
|
||||
span.set_status(Status(StatusCode.ERROR, str(e)))
|
||||
span.record_exception(e)
|
||||
raise
|
||||
|
||||
return _span()
|
||||
@@ -10,6 +10,10 @@ SOLID Principles:
|
||||
- Single Responsibility: Only handles worker coordination
|
||||
- Dependency Inversion: Depends on HTTP abstraction, not implementation
|
||||
- Open/Closed: Can add new workers without modifying existing code
|
||||
|
||||
Observability:
|
||||
- Propagates trace context through Cloud Tasks for distributed tracing
|
||||
- All worker invocations create spans linked to original request trace
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
@@ -18,6 +22,7 @@ from typing import Optional
|
||||
import httpx
|
||||
|
||||
from backend.config import get_settings
|
||||
from backend.services.tracing import inject_trace_context
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -180,6 +185,9 @@ class WorkerService:
|
||||
- Deduplication (via task name if needed)
|
||||
- OIDC authentication for Cloud Run
|
||||
|
||||
Observability:
|
||||
- Injects trace context headers so worker spans link to parent trace
|
||||
|
||||
Args:
|
||||
worker_type: Worker type
|
||||
job_id: Job ID to process
|
||||
@@ -205,14 +213,30 @@ class WorkerService:
|
||||
# Build queue path
|
||||
parent = self.tasks_client.queue_path(project, location, queue_name)
|
||||
|
||||
# Build base headers
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Add admin auth header if available
|
||||
# This is needed for the internal endpoint's require_admin dependency
|
||||
if self._admin_token:
|
||||
headers["Authorization"] = f"Bearer {self._admin_token}"
|
||||
logger.debug(
|
||||
f"[job:{job_id}] Using admin token for Cloud Task auth, "
|
||||
f"token prefix: {self._admin_token[:8]}..., len={len(self._admin_token)}"
|
||||
)
|
||||
|
||||
# Inject trace context for distributed tracing
|
||||
# This allows worker spans to link back to the original request trace
|
||||
headers = inject_trace_context(headers)
|
||||
|
||||
# Build task payload
|
||||
task = {
|
||||
"http_request": {
|
||||
"http_method": tasks_v2.HttpMethod.POST,
|
||||
"url": f"{self._base_url}/api/internal/workers/{worker_type}",
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
"headers": headers,
|
||||
"body": json.dumps({"job_id": job_id}).encode(),
|
||||
# Use OIDC token for Cloud Run authentication
|
||||
# This allows Cloud Tasks to invoke the Cloud Run service
|
||||
@@ -222,23 +246,18 @@ class WorkerService:
|
||||
},
|
||||
}
|
||||
|
||||
# Add admin auth header if available
|
||||
# This is needed for the internal endpoint's require_admin dependency
|
||||
if self._admin_token:
|
||||
task["http_request"]["headers"]["Authorization"] = f"Bearer {self._admin_token}"
|
||||
|
||||
# Create task
|
||||
# Note: We don't set a task name, allowing Cloud Tasks to generate one
|
||||
# This prevents duplicate task errors on retries
|
||||
response = self.tasks_client.create_task(parent=parent, task=task)
|
||||
logger.info(
|
||||
f"Created Cloud Task for {worker_type} worker, job {job_id}: {response.name}"
|
||||
f"[job:{job_id}] Created Cloud Task for {worker_type} worker: {response.name}"
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to enqueue Cloud Task for {worker_type}/{job_id}: {e}",
|
||||
f"[job:{job_id}] Failed to enqueue Cloud Task for {worker_type}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
return False
|
||||
@@ -255,6 +274,9 @@ class WorkerService:
|
||||
This is the original implementation - direct HTTP POST to internal endpoint.
|
||||
The endpoint adds the worker function to FastAPI BackgroundTasks.
|
||||
|
||||
Observability:
|
||||
- Injects trace context headers so worker spans link to parent trace
|
||||
|
||||
Args:
|
||||
worker_type: Worker type
|
||||
job_id: Job ID to process
|
||||
@@ -269,6 +291,9 @@ class WorkerService:
|
||||
if self._admin_token:
|
||||
headers["Authorization"] = f"Bearer {self._admin_token}"
|
||||
|
||||
# Inject trace context for distributed tracing
|
||||
headers = inject_trace_context(headers)
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
|
||||
url = f"{self._base_url}/api/internal/workers/{worker_type}"
|
||||
|
||||
@@ -279,22 +304,22 @@ class WorkerService:
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info(f"Successfully triggered {worker_type} worker for job {job_id}")
|
||||
logger.info(f"[job:{job_id}] Successfully triggered {worker_type} worker")
|
||||
return True
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to trigger {worker_type} worker for job {job_id}: "
|
||||
f"[job:{job_id}] Failed to trigger {worker_type} worker: "
|
||||
f"HTTP {response.status_code} - {response.text}"
|
||||
)
|
||||
return False
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.error(f"Timeout triggering {worker_type} worker for job {job_id}")
|
||||
logger.error(f"[job:{job_id}] Timeout triggering {worker_type} worker")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error triggering {worker_type} worker for job {job_id}: {e}",
|
||||
f"[job:{job_id}] Error triggering {worker_type} worker: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
Tests for audio_analysis_service.py - Audio analysis for backing vocals.
|
||||
|
||||
These tests mock the storage service and shared karaoke_gen modules to verify:
|
||||
- GCS file download/upload operations
|
||||
- Audio analysis delegation to shared AudioAnalyzer
|
||||
- Waveform generation delegation to shared WaveformGenerator
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch, ANY
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
|
||||
class TestAudioAnalysisServiceInit:
|
||||
"""Test AudioAnalysisService initialization."""
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_init_creates_dependencies(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test initialization creates storage service and analyzers."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
service = AudioAnalysisService()
|
||||
|
||||
mock_storage_class.assert_called_once()
|
||||
mock_analyzer_class.assert_called_once_with(
|
||||
silence_threshold_db=-40.0,
|
||||
min_segment_duration_ms=100,
|
||||
)
|
||||
mock_waveform_class.assert_called_once()
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_init_with_custom_params(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test initialization with custom threshold and duration."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
service = AudioAnalysisService(
|
||||
silence_threshold_db=-50.0,
|
||||
min_segment_duration_ms=200,
|
||||
)
|
||||
|
||||
mock_analyzer_class.assert_called_once_with(
|
||||
silence_threshold_db=-50.0,
|
||||
min_segment_duration_ms=200,
|
||||
)
|
||||
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_init_with_provided_storage_service(
|
||||
self, mock_waveform_class, mock_analyzer_class
|
||||
):
|
||||
"""Test initialization with externally provided storage service."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
mock_storage = Mock()
|
||||
service = AudioAnalysisService(storage_service=mock_storage)
|
||||
|
||||
assert service.storage_service is mock_storage
|
||||
|
||||
|
||||
class TestAnalyzeBackingVocals:
|
||||
"""Test analyze_backing_vocals method."""
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_analyze_downloads_and_analyzes(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test analyze_backing_vocals downloads file and runs analysis."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
# Setup mocks
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.has_audible_content = True
|
||||
mock_result.segment_count = 5
|
||||
mock_result.recommended_selection = Mock(value="with_backing")
|
||||
|
||||
mock_analyzer = Mock()
|
||||
mock_analyzer.analyze.return_value = mock_result
|
||||
mock_analyzer_class.return_value = mock_analyzer
|
||||
|
||||
service = AudioAnalysisService()
|
||||
|
||||
# Call method
|
||||
result = service.analyze_backing_vocals(
|
||||
gcs_audio_path="uploads/job123/backing_vocals.flac",
|
||||
job_id="job123",
|
||||
)
|
||||
|
||||
# Verify storage download was called
|
||||
mock_storage.download_file.assert_called_once()
|
||||
call_args = mock_storage.download_file.call_args
|
||||
assert call_args[0][0] == "uploads/job123/backing_vocals.flac"
|
||||
|
||||
# Verify analyzer was called
|
||||
mock_analyzer.analyze.assert_called_once()
|
||||
|
||||
# Verify result
|
||||
assert result is mock_result
|
||||
assert result.has_audible_content is True
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_analyze_cleans_up_temp_files(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test that temp files are cleaned up after analysis."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.has_audible_content = False
|
||||
mock_result.segment_count = 0
|
||||
mock_result.recommended_selection = Mock(value="clean")
|
||||
|
||||
mock_analyzer = Mock()
|
||||
mock_analyzer.analyze.return_value = mock_result
|
||||
mock_analyzer_class.return_value = mock_analyzer
|
||||
|
||||
service = AudioAnalysisService()
|
||||
|
||||
# Call method
|
||||
service.analyze_backing_vocals(
|
||||
gcs_audio_path="uploads/job123/backing.flac",
|
||||
job_id="job123",
|
||||
)
|
||||
|
||||
# Temp directory should be cleaned up (no assertion needed,
|
||||
# TemporaryDirectory context manager handles this)
|
||||
# Just verify the method completes without error
|
||||
|
||||
|
||||
class TestAnalyzeAndGenerateWaveform:
|
||||
"""Test analyze_and_generate_waveform method."""
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_analyze_and_generate_waveform(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test analyze_and_generate_waveform does both operations."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.has_audible_content = True
|
||||
mock_result.audible_segments = [Mock(), Mock()]
|
||||
|
||||
mock_analyzer = Mock()
|
||||
mock_analyzer.analyze.return_value = mock_result
|
||||
mock_analyzer.silence_threshold_db = -40.0
|
||||
mock_analyzer_class.return_value = mock_analyzer
|
||||
|
||||
mock_waveform = Mock()
|
||||
mock_waveform_class.return_value = mock_waveform
|
||||
|
||||
service = AudioAnalysisService()
|
||||
|
||||
# Call method
|
||||
result, waveform_path = service.analyze_and_generate_waveform(
|
||||
gcs_audio_path="uploads/job123/backing.flac",
|
||||
job_id="job123",
|
||||
gcs_waveform_destination="uploads/job123/waveform.png",
|
||||
)
|
||||
|
||||
# Verify analysis was performed
|
||||
mock_analyzer.analyze.assert_called_once()
|
||||
|
||||
# Verify waveform was generated
|
||||
mock_waveform.generate.assert_called_once()
|
||||
call_kwargs = mock_waveform.generate.call_args.kwargs
|
||||
assert call_kwargs["segments"] == mock_result.audible_segments
|
||||
assert call_kwargs["show_time_axis"] is True
|
||||
|
||||
# Verify waveform was uploaded
|
||||
assert mock_storage.upload_file.call_count == 1
|
||||
|
||||
# Verify return values
|
||||
assert result is mock_result
|
||||
assert waveform_path == "uploads/job123/waveform.png"
|
||||
|
||||
|
||||
class TestGetWaveformData:
|
||||
"""Test get_waveform_data method."""
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_get_waveform_data(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test get_waveform_data returns amplitude data."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_waveform = Mock()
|
||||
mock_waveform.generate_data_only.return_value = (
|
||||
[0.1, 0.3, 0.5, 0.3, 0.1], # amplitudes
|
||||
180.5, # duration
|
||||
)
|
||||
mock_waveform_class.return_value = mock_waveform
|
||||
|
||||
service = AudioAnalysisService()
|
||||
|
||||
amplitudes, duration = service.get_waveform_data(
|
||||
gcs_audio_path="uploads/job123/backing.flac",
|
||||
job_id="job123",
|
||||
num_points=500,
|
||||
)
|
||||
|
||||
# Verify storage download
|
||||
mock_storage.download_file.assert_called_once()
|
||||
|
||||
# Verify waveform generator called
|
||||
mock_waveform.generate_data_only.assert_called_once()
|
||||
call_kwargs = mock_waveform.generate_data_only.call_args.kwargs
|
||||
assert call_kwargs["num_points"] == 500
|
||||
|
||||
# Verify return values
|
||||
assert amplitudes == [0.1, 0.3, 0.5, 0.3, 0.1]
|
||||
assert duration == 180.5
|
||||
|
||||
|
||||
class TestGenerateWaveformWithMutes:
|
||||
"""Test generate_waveform_with_mutes method."""
|
||||
|
||||
@patch("backend.services.audio_analysis_service.StorageService")
|
||||
@patch("backend.services.audio_analysis_service.AudioAnalyzer")
|
||||
@patch("backend.services.audio_analysis_service.WaveformGenerator")
|
||||
def test_generate_waveform_with_mutes(
|
||||
self, mock_waveform_class, mock_analyzer_class, mock_storage_class
|
||||
):
|
||||
"""Test generate_waveform_with_mutes highlights mute regions."""
|
||||
from backend.services.audio_analysis_service import AudioAnalysisService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.audible_segments = [Mock(), Mock()]
|
||||
|
||||
mock_analyzer = Mock()
|
||||
mock_analyzer.analyze.return_value = mock_result
|
||||
mock_analyzer_class.return_value = mock_analyzer
|
||||
|
||||
mock_waveform = Mock()
|
||||
mock_waveform_class.return_value = mock_waveform
|
||||
|
||||
# Create mock mute regions
|
||||
mock_mute1 = Mock()
|
||||
mock_mute2 = Mock()
|
||||
mute_regions = [mock_mute1, mock_mute2]
|
||||
|
||||
service = AudioAnalysisService()
|
||||
|
||||
result_path = service.generate_waveform_with_mutes(
|
||||
gcs_audio_path="uploads/job123/backing.flac",
|
||||
job_id="job123",
|
||||
gcs_waveform_destination="uploads/job123/waveform_muted.png",
|
||||
mute_regions=mute_regions,
|
||||
)
|
||||
|
||||
# Verify waveform generator was called with mute_regions
|
||||
mock_waveform.generate.assert_called_once()
|
||||
call_kwargs = mock_waveform.generate.call_args.kwargs
|
||||
assert call_kwargs["mute_regions"] == mute_regions
|
||||
assert call_kwargs["segments"] == mock_result.audible_segments
|
||||
|
||||
# Verify upload
|
||||
mock_storage.upload_file.assert_called_once()
|
||||
|
||||
# Verify return
|
||||
assert result_path == "uploads/job123/waveform_muted.png"
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
"""
|
||||
Tests for audio_editing_service.py - Custom instrumental creation.
|
||||
|
||||
These tests mock the storage service and shared karaoke_gen modules to verify:
|
||||
- GCS file download/upload operations
|
||||
- Audio editing delegation to shared AudioEditor
|
||||
- Mute region validation
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch, ANY
|
||||
|
||||
|
||||
class TestAudioEditingServiceInit:
|
||||
"""Test AudioEditingService initialization."""
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_init_creates_dependencies(self, mock_editor_class, mock_storage_class):
|
||||
"""Test initialization creates storage service and editor."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
mock_storage_class.assert_called_once()
|
||||
mock_editor_class.assert_called_once_with(output_format="flac")
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_init_with_custom_format(self, mock_editor_class, mock_storage_class):
|
||||
"""Test initialization with custom output format."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
service = AudioEditingService(output_format="wav")
|
||||
|
||||
mock_editor_class.assert_called_once_with(output_format="wav")
|
||||
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_init_with_provided_storage_service(self, mock_editor_class):
|
||||
"""Test initialization with externally provided storage service."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
mock_storage = Mock()
|
||||
service = AudioEditingService(storage_service=mock_storage)
|
||||
|
||||
assert service.storage_service is mock_storage
|
||||
|
||||
|
||||
class TestCreateCustomInstrumental:
|
||||
"""Test create_custom_instrumental method."""
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_create_custom_instrumental_success(
|
||||
self, mock_editor_class, mock_storage_class
|
||||
):
|
||||
"""Test creating a custom instrumental successfully."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.total_muted_duration_seconds = 15.5
|
||||
mock_result.mute_regions_applied = [Mock(), Mock(), Mock()]
|
||||
mock_result.output_path = None # Will be set by service
|
||||
|
||||
mock_editor = Mock()
|
||||
mock_editor.create_custom_instrumental.return_value = mock_result
|
||||
mock_editor_class.return_value = mock_editor
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
# Create mock mute regions
|
||||
mock_mute1 = Mock()
|
||||
mock_mute2 = Mock()
|
||||
mute_regions = [mock_mute1, mock_mute2]
|
||||
|
||||
result = service.create_custom_instrumental(
|
||||
gcs_clean_instrumental_path="uploads/job123/clean.flac",
|
||||
gcs_backing_vocals_path="uploads/job123/backing.flac",
|
||||
mute_regions=mute_regions,
|
||||
gcs_output_path="uploads/job123/custom.flac",
|
||||
job_id="job123",
|
||||
)
|
||||
|
||||
# Verify downloads
|
||||
assert mock_storage.download_file.call_count == 2
|
||||
|
||||
# Verify editor was called
|
||||
mock_editor.create_custom_instrumental.assert_called_once()
|
||||
call_kwargs = mock_editor.create_custom_instrumental.call_args.kwargs
|
||||
assert call_kwargs["mute_regions"] == mute_regions
|
||||
|
||||
# Verify upload
|
||||
mock_storage.upload_file.assert_called_once()
|
||||
|
||||
# Verify result path was updated
|
||||
assert result.output_path == "uploads/job123/custom.flac"
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_create_custom_instrumental_downloads_both_files(
|
||||
self, mock_editor_class, mock_storage_class
|
||||
):
|
||||
"""Test that both input files are downloaded."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.total_muted_duration_seconds = 0
|
||||
mock_result.mute_regions_applied = []
|
||||
|
||||
mock_editor = Mock()
|
||||
mock_editor.create_custom_instrumental.return_value = mock_result
|
||||
mock_editor_class.return_value = mock_editor
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
service.create_custom_instrumental(
|
||||
gcs_clean_instrumental_path="uploads/job123/clean.flac",
|
||||
gcs_backing_vocals_path="uploads/job123/backing.flac",
|
||||
mute_regions=[],
|
||||
gcs_output_path="uploads/job123/output.flac",
|
||||
job_id="job123",
|
||||
)
|
||||
|
||||
# Verify both files were downloaded
|
||||
download_calls = mock_storage.download_file.call_args_list
|
||||
assert len(download_calls) == 2
|
||||
|
||||
# First call should be clean instrumental
|
||||
assert download_calls[0][0][0] == "uploads/job123/clean.flac"
|
||||
# Second call should be backing vocals
|
||||
assert download_calls[1][0][0] == "uploads/job123/backing.flac"
|
||||
|
||||
|
||||
class TestCreatePreview:
|
||||
"""Test create_preview method."""
|
||||
|
||||
@patch("pydub.AudioSegment")
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_create_preview_success(
|
||||
self, mock_editor_class, mock_storage_class, mock_audio_segment
|
||||
):
|
||||
"""Test creating a preview successfully."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_preview = Mock()
|
||||
mock_preview.export = Mock()
|
||||
mock_preview.__getitem__ = Mock(return_value=mock_preview)
|
||||
|
||||
mock_editor = Mock()
|
||||
mock_editor.preview_with_mutes.return_value = mock_preview
|
||||
mock_editor_class.return_value = mock_editor
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
result = service.create_preview(
|
||||
gcs_clean_instrumental_path="uploads/job123/clean.flac",
|
||||
gcs_backing_vocals_path="uploads/job123/backing.flac",
|
||||
mute_regions=[Mock(), Mock()],
|
||||
gcs_preview_path="uploads/job123/preview.flac",
|
||||
job_id="job123",
|
||||
)
|
||||
|
||||
# Verify editor was called
|
||||
mock_editor.preview_with_mutes.assert_called_once()
|
||||
|
||||
# Verify result
|
||||
assert result == "uploads/job123/preview.flac"
|
||||
|
||||
@patch("pydub.AudioSegment")
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_create_preview_with_duration_limit(
|
||||
self, mock_editor_class, mock_storage_class, mock_audio_segment
|
||||
):
|
||||
"""Test creating a preview with duration limit."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_preview = Mock()
|
||||
mock_truncated = Mock()
|
||||
mock_preview.__getitem__ = Mock(return_value=mock_truncated)
|
||||
mock_truncated.export = Mock()
|
||||
|
||||
mock_editor = Mock()
|
||||
mock_editor.preview_with_mutes.return_value = mock_preview
|
||||
mock_editor_class.return_value = mock_editor
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
service.create_preview(
|
||||
gcs_clean_instrumental_path="uploads/job123/clean.flac",
|
||||
gcs_backing_vocals_path="uploads/job123/backing.flac",
|
||||
mute_regions=[],
|
||||
gcs_preview_path="uploads/job123/preview.flac",
|
||||
job_id="job123",
|
||||
preview_duration_seconds=30.0,
|
||||
)
|
||||
|
||||
# Verify preview was truncated (30 seconds = 30000ms)
|
||||
mock_preview.__getitem__.assert_called_once_with(slice(None, 30000))
|
||||
|
||||
|
||||
class TestMuteBackingVocalsOnly:
|
||||
"""Test mute_backing_vocals_only method."""
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_mute_backing_vocals_only(self, mock_editor_class, mock_storage_class):
|
||||
"""Test muting backing vocals without combining."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
mock_storage = Mock()
|
||||
mock_storage_class.return_value = mock_storage
|
||||
|
||||
mock_editor = Mock()
|
||||
mock_editor_class.return_value = mock_editor
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
mute_regions = [Mock(), Mock()]
|
||||
|
||||
result = service.mute_backing_vocals_only(
|
||||
gcs_backing_vocals_path="uploads/job123/backing.flac",
|
||||
mute_regions=mute_regions,
|
||||
gcs_output_path="uploads/job123/muted_backing.flac",
|
||||
job_id="job123",
|
||||
)
|
||||
|
||||
# Verify only one file was downloaded
|
||||
mock_storage.download_file.assert_called_once()
|
||||
assert mock_storage.download_file.call_args[0][0] == "uploads/job123/backing.flac"
|
||||
|
||||
# Verify editor method was called
|
||||
mock_editor.apply_mute_to_single_track.assert_called_once()
|
||||
call_kwargs = mock_editor.apply_mute_to_single_track.call_args.kwargs
|
||||
assert call_kwargs["mute_regions"] == mute_regions
|
||||
|
||||
# Verify upload
|
||||
mock_storage.upload_file.assert_called_once()
|
||||
|
||||
# Verify result
|
||||
assert result == "uploads/job123/muted_backing.flac"
|
||||
|
||||
|
||||
class TestValidateMuteRegions:
|
||||
"""Test validate_mute_regions method.
|
||||
|
||||
Note: MuteRegion uses pydantic validation that rejects invalid values at
|
||||
creation time. These tests use Mock objects to test the validation logic
|
||||
in the service, simulating edge cases that would normally be rejected.
|
||||
"""
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_valid_mute_regions(self, mock_editor_class, mock_storage_class):
|
||||
"""Test validation passes for valid mute regions."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
from karaoke_gen.instrumental_review import MuteRegion
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
mute_regions = [
|
||||
MuteRegion(start_seconds=10.0, end_seconds=15.0),
|
||||
MuteRegion(start_seconds=30.0, end_seconds=45.0),
|
||||
]
|
||||
|
||||
errors = service.validate_mute_regions(mute_regions, total_duration_seconds=180.0)
|
||||
|
||||
assert errors == []
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_negative_start_seconds(self, mock_editor_class, mock_storage_class):
|
||||
"""Test validation catches negative start time.
|
||||
|
||||
Uses Mock since MuteRegion pydantic model rejects negative values.
|
||||
"""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
# Use mock to simulate invalid region (pydantic would reject this)
|
||||
mock_region = Mock()
|
||||
mock_region.start_seconds = -5.0
|
||||
mock_region.end_seconds = 10.0
|
||||
|
||||
errors = service.validate_mute_regions([mock_region], total_duration_seconds=180.0)
|
||||
|
||||
assert len(errors) == 1
|
||||
assert "cannot be negative" in errors[0]
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_end_before_start(self, mock_editor_class, mock_storage_class):
|
||||
"""Test validation catches end time before start time."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
# Use mock to simulate invalid region
|
||||
mock_region = Mock()
|
||||
mock_region.start_seconds = 30.0
|
||||
mock_region.end_seconds = 20.0
|
||||
|
||||
errors = service.validate_mute_regions([mock_region], total_duration_seconds=180.0)
|
||||
|
||||
assert len(errors) == 1
|
||||
assert "must be after" in errors[0]
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_start_exceeds_duration(self, mock_editor_class, mock_storage_class):
|
||||
"""Test validation catches start time exceeding audio duration."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
from karaoke_gen.instrumental_review import MuteRegion
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
mute_regions = [
|
||||
MuteRegion(start_seconds=200.0, end_seconds=210.0),
|
||||
]
|
||||
|
||||
errors = service.validate_mute_regions(mute_regions, total_duration_seconds=180.0)
|
||||
|
||||
assert len(errors) == 1
|
||||
assert "exceeds audio duration" in errors[0]
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_end_exceeds_duration_is_warning_not_error(
|
||||
self, mock_editor_class, mock_storage_class
|
||||
):
|
||||
"""Test that end time exceeding duration is logged as warning, not error."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
from karaoke_gen.instrumental_review import MuteRegion
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
mute_regions = [
|
||||
MuteRegion(start_seconds=170.0, end_seconds=200.0), # End exceeds 180s
|
||||
]
|
||||
|
||||
errors = service.validate_mute_regions(mute_regions, total_duration_seconds=180.0)
|
||||
|
||||
# Should not be an error (will be clamped)
|
||||
assert len(errors) == 0
|
||||
|
||||
@patch("backend.services.audio_editing_service.StorageService")
|
||||
@patch("backend.services.audio_editing_service.AudioEditor")
|
||||
def test_multiple_errors(self, mock_editor_class, mock_storage_class):
|
||||
"""Test validation reports multiple errors."""
|
||||
from backend.services.audio_editing_service import AudioEditingService
|
||||
from karaoke_gen.instrumental_review import MuteRegion
|
||||
|
||||
service = AudioEditingService()
|
||||
|
||||
# Use mocks for invalid regions
|
||||
mock_region1 = Mock()
|
||||
mock_region1.start_seconds = -5.0
|
||||
mock_region1.end_seconds = 10.0
|
||||
|
||||
mock_region2 = Mock()
|
||||
mock_region2.start_seconds = 30.0
|
||||
mock_region2.end_seconds = 20.0 # End before start
|
||||
|
||||
mock_region3 = Mock()
|
||||
mock_region3.start_seconds = 200.0 # Exceeds duration
|
||||
mock_region3.end_seconds = 210.0
|
||||
|
||||
mute_regions = [mock_region1, mock_region2, mock_region3]
|
||||
|
||||
errors = service.validate_mute_regions(mute_regions, total_duration_seconds=180.0)
|
||||
|
||||
assert len(errors) == 3
|
||||
|
||||
@@ -99,7 +99,11 @@ class TestJobStatusAudioSearchStates:
|
||||
|
||||
|
||||
class TestAudioSearchResult:
|
||||
"""Test AudioSearchResult dataclass."""
|
||||
"""Test AudioSearchResult dataclass.
|
||||
|
||||
AudioSearchResult is now imported from karaoke_gen.audio_fetcher.
|
||||
These tests verify the backend can use it correctly.
|
||||
"""
|
||||
|
||||
def test_create_audio_search_result(self):
|
||||
"""Test creating an AudioSearchResult."""
|
||||
@@ -140,6 +144,8 @@ class TestAudioSearchResult:
|
||||
assert data['provider'] == "YouTube"
|
||||
assert data['url'] == "https://youtube.com/watch?v=abc123"
|
||||
assert data['index'] == 0
|
||||
# raw_result should NOT be in serialized dict
|
||||
assert 'raw_result' not in data
|
||||
|
||||
def test_from_dict(self):
|
||||
"""Test creating from dict."""
|
||||
@@ -160,6 +166,7 @@ class TestAudioSearchResult:
|
||||
assert result.artist == "ABBA"
|
||||
assert result.provider == "YouTube"
|
||||
assert result.index == 0
|
||||
assert result.raw_result is None # Not set from dict
|
||||
|
||||
|
||||
class TestAudioSearchServiceInit:
|
||||
@@ -171,10 +178,9 @@ class TestAudioSearchServiceInit:
|
||||
# Pass explicit None to override environment
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
|
||||
# When explicitly passed None, it should be None (not from env)
|
||||
assert service._redacted_api_key is None
|
||||
assert service._ops_api_key is None
|
||||
assert service._manager is None
|
||||
# Service should have a FlacFetcher internally
|
||||
assert service._fetcher is not None
|
||||
assert service._cached_results == []
|
||||
|
||||
def test_init_with_keys(self):
|
||||
"""Test initialization with API keys."""
|
||||
@@ -183,41 +189,40 @@ class TestAudioSearchServiceInit:
|
||||
ops_api_key="test_ops_key",
|
||||
)
|
||||
|
||||
assert service._redacted_api_key == "test_redacted_key"
|
||||
assert service._ops_api_key == "test_ops_key"
|
||||
# Keys are passed to FlacFetcher
|
||||
assert service._fetcher._redacted_api_key == "test_redacted_key"
|
||||
assert service._fetcher._ops_api_key == "test_ops_key"
|
||||
|
||||
def test_init_reads_from_environment(self):
|
||||
"""Test initialization reads keys from environment variables."""
|
||||
import os
|
||||
|
||||
# Just test that the service can be initialized
|
||||
service = AudioSearchService()
|
||||
|
||||
# The service should have API keys from env or None
|
||||
assert service._manager is None # Not yet initialized
|
||||
# The service should have a FlacFetcher
|
||||
assert service._fetcher is not None
|
||||
|
||||
|
||||
class TestAudioSearchServiceSearch:
|
||||
"""Test AudioSearchService.search() method."""
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_search_returns_results(self, mock_get_manager):
|
||||
def test_search_returns_results(self):
|
||||
"""Test search returns AudioSearchResult list."""
|
||||
# Mock the flacfetch result
|
||||
mock_result = Mock()
|
||||
mock_result.title = "Waterloo"
|
||||
mock_result.artist = "ABBA"
|
||||
mock_result.provider = "YouTube"
|
||||
mock_result.url = "https://youtube.com/watch?v=abc123"
|
||||
mock_result.duration = 180
|
||||
mock_result.quality = "FLAC"
|
||||
mock_result.id = "abc123"
|
||||
# Create mock result
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
duration=180,
|
||||
quality="FLAC",
|
||||
source_id="abc123",
|
||||
index=0,
|
||||
)
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = [mock_result]
|
||||
mock_get_manager.return_value = mock_manager
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = [mock_result]
|
||||
|
||||
service = AudioSearchService()
|
||||
results = service.search("ABBA", "Waterloo")
|
||||
|
||||
assert len(results) == 1
|
||||
@@ -226,40 +231,36 @@ class TestAudioSearchServiceSearch:
|
||||
assert results[0].provider == "YouTube"
|
||||
assert results[0].index == 0
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_search_no_results_raises_error(self, mock_get_manager):
|
||||
def test_search_no_results_raises_error(self):
|
||||
"""Test search raises NoResultsError when no results."""
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = []
|
||||
mock_get_manager.return_value = mock_manager
|
||||
|
||||
service = AudioSearchService()
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.side_effect = NoResultsError("No results found")
|
||||
|
||||
with pytest.raises(NoResultsError) as exc_info:
|
||||
service.search("Unknown Artist", "Unknown Song")
|
||||
|
||||
assert "No results found" in str(exc_info.value)
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_search_multiple_results(self, mock_get_manager):
|
||||
def test_search_multiple_results(self):
|
||||
"""Test search returns multiple results with correct indices."""
|
||||
mock_results = []
|
||||
for i in range(3):
|
||||
mock_result = Mock()
|
||||
mock_result.title = f"Song {i}"
|
||||
mock_result.artist = "Artist"
|
||||
mock_result.provider = ["YouTube", "Redacted", "OPS"][i]
|
||||
mock_result.url = f"https://example.com/{i}"
|
||||
mock_result.duration = 180 + i * 10
|
||||
mock_result.quality = ["320kbps", "FLAC", "FLAC"][i]
|
||||
mock_result.id = str(i)
|
||||
mock_results.append(mock_result)
|
||||
mock_results.append(AudioSearchResult(
|
||||
title=f"Song {i}",
|
||||
artist="Artist",
|
||||
provider=["YouTube", "Redacted", "OPS"][i],
|
||||
url=f"https://example.com/{i}",
|
||||
duration=180 + i * 10,
|
||||
quality=["320kbps", "FLAC", "FLAC"][i],
|
||||
source_id=str(i),
|
||||
index=i,
|
||||
))
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = mock_results
|
||||
mock_get_manager.return_value = mock_manager
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = mock_results
|
||||
|
||||
service = AudioSearchService()
|
||||
results = service.search("Artist", "Song")
|
||||
|
||||
assert len(results) == 3
|
||||
@@ -271,34 +272,39 @@ class TestAudioSearchServiceSearch:
|
||||
class TestAudioSearchServiceSelectBest:
|
||||
"""Test AudioSearchService.select_best() method."""
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_select_best_uses_manager(self, mock_get_manager):
|
||||
"""Test select_best uses flacfetch's select_best."""
|
||||
# First need to do a search to populate cache
|
||||
mock_result = Mock()
|
||||
mock_result.title = "Waterloo"
|
||||
mock_result.artist = "ABBA"
|
||||
mock_result.provider = "YouTube"
|
||||
mock_result.url = "https://youtube.com/watch?v=abc123"
|
||||
mock_result.duration = 180
|
||||
mock_result.quality = "FLAC"
|
||||
mock_result.id = "abc123"
|
||||
def test_select_best_delegates_to_fetcher(self):
|
||||
"""Test select_best uses FlacFetcher's select_best."""
|
||||
mock_results = [
|
||||
AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
index=0,
|
||||
),
|
||||
AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted",
|
||||
url="https://example.com/456",
|
||||
index=1,
|
||||
),
|
||||
]
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = [mock_result]
|
||||
mock_manager.select_best.return_value = mock_result
|
||||
mock_get_manager.return_value = mock_manager
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.select_best.return_value = 1
|
||||
|
||||
service = AudioSearchService()
|
||||
results = service.search("ABBA", "Waterloo")
|
||||
best_index = service.select_best(mock_results)
|
||||
|
||||
best_index = service.select_best(results)
|
||||
|
||||
assert best_index == 0
|
||||
assert best_index == 1
|
||||
service._fetcher.select_best.assert_called_once_with(mock_results)
|
||||
|
||||
def test_select_best_empty_list_returns_zero(self):
|
||||
"""Test select_best returns 0 for empty list."""
|
||||
service = AudioSearchService()
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.select_best.return_value = 0
|
||||
|
||||
best_index = service.select_best([])
|
||||
|
||||
@@ -308,24 +314,18 @@ class TestAudioSearchServiceSelectBest:
|
||||
class TestAudioSearchServiceDownload:
|
||||
"""Test AudioSearchService.download() method."""
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_download_without_search_raises_error(self, mock_get_manager):
|
||||
def test_download_without_search_raises_error(self):
|
||||
"""Test download without prior search raises error."""
|
||||
mock_manager = Mock()
|
||||
mock_get_manager.return_value = mock_manager
|
||||
|
||||
service = AudioSearchService()
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
|
||||
with pytest.raises(DownloadError) as exc_info:
|
||||
service.download(0, "/tmp")
|
||||
|
||||
assert "No cached result" in str(exc_info.value)
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_download_after_search(self, mock_get_manager):
|
||||
def test_download_after_search(self):
|
||||
"""Test download after search works."""
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Create a temp file to simulate downloaded file
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
@@ -333,23 +333,32 @@ class TestAudioSearchServiceDownload:
|
||||
with open(temp_file, 'w') as f:
|
||||
f.write("test")
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.title = "Waterloo"
|
||||
mock_result.artist = "ABBA"
|
||||
mock_result.provider = "YouTube"
|
||||
mock_result.url = "https://youtube.com/watch?v=abc123"
|
||||
mock_result.duration = 180
|
||||
mock_result.quality = "FLAC"
|
||||
mock_result.id = "abc123"
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
duration=180,
|
||||
quality="FLAC",
|
||||
source_id="abc123",
|
||||
index=0,
|
||||
)
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = [mock_result]
|
||||
mock_manager.download.return_value = temp_file
|
||||
mock_get_manager.return_value = mock_manager
|
||||
mock_fetch_result = AudioDownloadResult(
|
||||
filepath=temp_file,
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="YouTube",
|
||||
duration=180,
|
||||
quality="FLAC",
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = [mock_result]
|
||||
service._fetcher.download.return_value = mock_fetch_result
|
||||
|
||||
service = AudioSearchService()
|
||||
service.search("ABBA", "Waterloo")
|
||||
|
||||
result = service.download(0, temp_dir)
|
||||
|
||||
assert result.filepath == temp_file
|
||||
@@ -360,6 +369,27 @@ class TestAudioSearchServiceDownload:
|
||||
# Cleanup
|
||||
os.remove(temp_file)
|
||||
os.rmdir(temp_dir)
|
||||
|
||||
def test_download_invalid_index_raises_error(self):
|
||||
"""Test download with invalid index raises error."""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = [mock_result]
|
||||
|
||||
service.search("ABBA", "Waterloo")
|
||||
|
||||
with pytest.raises(DownloadError) as exc_info:
|
||||
service.download(99, "/tmp")
|
||||
|
||||
assert "No cached result for index 99" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestGetAudioSearchService:
|
||||
@@ -384,3 +414,653 @@ class TestGetAudioSearchService:
|
||||
assert service1 is service2
|
||||
|
||||
|
||||
class TestRemoteDownloadPath:
|
||||
"""Test remote download functionality when flacfetch service is configured.
|
||||
|
||||
These tests verify the code path that uses the remote flacfetch service
|
||||
for torrent downloads (Redacted/OPS providers). This would have caught
|
||||
the 'extra_info' attribute error.
|
||||
|
||||
The tests directly set up the service state without calling search() to avoid
|
||||
the complexity of mocking async remote search operations.
|
||||
"""
|
||||
|
||||
def test_download_uses_remote_for_redacted_provider(self):
|
||||
"""Test download routes to remote service for Redacted provider.
|
||||
|
||||
This test would have caught: 'AudioSearchResult' object has no attribute 'extra_info'
|
||||
"""
|
||||
# Create a Redacted result (torrent source)
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted", # Torrent provider - should use remote
|
||||
url="", # No URL for torrent sources
|
||||
quality="FLAC 16bit CD",
|
||||
seeders=50,
|
||||
target_file="Waterloo.flac",
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Create service with mocked remote client
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
|
||||
# Directly set cached results (simulating after search)
|
||||
service._cached_results = [mock_result]
|
||||
service._remote_search_id = "remote_search_123" # Remote search was performed
|
||||
|
||||
# Mock the remote client
|
||||
mock_remote_client = Mock()
|
||||
service._remote_client = mock_remote_client
|
||||
|
||||
# Mock the remote download method
|
||||
mock_download_result = AudioDownloadResult(
|
||||
filepath="gs://bucket/uploads/job123/audio/Waterloo.flac",
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="Redacted",
|
||||
quality="FLAC 16bit CD",
|
||||
)
|
||||
service._download_remote = Mock(return_value=mock_download_result)
|
||||
|
||||
# Call download - should route to remote
|
||||
result = service.download(0, "/tmp", gcs_path="uploads/job123/audio/")
|
||||
|
||||
# Verify remote download was called
|
||||
service._download_remote.assert_called_once_with(0, "/tmp", None, "uploads/job123/audio/")
|
||||
assert result.filepath == "gs://bucket/uploads/job123/audio/Waterloo.flac"
|
||||
|
||||
def test_download_uses_remote_for_ops_provider(self):
|
||||
"""Test download routes to remote service for OPS provider."""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="OPS", # Torrent provider
|
||||
url="",
|
||||
quality="FLAC 16bit CD",
|
||||
seeders=30,
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
|
||||
# Directly set cached results
|
||||
service._cached_results = [mock_result]
|
||||
service._remote_search_id = "remote_search_456"
|
||||
|
||||
# Mock remote client
|
||||
mock_remote_client = Mock()
|
||||
service._remote_client = mock_remote_client
|
||||
|
||||
# Mock remote download
|
||||
mock_download_result = AudioDownloadResult(
|
||||
filepath="gs://bucket/test.flac",
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="OPS",
|
||||
quality="FLAC",
|
||||
)
|
||||
service._download_remote = Mock(return_value=mock_download_result)
|
||||
|
||||
# Call download
|
||||
result = service.download(0, "/tmp")
|
||||
|
||||
# Should use remote for OPS
|
||||
service._download_remote.assert_called_once()
|
||||
|
||||
def test_download_uses_local_for_youtube_even_with_remote_client(self):
|
||||
"""Test YouTube downloads use local even when remote client is configured."""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube", # NOT a torrent provider
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
quality="Opus 128kbps",
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
|
||||
# Directly set cached results
|
||||
service._cached_results = [mock_result]
|
||||
service._remote_search_id = "remote_search_789" # Remote search was performed
|
||||
|
||||
# Mock remote client (configured but shouldn't be used for YouTube)
|
||||
mock_remote_client = Mock()
|
||||
service._remote_client = mock_remote_client
|
||||
|
||||
# Mock local download
|
||||
mock_fetch_result = AudioDownloadResult(
|
||||
filepath="/tmp/test.opus",
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="YouTube",
|
||||
quality="Opus 128kbps",
|
||||
)
|
||||
service._fetcher.download.return_value = mock_fetch_result
|
||||
|
||||
# Mock _download_remote to ensure it's NOT called
|
||||
service._download_remote = Mock()
|
||||
|
||||
# Call download
|
||||
result = service.download(0, "/tmp")
|
||||
|
||||
# Should use local for YouTube
|
||||
service._download_remote.assert_not_called()
|
||||
service._fetcher.download.assert_called_once()
|
||||
|
||||
def test_download_uses_local_when_no_remote_client(self):
|
||||
"""Test download uses local when remote client is not configured."""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted", # Would normally use remote
|
||||
url="",
|
||||
quality="FLAC",
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
|
||||
# Directly set cached results
|
||||
service._cached_results = [mock_result]
|
||||
|
||||
# No remote client
|
||||
service._remote_client = None
|
||||
service._remote_search_id = None # No remote search
|
||||
|
||||
# Mock local download
|
||||
mock_fetch_result = AudioDownloadResult(
|
||||
filepath="/tmp/test.flac",
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="Redacted",
|
||||
quality="FLAC",
|
||||
)
|
||||
service._fetcher.download.return_value = mock_fetch_result
|
||||
|
||||
# Call download
|
||||
result = service.download(0, "/tmp")
|
||||
|
||||
# Should use local even though it's Redacted
|
||||
service._fetcher.download.assert_called_once()
|
||||
|
||||
def test_download_uses_local_when_no_remote_search_id(self):
|
||||
"""Test download uses local when search wasn't done remotely."""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted",
|
||||
url="",
|
||||
quality="FLAC",
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
|
||||
# Directly set cached results
|
||||
service._cached_results = [mock_result]
|
||||
|
||||
# Remote client configured but search was local (fallback scenario)
|
||||
service._remote_client = Mock()
|
||||
service._remote_search_id = None # Search was local, not remote
|
||||
|
||||
# Mock local download
|
||||
mock_fetch_result = AudioDownloadResult(
|
||||
filepath="/tmp/test.flac",
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="Redacted",
|
||||
quality="FLAC",
|
||||
)
|
||||
service._fetcher.download.return_value = mock_fetch_result
|
||||
|
||||
# Call download
|
||||
result = service.download(0, "/tmp")
|
||||
|
||||
# Should use local since no remote search ID
|
||||
service._fetcher.download.assert_called_once()
|
||||
|
||||
def test_torrent_provider_routing_checks_both_conditions(self):
|
||||
"""Test that torrent provider routing requires BOTH remote_search_id AND remote_client.
|
||||
|
||||
This test verifies the logical AND condition in the download routing.
|
||||
"""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted",
|
||||
url="",
|
||||
quality="FLAC",
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._cached_results = [mock_result]
|
||||
|
||||
# Test: remote_client set but no remote_search_id -> should use local
|
||||
service._remote_client = Mock()
|
||||
service._remote_search_id = None
|
||||
|
||||
mock_fetch_result = AudioDownloadResult(
|
||||
filepath="/tmp/test.flac", artist="ABBA", title="Waterloo",
|
||||
provider="Redacted", quality="FLAC",
|
||||
)
|
||||
service._fetcher.download.return_value = mock_fetch_result
|
||||
service._download_remote = Mock()
|
||||
|
||||
service.download(0, "/tmp")
|
||||
service._download_remote.assert_not_called()
|
||||
service._fetcher.download.assert_called_once()
|
||||
|
||||
# Reset mocks
|
||||
service._fetcher.download.reset_mock()
|
||||
service._download_remote.reset_mock()
|
||||
|
||||
# Test: remote_search_id set but no remote_client -> should use local
|
||||
service._remote_client = None
|
||||
service._remote_search_id = "search_123"
|
||||
|
||||
service.download(0, "/tmp")
|
||||
service._download_remote.assert_not_called()
|
||||
service._fetcher.download.assert_called_once()
|
||||
|
||||
def test_download_does_not_access_nonexistent_attributes(self):
|
||||
"""Test that download() doesn't try to access nonexistent attributes.
|
||||
|
||||
This test would have caught the 'extra_info' attribute error:
|
||||
AttributeError: 'AudioSearchResult' object has no attribute 'extra_info'
|
||||
|
||||
AudioSearchResult only has: title, artist, url, provider, duration,
|
||||
quality, source_id, index, seeders, target_file, raw_result
|
||||
"""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted",
|
||||
url="",
|
||||
quality="FLAC 16bit CD",
|
||||
seeders=50,
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Verify AudioSearchResult doesn't have extra_info
|
||||
assert not hasattr(mock_result, 'extra_info')
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._cached_results = [mock_result]
|
||||
service._remote_search_id = "search_123"
|
||||
service._remote_client = Mock()
|
||||
|
||||
# Mock _download_remote to avoid actual network call
|
||||
mock_download_result = AudioDownloadResult(
|
||||
filepath="/tmp/test.flac",
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="Redacted",
|
||||
quality="FLAC",
|
||||
)
|
||||
service._download_remote = Mock(return_value=mock_download_result)
|
||||
|
||||
# This should NOT raise AttributeError: 'AudioSearchResult' object has no attribute 'extra_info'
|
||||
result = service.download(0, "/tmp")
|
||||
|
||||
# Should succeed and use remote download for Redacted provider
|
||||
service._download_remote.assert_called_once()
|
||||
assert result.filepath == "/tmp/test.flac"
|
||||
|
||||
|
||||
class TestFlacfetchIntegration:
|
||||
"""
|
||||
Integration tests that verify flacfetch imports work correctly.
|
||||
|
||||
These tests ensure the backend code is compatible with the installed
|
||||
flacfetch version and would catch issues like renamed classes.
|
||||
|
||||
Note: Some tests are marked with pytest.importorskip for flacfetch
|
||||
since it may not be installed in all test environments (e.g., CI).
|
||||
"""
|
||||
|
||||
def test_flacfetch_imports_work(self):
|
||||
"""Test that all flacfetch imports in audio_search_service work.
|
||||
|
||||
This test would have caught the YouTubeProvider -> YoutubeProvider rename.
|
||||
"""
|
||||
flacfetch = pytest.importorskip("flacfetch")
|
||||
|
||||
# These imports match what karaoke_gen.audio_fetcher does
|
||||
from flacfetch.core.manager import FetchManager
|
||||
from flacfetch.providers.youtube import YoutubeProvider
|
||||
from flacfetch.core.models import TrackQuery
|
||||
|
||||
# Verify the classes exist and are callable
|
||||
assert FetchManager is not None
|
||||
assert YoutubeProvider is not None
|
||||
assert TrackQuery is not None
|
||||
|
||||
def test_karaoke_gen_audio_fetcher_imports_work(self):
|
||||
"""Test that karaoke_gen.audio_fetcher imports work.
|
||||
|
||||
Since backend now imports from karaoke_gen, this is the critical test.
|
||||
This should work even without flacfetch installed (lazy import).
|
||||
"""
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
FlacFetcher,
|
||||
AudioSearchResult,
|
||||
AudioFetchResult,
|
||||
AudioFetcherError,
|
||||
NoResultsError,
|
||||
DownloadError,
|
||||
)
|
||||
|
||||
assert FlacFetcher is not None
|
||||
assert AudioSearchResult is not None
|
||||
assert AudioFetchResult is not None
|
||||
assert AudioFetcherError is not None
|
||||
assert NoResultsError is not None
|
||||
assert DownloadError is not None
|
||||
|
||||
def test_audio_search_service_can_initialize_fetcher(self):
|
||||
"""Test AudioSearchService can initialize its FlacFetcher.
|
||||
|
||||
This verifies the actual initialization code path works.
|
||||
"""
|
||||
pytest.importorskip("flacfetch")
|
||||
|
||||
service = AudioSearchService(
|
||||
redacted_api_key=None,
|
||||
ops_api_key=None,
|
||||
)
|
||||
|
||||
# Verify FlacFetcher was initialized
|
||||
assert service._fetcher is not None
|
||||
|
||||
# FlacFetcher can initialize its manager (tests actual flacfetch)
|
||||
manager = service._fetcher._get_manager()
|
||||
assert manager is not None
|
||||
|
||||
def test_shared_classes_are_same_as_karaoke_gen(self):
|
||||
"""Verify backend uses the same classes as karaoke_gen."""
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
AudioSearchResult as KGAudioSearchResult,
|
||||
AudioFetchResult as KGAudioFetchResult,
|
||||
NoResultsError as KGNoResultsError,
|
||||
DownloadError as KGDownloadError,
|
||||
)
|
||||
|
||||
# These should be the exact same classes, not copies
|
||||
assert AudioSearchResult is KGAudioSearchResult
|
||||
assert AudioDownloadResult is KGAudioFetchResult
|
||||
assert NoResultsError is KGNoResultsError
|
||||
assert DownloadError is KGDownloadError
|
||||
|
||||
|
||||
class TestAudioSearchApiRouteDownload:
|
||||
"""
|
||||
Tests for _download_and_start_processing function in audio_search routes.
|
||||
|
||||
This tests the API layer's handling of the download flow, including:
|
||||
- Correct routing between local and remote downloads
|
||||
- GCS path handling for remote downloads
|
||||
- Error handling
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(self):
|
||||
"""Mock the job manager singleton."""
|
||||
with patch('backend.api.routes.audio_search.job_manager') as mock:
|
||||
yield mock
|
||||
|
||||
@pytest.fixture
|
||||
def mock_storage_service(self):
|
||||
"""Mock the storage service singleton."""
|
||||
with patch('backend.api.routes.audio_search.storage_service') as mock:
|
||||
yield mock
|
||||
|
||||
def test_remote_download_passes_gcs_path(self, mock_job_manager, mock_storage_service):
|
||||
"""
|
||||
Test that remote torrent downloads include GCS path for direct upload.
|
||||
|
||||
This was the bug: we were calling download() without gcs_path for
|
||||
torrent sources, causing flacfetch VM to return a local path that
|
||||
Cloud Run couldn't access.
|
||||
"""
|
||||
from backend.api.routes.audio_search import _download_and_start_processing
|
||||
import asyncio
|
||||
|
||||
# Setup mock job with Redacted search result
|
||||
mock_job = Mock()
|
||||
mock_job.state_data = {
|
||||
'audio_search_results': [{
|
||||
'title': 'Unwanted',
|
||||
'artist': 'Avril Lavigne',
|
||||
'provider': 'Redacted', # Torrent source
|
||||
'quality': 'FLAC 16bit CD',
|
||||
}]
|
||||
}
|
||||
mock_job.audio_search_artist = 'Avril Lavigne'
|
||||
mock_job.audio_search_title = 'Unwanted'
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
|
||||
# Create mock audio search service with remote client
|
||||
mock_audio_service = Mock()
|
||||
mock_audio_service.is_remote_enabled.return_value = True
|
||||
|
||||
# Mock download to return GCS path
|
||||
mock_download_result = Mock()
|
||||
mock_download_result.filepath = "gs://bucket/uploads/job123/audio/Avril Lavigne - Unwanted.flac"
|
||||
mock_audio_service.download.return_value = mock_download_result
|
||||
|
||||
# Create mock background tasks
|
||||
mock_bg_tasks = Mock()
|
||||
|
||||
# Run the async function
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
_download_and_start_processing(
|
||||
job_id="job123",
|
||||
selection_index=0,
|
||||
audio_search_service=mock_audio_service,
|
||||
background_tasks=mock_bg_tasks,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# CRITICAL: Verify download was called with gcs_path for remote torrent source
|
||||
mock_audio_service.download.assert_called_once()
|
||||
call_kwargs = mock_audio_service.download.call_args.kwargs
|
||||
|
||||
# The gcs_path should be set for remote torrent downloads
|
||||
assert 'gcs_path' in call_kwargs
|
||||
assert call_kwargs['gcs_path'] == "uploads/job123/audio/"
|
||||
|
||||
def test_local_youtube_download_does_not_pass_gcs_path(self, mock_job_manager, mock_storage_service):
|
||||
"""Test that YouTube downloads don't use gcs_path (download locally, upload manually)."""
|
||||
from backend.api.routes.audio_search import _download_and_start_processing
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
# Setup mock job with YouTube search result
|
||||
mock_job = Mock()
|
||||
mock_job.state_data = {
|
||||
'audio_search_results': [{
|
||||
'title': 'Unwanted',
|
||||
'artist': 'Avril Lavigne',
|
||||
'provider': 'YouTube', # NOT a torrent source
|
||||
'quality': 'Opus 128kbps',
|
||||
}]
|
||||
}
|
||||
mock_job.audio_search_artist = 'Avril Lavigne'
|
||||
mock_job.audio_search_title = 'Unwanted'
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
|
||||
# Create mock audio search service with remote client (but YouTube doesn't use it)
|
||||
mock_audio_service = Mock()
|
||||
mock_audio_service.is_remote_enabled.return_value = True
|
||||
|
||||
# Create a temp file to simulate downloaded file
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix='.opus', delete=False)
|
||||
temp_file.write(b'fake audio data')
|
||||
temp_file.close()
|
||||
|
||||
try:
|
||||
# Mock download to return local path
|
||||
mock_download_result = Mock()
|
||||
mock_download_result.filepath = temp_file.name
|
||||
mock_audio_service.download.return_value = mock_download_result
|
||||
|
||||
# Create mock background tasks
|
||||
mock_bg_tasks = Mock()
|
||||
|
||||
# Run the async function
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
_download_and_start_processing(
|
||||
job_id="job456",
|
||||
selection_index=0,
|
||||
audio_search_service=mock_audio_service,
|
||||
background_tasks=mock_bg_tasks,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# YouTube should NOT have gcs_path in kwargs
|
||||
mock_audio_service.download.assert_called_once()
|
||||
call_kwargs = mock_audio_service.download.call_args.kwargs
|
||||
|
||||
# For YouTube, gcs_path should NOT be set (or should be None)
|
||||
assert call_kwargs.get('gcs_path') is None
|
||||
|
||||
# Instead, storage service should be called to upload
|
||||
mock_storage_service.upload_fileobj.assert_called_once()
|
||||
finally:
|
||||
import os
|
||||
try:
|
||||
os.unlink(temp_file.name)
|
||||
except:
|
||||
pass
|
||||
|
||||
def test_handles_gcs_path_response_correctly(self, mock_job_manager, mock_storage_service):
|
||||
"""Test that GCS path responses are parsed correctly."""
|
||||
from backend.api.routes.audio_search import _download_and_start_processing
|
||||
import asyncio
|
||||
|
||||
# Setup mock job with Redacted search result
|
||||
mock_job = Mock()
|
||||
mock_job.state_data = {
|
||||
'audio_search_results': [{
|
||||
'title': 'Test',
|
||||
'artist': 'Test Artist',
|
||||
'provider': 'Redacted',
|
||||
'quality': 'FLAC',
|
||||
}]
|
||||
}
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
|
||||
# Create mock audio search service
|
||||
mock_audio_service = Mock()
|
||||
mock_audio_service.is_remote_enabled.return_value = True
|
||||
|
||||
# Mock download to return full GCS path
|
||||
mock_download_result = Mock()
|
||||
mock_download_result.filepath = "gs://karaoke-gen-bucket/uploads/job789/audio/test.flac"
|
||||
mock_audio_service.download.return_value = mock_download_result
|
||||
|
||||
# Run the async function
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
_download_and_start_processing(
|
||||
job_id="job789",
|
||||
selection_index=0,
|
||||
audio_search_service=mock_audio_service,
|
||||
background_tasks=Mock(),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# Verify job was updated with correct GCS path (without gs://bucket/ prefix)
|
||||
update_calls = mock_job_manager.update_job.call_args_list
|
||||
|
||||
# Find the call that sets input_media_gcs_path
|
||||
gcs_path_set = False
|
||||
for call in update_calls:
|
||||
if 'input_media_gcs_path' in call.args[1]:
|
||||
gcs_path = call.args[1]['input_media_gcs_path']
|
||||
# The path stored should be the relative path, not the full gs:// URL
|
||||
assert gcs_path == "uploads/job789/audio/test.flac"
|
||||
gcs_path_set = True
|
||||
break
|
||||
|
||||
assert gcs_path_set, "input_media_gcs_path was not set in job update"
|
||||
|
||||
def test_remote_disabled_always_uses_local(self, mock_job_manager, mock_storage_service):
|
||||
"""Test that when remote is disabled, even torrent sources use local download."""
|
||||
from backend.api.routes.audio_search import _download_and_start_processing
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
# Setup mock job with Redacted search result
|
||||
mock_job = Mock()
|
||||
mock_job.state_data = {
|
||||
'audio_search_results': [{
|
||||
'title': 'Test',
|
||||
'artist': 'Test',
|
||||
'provider': 'Redacted', # Torrent source, but remote is disabled
|
||||
'quality': 'FLAC',
|
||||
}]
|
||||
}
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
|
||||
# Create temp file
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix='.flac', delete=False)
|
||||
temp_file.write(b'fake')
|
||||
temp_file.close()
|
||||
|
||||
try:
|
||||
# Create mock audio search service WITHOUT remote client
|
||||
mock_audio_service = Mock()
|
||||
mock_audio_service.is_remote_enabled.return_value = False # REMOTE DISABLED
|
||||
|
||||
mock_download_result = Mock()
|
||||
mock_download_result.filepath = temp_file.name
|
||||
mock_audio_service.download.return_value = mock_download_result
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
_download_and_start_processing(
|
||||
job_id="job_no_remote",
|
||||
selection_index=0,
|
||||
audio_search_service=mock_audio_service,
|
||||
background_tasks=Mock(),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# When remote is disabled, gcs_path should NOT be passed
|
||||
mock_audio_service.download.assert_called_once()
|
||||
call_kwargs = mock_audio_service.download.call_args.kwargs
|
||||
assert call_kwargs.get('gcs_path') is None
|
||||
|
||||
# And storage should upload manually
|
||||
mock_storage_service.upload_fileobj.assert_called_once()
|
||||
finally:
|
||||
import os
|
||||
try:
|
||||
os.unlink(temp_file.name)
|
||||
except:
|
||||
pass
|
||||
@@ -0,0 +1,408 @@
|
||||
"""
|
||||
Tests for dropbox_service.py - Dropbox file operations.
|
||||
|
||||
These tests mock the Dropbox SDK and Secret Manager to verify:
|
||||
- Credential loading from Secret Manager
|
||||
- Folder listing and brand code calculation
|
||||
- File and folder uploads
|
||||
- Shared link creation
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
|
||||
|
||||
class TestDropboxServiceInit:
|
||||
"""Test DropboxService initialization."""
|
||||
|
||||
def test_init_creates_service(self):
|
||||
"""Test initialization creates service with no client."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
assert service._client is None
|
||||
assert service._is_configured is False
|
||||
|
||||
|
||||
class TestLoadCredentials:
|
||||
"""Test _load_credentials method."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_load_credentials_success(self, mock_sm_client_class):
|
||||
"""Test successful credential loading from Secret Manager."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({
|
||||
"access_token": "access-token-123",
|
||||
"refresh_token": "refresh-token-456",
|
||||
"app_key": "app-key",
|
||||
"app_secret": "app-secret",
|
||||
}).encode("UTF-8")
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
creds = service._load_credentials()
|
||||
|
||||
assert creds is not None
|
||||
assert creds["access_token"] == "access-token-123"
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_load_credentials_failure(self, mock_sm_client_class):
|
||||
"""Test handling when Secret Manager fails."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_sm_client.access_secret_version.side_effect = Exception("Access denied")
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
creds = service._load_credentials()
|
||||
|
||||
assert creds is None
|
||||
|
||||
|
||||
class TestIsConfigured:
|
||||
"""Test is_configured property."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_is_configured_true(self, mock_sm_client_class):
|
||||
"""Test is_configured returns True when credentials available."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({
|
||||
"access_token": "token"
|
||||
}).encode("UTF-8")
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
assert service.is_configured is True
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_is_configured_false_no_token(self, mock_sm_client_class):
|
||||
"""Test is_configured returns False when no access_token."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({
|
||||
"refresh_token": "refresh" # Missing access_token
|
||||
}).encode("UTF-8")
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
assert service.is_configured is False
|
||||
|
||||
|
||||
class TestDropboxClient:
|
||||
"""Test client property."""
|
||||
|
||||
@patch("dropbox.Dropbox")
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_client_creates_dropbox_instance(self, mock_sm_client_class, mock_dropbox_class):
|
||||
"""Test client property creates Dropbox SDK client."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({
|
||||
"access_token": "token",
|
||||
"refresh_token": "refresh",
|
||||
"app_key": "key",
|
||||
"app_secret": "secret",
|
||||
}).encode("UTF-8")
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox_class.return_value = mock_dropbox
|
||||
|
||||
service = DropboxService()
|
||||
client = service.client
|
||||
|
||||
mock_dropbox_class.assert_called_once_with(
|
||||
oauth2_access_token="token",
|
||||
oauth2_refresh_token="refresh",
|
||||
app_key="key",
|
||||
app_secret="secret",
|
||||
)
|
||||
assert client == mock_dropbox
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_client_raises_on_missing_credentials(self, mock_sm_client_class):
|
||||
"""Test client raises RuntimeError when credentials missing."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_sm_client.access_secret_version.side_effect = Exception("Not found")
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_ = service.client
|
||||
|
||||
assert "not configured" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestListFolders:
|
||||
"""Test list_folders method."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_list_folders(self, mock_sm_client_class):
|
||||
"""Test listing folders at a path."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
from dropbox.files import FolderMetadata
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
# Create mock folder entries
|
||||
mock_folder1 = Mock(spec=FolderMetadata)
|
||||
mock_folder1.name = "NOMAD-0001"
|
||||
mock_folder2 = Mock(spec=FolderMetadata)
|
||||
mock_folder2.name = "NOMAD-0002"
|
||||
mock_file = Mock() # Not a FolderMetadata
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.entries = [mock_folder1, mock_folder2, mock_file]
|
||||
mock_result.has_more = False
|
||||
|
||||
service = DropboxService()
|
||||
# Directly set the client to avoid needing to mock the whole init chain
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.files_list_folder.return_value = mock_result
|
||||
service._client = mock_dropbox
|
||||
|
||||
folders = service.list_folders("/Karaoke/Tracks")
|
||||
|
||||
mock_dropbox.files_list_folder.assert_called_once_with("/Karaoke/Tracks")
|
||||
assert folders == ["NOMAD-0001", "NOMAD-0002"]
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_list_folders_adds_leading_slash(self, mock_sm_client_class):
|
||||
"""Test that path without leading slash gets one added."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.entries = []
|
||||
mock_result.has_more = False
|
||||
|
||||
service = DropboxService()
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.files_list_folder.return_value = mock_result
|
||||
service._client = mock_dropbox
|
||||
|
||||
service.list_folders("path/without/slash")
|
||||
|
||||
mock_dropbox.files_list_folder.assert_called_once_with("/path/without/slash")
|
||||
|
||||
|
||||
class TestGetNextBrandCode:
|
||||
"""Test get_next_brand_code method."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_get_next_brand_code(self, mock_sm_client_class):
|
||||
"""Test calculating next brand code from existing folders."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
# Mock list_folders to return existing codes
|
||||
with patch.object(service, "list_folders") as mock_list:
|
||||
mock_list.return_value = [
|
||||
"NOMAD-1161",
|
||||
"NOMAD-1162",
|
||||
"NOMAD-1163",
|
||||
"Other Folder",
|
||||
"NOMAD-0001",
|
||||
]
|
||||
|
||||
next_code = service.get_next_brand_code("/path", "NOMAD")
|
||||
|
||||
assert next_code == "NOMAD-1164"
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_get_next_brand_code_empty_folder(self, mock_sm_client_class):
|
||||
"""Test brand code calculation with no existing codes."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
with patch.object(service, "list_folders") as mock_list:
|
||||
mock_list.return_value = []
|
||||
|
||||
next_code = service.get_next_brand_code("/path", "BRAND")
|
||||
|
||||
assert next_code == "BRAND-0001"
|
||||
|
||||
|
||||
class TestUploadFile:
|
||||
"""Test upload_file method."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_upload_small_file(self, mock_sm_client_class, tmp_path):
|
||||
"""Test uploading a small file directly."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
# Create test file
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("Small file content")
|
||||
|
||||
service = DropboxService()
|
||||
mock_dropbox = Mock()
|
||||
service._client = mock_dropbox
|
||||
|
||||
service.upload_file(str(test_file), "/Uploads/test.txt")
|
||||
|
||||
mock_dropbox.files_upload.assert_called_once()
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_upload_file_adds_leading_slash(self, mock_sm_client_class, tmp_path):
|
||||
"""Test upload adds leading slash to remote path."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("content")
|
||||
|
||||
service = DropboxService()
|
||||
mock_dropbox = Mock()
|
||||
service._client = mock_dropbox
|
||||
|
||||
service.upload_file(str(test_file), "uploads/test.txt")
|
||||
|
||||
# Check that the path has leading slash
|
||||
call_args = mock_dropbox.files_upload.call_args
|
||||
assert call_args[0][1] == "/uploads/test.txt"
|
||||
|
||||
|
||||
class TestUploadFolder:
|
||||
"""Test upload_folder method."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_upload_folder(self, mock_sm_client_class, tmp_path):
|
||||
"""Test uploading a folder with multiple files."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
# Create test folder with files
|
||||
(tmp_path / "file1.txt").write_text("content1")
|
||||
(tmp_path / "file2.txt").write_text("content2")
|
||||
(tmp_path / "subdir").mkdir() # Should be skipped
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
with patch.object(service, "upload_file") as mock_upload:
|
||||
service.upload_folder(str(tmp_path), "/Uploads/folder")
|
||||
|
||||
# Should upload 2 files (not subdir)
|
||||
assert mock_upload.call_count == 2
|
||||
|
||||
|
||||
class TestCreateSharedLink:
|
||||
"""Test create_shared_link method."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_create_shared_link_new(self, mock_sm_client_class):
|
||||
"""Test creating a new shared link."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
mock_link = Mock()
|
||||
mock_link.url = "https://dropbox.com/s/abc123/file.mp4"
|
||||
|
||||
service = DropboxService()
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.sharing_create_shared_link_with_settings.return_value = mock_link
|
||||
service._client = mock_dropbox
|
||||
|
||||
url = service.create_shared_link("/Uploads/file.mp4")
|
||||
|
||||
assert url == "https://dropbox.com/s/abc123/file.mp4"
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_create_shared_link_existing(self, mock_sm_client_class):
|
||||
"""Test getting existing shared link when one already exists."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
from dropbox.exceptions import ApiError
|
||||
|
||||
mock_sm_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client.access_secret_version.return_value = mock_response
|
||||
mock_sm_client_class.return_value = mock_sm_client
|
||||
|
||||
# Simulate "link already exists" error
|
||||
mock_error = Mock()
|
||||
mock_error.is_shared_link_already_exists.return_value = True
|
||||
|
||||
mock_existing_link = Mock()
|
||||
mock_existing_link.url = "https://dropbox.com/s/existing/file.mp4"
|
||||
|
||||
mock_links_result = Mock()
|
||||
mock_links_result.links = [mock_existing_link]
|
||||
|
||||
service = DropboxService()
|
||||
mock_dropbox = Mock()
|
||||
mock_dropbox.sharing_create_shared_link_with_settings.side_effect = \
|
||||
ApiError("req_id", mock_error, "message", "headers")
|
||||
mock_dropbox.sharing_list_shared_links.return_value = mock_links_result
|
||||
service._client = mock_dropbox
|
||||
|
||||
url = service.create_shared_link("/Uploads/file.mp4")
|
||||
|
||||
assert url == "https://dropbox.com/s/existing/file.mp4"
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
"""
|
||||
Tests for the flacfetch remote HTTP API client.
|
||||
|
||||
This module tests FlacfetchClient which handles communication with the
|
||||
dedicated flacfetch VM for torrent/audio downloads.
|
||||
|
||||
These tests use mocked HTTP responses to verify:
|
||||
- Request formatting (headers, payloads)
|
||||
- Response parsing
|
||||
- Error handling
|
||||
- State management (singleton pattern)
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, Mock, patch, MagicMock
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.services.flacfetch_client import (
|
||||
FlacfetchClient,
|
||||
FlacfetchServiceError,
|
||||
get_flacfetch_client,
|
||||
reset_flacfetch_client,
|
||||
)
|
||||
|
||||
|
||||
class TestFlacfetchClientInit:
|
||||
"""Test FlacfetchClient initialization."""
|
||||
|
||||
def test_init_with_required_params(self):
|
||||
"""Test client initializes with base_url and api_key."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key-123",
|
||||
)
|
||||
|
||||
assert client.base_url == "http://localhost:8080"
|
||||
assert client.api_key == "test-key-123"
|
||||
assert client.timeout == 60 # Default timeout
|
||||
|
||||
def test_init_strips_trailing_slash(self):
|
||||
"""Test base_url trailing slash is stripped."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080/",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
assert client.base_url == "http://localhost:8080"
|
||||
|
||||
def test_init_with_custom_timeout(self):
|
||||
"""Test custom timeout is set."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
assert client.timeout == 120
|
||||
|
||||
def test_headers_include_api_key(self):
|
||||
"""Test _headers() includes X-API-Key."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="secret-key-456",
|
||||
)
|
||||
|
||||
headers = client._headers()
|
||||
|
||||
assert headers["X-API-Key"] == "secret-key-456"
|
||||
assert headers["Content-Type"] == "application/json"
|
||||
|
||||
|
||||
class TestFlacfetchClientHealthCheck:
|
||||
"""Test health_check() method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_success(self):
|
||||
"""Test successful health check."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {
|
||||
"status": "healthy",
|
||||
"transmission": {"available": True, "version": "4.0.5"},
|
||||
"disk": {"free_gb": 15.5},
|
||||
}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = await client.health_check()
|
||||
|
||||
assert result["status"] == "healthy"
|
||||
assert result["transmission"]["available"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_degraded_status(self):
|
||||
"""Test health check with degraded status still succeeds."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"status": "degraded"}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = await client.health_check()
|
||||
|
||||
assert result["status"] == "degraded"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_unhealthy_raises_error(self):
|
||||
"""Test health check with unhealthy status raises error."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"status": "unhealthy", "error": "Transmission down"}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with pytest.raises(FlacfetchServiceError) as exc_info:
|
||||
await client.health_check()
|
||||
|
||||
assert "unhealthy" in str(exc_info.value)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_network_error(self):
|
||||
"""Test health check handles network errors."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(side_effect=httpx.RequestError("Connection refused"))
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with pytest.raises(FlacfetchServiceError) as exc_info:
|
||||
await client.health_check()
|
||||
|
||||
assert "Cannot reach" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestFlacfetchClientSearch:
|
||||
"""Test search() method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_success(self):
|
||||
"""Test successful search returns results."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"search_id": "search_abc123",
|
||||
"results": [
|
||||
{
|
||||
"title": "Waterloo",
|
||||
"artist": "ABBA",
|
||||
"provider": "Redacted",
|
||||
"quality": "FLAC 16bit CD",
|
||||
"seeders": 50,
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"results_count": 1,
|
||||
}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = await client.search("ABBA", "Waterloo")
|
||||
|
||||
assert result["search_id"] == "search_abc123"
|
||||
assert len(result["results"]) == 1
|
||||
assert result["results"][0]["title"] == "Waterloo"
|
||||
|
||||
# Verify request was made correctly
|
||||
mock_client.post.assert_called_once()
|
||||
call_args = mock_client.post.call_args
|
||||
assert call_args.kwargs["json"] == {"artist": "ABBA", "title": "Waterloo"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_no_results_returns_empty(self):
|
||||
"""Test search with no results returns empty list."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 404
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
result = await client.search("Unknown", "Artist")
|
||||
|
||||
assert result["results"] == []
|
||||
assert result["search_id"] is None
|
||||
|
||||
|
||||
class TestFlacfetchClientDownload:
|
||||
"""Test download() method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_starts_successfully(self):
|
||||
"""Test download() returns download_id."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {
|
||||
"download_id": "dl_xyz789",
|
||||
"status": "queued",
|
||||
}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
download_id = await client.download(
|
||||
search_id="search_abc123",
|
||||
result_index=0,
|
||||
)
|
||||
|
||||
assert download_id == "dl_xyz789"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_with_gcs_path(self):
|
||||
"""Test download() includes GCS upload parameters."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"download_id": "dl_xyz789"}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
await client.download(
|
||||
search_id="search_abc123",
|
||||
result_index=0,
|
||||
gcs_path="uploads/job123/audio/",
|
||||
)
|
||||
|
||||
# Verify GCS params were included
|
||||
call_args = mock_client.post.call_args
|
||||
payload = call_args.kwargs["json"]
|
||||
assert payload["upload_to_gcs"] is True
|
||||
assert payload["gcs_path"] == "uploads/job123/audio/"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_with_custom_filename(self):
|
||||
"""Test download() with custom output filename."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"download_id": "dl_xyz789"}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
await client.download(
|
||||
search_id="search_abc123",
|
||||
result_index=0,
|
||||
output_filename="ABBA - Waterloo",
|
||||
)
|
||||
|
||||
call_args = mock_client.post.call_args
|
||||
payload = call_args.kwargs["json"]
|
||||
assert payload["output_filename"] == "ABBA - Waterloo"
|
||||
|
||||
|
||||
class TestFlacfetchClientGetDownloadStatus:
|
||||
"""Test get_download_status() method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_download_status_success(self):
|
||||
"""Test getting download status."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {
|
||||
"download_id": "dl_xyz789",
|
||||
"status": "downloading",
|
||||
"progress": 45.5,
|
||||
"download_speed_kbps": 1250.0,
|
||||
"peers": 5,
|
||||
}
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
status = await client.get_download_status("dl_xyz789")
|
||||
|
||||
assert status["status"] == "downloading"
|
||||
assert status["progress"] == 45.5
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_download_status_not_found(self):
|
||||
"""Test getting status for non-existent download."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 404
|
||||
mock_response.raise_for_status = Mock(
|
||||
side_effect=httpx.HTTPStatusError("Not found", request=Mock(), response=mock_response)
|
||||
)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with pytest.raises(FlacfetchServiceError) as exc_info:
|
||||
await client.get_download_status("nonexistent")
|
||||
|
||||
assert "not found" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
class TestFlacfetchClientWaitForDownload:
|
||||
"""Test wait_for_download() method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_download_completes(self):
|
||||
"""Test waiting for download to complete."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
# Mock get_download_status to return complete status
|
||||
async def mock_get_status(download_id):
|
||||
return {
|
||||
"download_id": download_id,
|
||||
"status": "complete",
|
||||
"gcs_path": "gs://bucket/uploads/test.flac",
|
||||
}
|
||||
|
||||
client.get_download_status = mock_get_status
|
||||
|
||||
result = await client.wait_for_download("dl_xyz789", timeout=10)
|
||||
|
||||
assert result["status"] == "complete"
|
||||
assert result["gcs_path"] == "gs://bucket/uploads/test.flac"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_download_seeding_status_completes(self):
|
||||
"""Test that 'seeding' status is treated as complete."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
async def mock_get_status(download_id):
|
||||
return {
|
||||
"download_id": download_id,
|
||||
"status": "seeding",
|
||||
"output_path": "/downloads/test.flac",
|
||||
}
|
||||
|
||||
client.get_download_status = mock_get_status
|
||||
|
||||
result = await client.wait_for_download("dl_xyz789", timeout=10)
|
||||
|
||||
assert result["status"] == "seeding"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_download_failed(self):
|
||||
"""Test wait raises error when download fails."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
async def mock_get_status(download_id):
|
||||
return {
|
||||
"download_id": download_id,
|
||||
"status": "failed",
|
||||
"error": "Torrent download failed: no peers",
|
||||
}
|
||||
|
||||
client.get_download_status = mock_get_status
|
||||
|
||||
with pytest.raises(FlacfetchServiceError) as exc_info:
|
||||
await client.wait_for_download("dl_xyz789", timeout=10)
|
||||
|
||||
assert "failed" in str(exc_info.value).lower()
|
||||
assert "no peers" in str(exc_info.value)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_download_cancelled(self):
|
||||
"""Test wait raises error when download is cancelled."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
async def mock_get_status(download_id):
|
||||
return {
|
||||
"download_id": download_id,
|
||||
"status": "cancelled",
|
||||
}
|
||||
|
||||
client.get_download_status = mock_get_status
|
||||
|
||||
with pytest.raises(FlacfetchServiceError) as exc_info:
|
||||
await client.wait_for_download("dl_xyz789", timeout=10)
|
||||
|
||||
assert "cancelled" in str(exc_info.value).lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_download_timeout(self):
|
||||
"""Test wait raises error on timeout."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
call_count = 0
|
||||
async def mock_get_status(download_id):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return {
|
||||
"download_id": download_id,
|
||||
"status": "downloading",
|
||||
"progress": 10.0 * call_count,
|
||||
}
|
||||
|
||||
client.get_download_status = mock_get_status
|
||||
|
||||
with pytest.raises(FlacfetchServiceError) as exc_info:
|
||||
# Very short timeout with short poll interval
|
||||
await client.wait_for_download("dl_xyz789", timeout=0.2, poll_interval=0.1)
|
||||
|
||||
assert "timed out" in str(exc_info.value).lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_download_calls_progress_callback(self):
|
||||
"""Test progress callback is called during wait."""
|
||||
client = FlacfetchClient(
|
||||
base_url="http://localhost:8080",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
call_count = 0
|
||||
async def mock_get_status(download_id):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count >= 3:
|
||||
return {"status": "complete", "gcs_path": "gs://test"}
|
||||
return {"status": "downloading", "progress": 33.0 * call_count}
|
||||
|
||||
client.get_download_status = mock_get_status
|
||||
|
||||
progress_updates = []
|
||||
def progress_callback(status):
|
||||
progress_updates.append(status)
|
||||
|
||||
await client.wait_for_download(
|
||||
"dl_xyz789",
|
||||
timeout=10,
|
||||
poll_interval=0.01,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
# Should have received progress updates
|
||||
assert len(progress_updates) >= 2
|
||||
|
||||
|
||||
class TestGetFlacfetchClient:
|
||||
"""Test get_flacfetch_client() singleton factory."""
|
||||
|
||||
def test_returns_none_when_not_configured(self):
|
||||
"""Test returns None when FLACFETCH_API_URL not set."""
|
||||
reset_flacfetch_client()
|
||||
|
||||
with patch("backend.config.get_settings") as mock_settings:
|
||||
mock_settings.return_value = Mock(
|
||||
flacfetch_api_url=None,
|
||||
flacfetch_api_key=None,
|
||||
)
|
||||
|
||||
client = get_flacfetch_client()
|
||||
|
||||
assert client is None
|
||||
|
||||
def test_returns_none_when_key_missing(self):
|
||||
"""Test returns None when API key not set."""
|
||||
reset_flacfetch_client()
|
||||
|
||||
with patch("backend.config.get_settings") as mock_settings:
|
||||
mock_settings.return_value = Mock(
|
||||
flacfetch_api_url="http://localhost:8080",
|
||||
flacfetch_api_key=None,
|
||||
)
|
||||
|
||||
client = get_flacfetch_client()
|
||||
|
||||
assert client is None
|
||||
|
||||
def test_returns_client_when_configured(self):
|
||||
"""Test returns FlacfetchClient when both URL and key are set."""
|
||||
reset_flacfetch_client()
|
||||
|
||||
with patch("backend.config.get_settings") as mock_settings:
|
||||
mock_settings.return_value = Mock(
|
||||
flacfetch_api_url="http://10.0.0.5:8080",
|
||||
flacfetch_api_key="secret-key",
|
||||
)
|
||||
|
||||
client = get_flacfetch_client()
|
||||
|
||||
assert client is not None
|
||||
assert isinstance(client, FlacfetchClient)
|
||||
assert client.base_url == "http://10.0.0.5:8080"
|
||||
assert client.api_key == "secret-key"
|
||||
|
||||
def test_returns_same_instance(self):
|
||||
"""Test singleton returns same instance."""
|
||||
reset_flacfetch_client()
|
||||
|
||||
with patch("backend.config.get_settings") as mock_settings:
|
||||
mock_settings.return_value = Mock(
|
||||
flacfetch_api_url="http://localhost:8080",
|
||||
flacfetch_api_key="test-key",
|
||||
)
|
||||
|
||||
client1 = get_flacfetch_client()
|
||||
client2 = get_flacfetch_client()
|
||||
|
||||
assert client1 is client2
|
||||
|
||||
def test_reset_clears_singleton(self):
|
||||
"""Test reset_flacfetch_client() clears the singleton."""
|
||||
reset_flacfetch_client()
|
||||
|
||||
with patch("backend.config.get_settings") as mock_settings:
|
||||
mock_settings.return_value = Mock(
|
||||
flacfetch_api_url="http://localhost:8080",
|
||||
flacfetch_api_key="test-key",
|
||||
)
|
||||
|
||||
client1 = get_flacfetch_client()
|
||||
reset_flacfetch_client()
|
||||
|
||||
# Change the settings
|
||||
mock_settings.return_value = Mock(
|
||||
flacfetch_api_url="http://other:8080",
|
||||
flacfetch_api_key="other-key",
|
||||
)
|
||||
|
||||
client2 = get_flacfetch_client()
|
||||
|
||||
assert client1 is not client2
|
||||
assert client2.base_url == "http://other:8080"
|
||||
|
||||
@@ -0,0 +1,524 @@
|
||||
"""
|
||||
Tests for gdrive_service.py - Google Drive file operations.
|
||||
|
||||
These tests mock the Google API client and Secret Manager to verify:
|
||||
- Credential loading from Secret Manager
|
||||
- Folder creation and lookup
|
||||
- File uploads with proper MIME types
|
||||
- Public share folder structure uploads
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
|
||||
|
||||
class TestGoogleDriveServiceInit:
|
||||
"""Test GoogleDriveService initialization."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_init_creates_service(self, mock_get_settings):
|
||||
"""Test initialization creates service with settings."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
assert service.settings == mock_settings
|
||||
assert service._service is None
|
||||
assert service._credentials_data is None
|
||||
assert service._loaded is False
|
||||
|
||||
|
||||
class TestLoadCredentials:
|
||||
"""Test _load_credentials method."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_load_credentials_success(self, mock_get_settings):
|
||||
"""Test successful credential loading from Secret Manager."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"token": "access-token",
|
||||
"refresh_token": "refresh-token",
|
||||
"client_id": "client-id",
|
||||
"client_secret": "client-secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
creds = service._load_credentials()
|
||||
|
||||
assert creds is not None
|
||||
assert creds["refresh_token"] == "refresh-token"
|
||||
mock_settings.get_secret.assert_called_once_with("gdrive-oauth-credentials")
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_load_credentials_fallback_to_youtube(self, mock_get_settings):
|
||||
"""Test fallback to YouTube credentials when Drive creds not found."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
# First call returns None (Drive creds not found)
|
||||
# Second call returns YouTube creds
|
||||
mock_settings.get_secret.side_effect = [
|
||||
None,
|
||||
json.dumps({
|
||||
"refresh_token": "youtube-token",
|
||||
"client_id": "youtube-id",
|
||||
"client_secret": "youtube-secret",
|
||||
})
|
||||
]
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
creds = service._load_credentials()
|
||||
|
||||
assert creds is not None
|
||||
assert creds["refresh_token"] == "youtube-token"
|
||||
assert mock_settings.get_secret.call_count == 2
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_load_credentials_not_found(self, mock_get_settings):
|
||||
"""Test handling when no credentials found."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
creds = service._load_credentials()
|
||||
|
||||
assert creds is None
|
||||
assert service._loaded is True
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_load_credentials_missing_required_fields(self, mock_get_settings):
|
||||
"""Test handling when credentials missing required fields."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"token": "access-token",
|
||||
# Missing: refresh_token, client_id, client_secret
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
creds = service._load_credentials()
|
||||
|
||||
assert creds is None
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_load_credentials_cached(self, mock_get_settings):
|
||||
"""Test credentials are cached after first load."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
creds1 = service._load_credentials()
|
||||
creds2 = service._load_credentials()
|
||||
|
||||
assert creds1 == creds2
|
||||
assert mock_settings.get_secret.call_count == 1
|
||||
|
||||
|
||||
class TestIsConfigured:
|
||||
"""Test is_configured property."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_is_configured_true(self, mock_get_settings):
|
||||
"""Test is_configured returns True when credentials exist."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
assert service.is_configured is True
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_is_configured_false(self, mock_get_settings):
|
||||
"""Test is_configured returns False when no credentials."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
assert service.is_configured is False
|
||||
|
||||
|
||||
class TestDriveService:
|
||||
"""Test service property."""
|
||||
|
||||
@patch("googleapiclient.discovery.build")
|
||||
@patch("google.oauth2.credentials.Credentials")
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_service_creates_drive_client(
|
||||
self, mock_get_settings, mock_creds_class, mock_build
|
||||
):
|
||||
"""Test service property creates Google Drive API client."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"token": "access-token",
|
||||
"refresh_token": "refresh-token",
|
||||
"client_id": "client-id",
|
||||
"client_secret": "client-secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
mock_creds = Mock()
|
||||
mock_creds_class.return_value = mock_creds
|
||||
|
||||
mock_drive_service = Mock()
|
||||
mock_build.return_value = mock_drive_service
|
||||
|
||||
service = GoogleDriveService()
|
||||
result = service.service
|
||||
|
||||
mock_build.assert_called_once_with("drive", "v3", credentials=mock_creds)
|
||||
assert result == mock_drive_service
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_service_raises_on_missing_credentials(self, mock_get_settings):
|
||||
"""Test service raises RuntimeError when credentials missing."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_ = service.service
|
||||
|
||||
assert "not configured" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestGetOrCreateFolder:
|
||||
"""Test get_or_create_folder method."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_get_existing_folder(self, mock_get_settings):
|
||||
"""Test finding an existing folder."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
# Mock the Drive API
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.return_value = {
|
||||
"files": [{"id": "folder-id-123", "name": "MP4"}]
|
||||
}
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
folder_id = service.get_or_create_folder("parent-123", "MP4")
|
||||
|
||||
assert folder_id == "folder-id-123"
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_create_new_folder(self, mock_get_settings):
|
||||
"""Test creating a new folder when it doesn't exist."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
# Mock the Drive API
|
||||
mock_files_api = Mock()
|
||||
|
||||
# list returns empty (folder doesn't exist)
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.return_value = {"files": []}
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
# create returns new folder
|
||||
mock_create_result = Mock()
|
||||
mock_create_result.execute.return_value = {"id": "new-folder-id"}
|
||||
mock_files_api.create.return_value = mock_create_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
folder_id = service.get_or_create_folder("parent-123", "NewFolder")
|
||||
|
||||
assert folder_id == "new-folder-id"
|
||||
mock_files_api.create.assert_called_once()
|
||||
|
||||
|
||||
class TestUploadFile:
|
||||
"""Test upload_file method."""
|
||||
|
||||
@patch("googleapiclient.http.MediaFileUpload")
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_upload_file_success(self, mock_get_settings, mock_media, tmp_path):
|
||||
"""Test uploading a file to Drive."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
# Create test file
|
||||
test_file = tmp_path / "test.mp4"
|
||||
test_file.write_bytes(b"video content")
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
|
||||
# list returns empty (no existing file)
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.return_value = {"files": []}
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
# create returns new file
|
||||
mock_create_result = Mock()
|
||||
mock_create_result.execute.return_value = {"id": "file-id-123"}
|
||||
mock_files_api.create.return_value = mock_create_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
file_id = service.upload_file(str(test_file), "parent-123", "video.mp4")
|
||||
|
||||
assert file_id == "file-id-123"
|
||||
mock_media.assert_called_once()
|
||||
# Verify MIME type was set correctly
|
||||
call_kwargs = mock_media.call_args.kwargs
|
||||
assert call_kwargs["mimetype"] == "video/mp4"
|
||||
|
||||
@patch("googleapiclient.http.MediaFileUpload")
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_upload_file_replaces_existing(self, mock_get_settings, mock_media, tmp_path):
|
||||
"""Test upload deletes existing file before upload."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
test_file = tmp_path / "test.mp4"
|
||||
test_file.write_bytes(b"content")
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
|
||||
# list returns existing file
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.return_value = {
|
||||
"files": [{"id": "existing-file-id"}]
|
||||
}
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_delete_result = Mock()
|
||||
mock_delete_result.execute.return_value = {}
|
||||
mock_files_api.delete.return_value = mock_delete_result
|
||||
|
||||
mock_create_result = Mock()
|
||||
mock_create_result.execute.return_value = {"id": "new-file-id"}
|
||||
mock_files_api.create.return_value = mock_create_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
file_id = service.upload_file(str(test_file), "parent-123", "video.mp4")
|
||||
|
||||
# Should have deleted existing file
|
||||
mock_files_api.delete.assert_called_once_with(fileId="existing-file-id")
|
||||
assert file_id == "new-file-id"
|
||||
|
||||
|
||||
class TestUploadToPublicShare:
|
||||
"""Test upload_to_public_share method."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_upload_to_public_share(self, mock_get_settings, tmp_path):
|
||||
"""Test uploading files to public share folder structure."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
# Create test files
|
||||
mp4_file = tmp_path / "output.mp4"
|
||||
mp4_file.write_bytes(b"4k video")
|
||||
mp4_720_file = tmp_path / "output_720p.mp4"
|
||||
mp4_720_file.write_bytes(b"720p video")
|
||||
cdg_file = tmp_path / "output.zip"
|
||||
cdg_file.write_bytes(b"cdg package")
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
# Mock methods
|
||||
with patch.object(service, "get_or_create_folder") as mock_get_folder:
|
||||
with patch.object(service, "upload_file") as mock_upload:
|
||||
mock_get_folder.side_effect = [
|
||||
"mp4-folder-id",
|
||||
"mp4-720-folder-id",
|
||||
"cdg-folder-id",
|
||||
]
|
||||
mock_upload.side_effect = [
|
||||
"mp4-file-id",
|
||||
"720p-file-id",
|
||||
"cdg-file-id",
|
||||
]
|
||||
|
||||
output_files = {
|
||||
"final_karaoke_lossy_mp4": str(mp4_file),
|
||||
"final_karaoke_lossy_720p_mp4": str(mp4_720_file),
|
||||
"final_karaoke_cdg_zip": str(cdg_file),
|
||||
}
|
||||
|
||||
result = service.upload_to_public_share(
|
||||
root_folder_id="root-123",
|
||||
brand_code="NOMAD-1163",
|
||||
base_name="Artist - Title",
|
||||
output_files=output_files,
|
||||
)
|
||||
|
||||
# Should have created/found 3 folders
|
||||
assert mock_get_folder.call_count == 3
|
||||
|
||||
# Should have uploaded 3 files
|
||||
assert mock_upload.call_count == 3
|
||||
|
||||
# Check result
|
||||
assert result["mp4"] == "mp4-file-id"
|
||||
assert result["mp4_720p"] == "720p-file-id"
|
||||
assert result["cdg"] == "cdg-file-id"
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_upload_to_public_share_skips_missing_files(
|
||||
self, mock_get_settings, tmp_path
|
||||
):
|
||||
"""Test upload skips files that don't exist."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
# Only create one file
|
||||
mp4_file = tmp_path / "output.mp4"
|
||||
mp4_file.write_bytes(b"video")
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
with patch.object(service, "get_or_create_folder") as mock_get_folder:
|
||||
with patch.object(service, "upload_file") as mock_upload:
|
||||
mock_get_folder.return_value = "folder-id"
|
||||
mock_upload.return_value = "file-id"
|
||||
|
||||
output_files = {
|
||||
"final_karaoke_lossy_mp4": str(mp4_file),
|
||||
"final_karaoke_lossy_720p_mp4": "/nonexistent/file.mp4",
|
||||
"final_karaoke_cdg_zip": None,
|
||||
}
|
||||
|
||||
result = service.upload_to_public_share(
|
||||
root_folder_id="root-123",
|
||||
brand_code="CODE",
|
||||
base_name="Name",
|
||||
output_files=output_files,
|
||||
)
|
||||
|
||||
# Should only upload the one file that exists
|
||||
assert mock_upload.call_count == 1
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
class TestGetGdriveService:
|
||||
"""Test get_gdrive_service singleton."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_get_gdrive_service_singleton(self, mock_get_settings):
|
||||
"""Test get_gdrive_service returns singleton instance."""
|
||||
from backend.services.gdrive_service import get_gdrive_service
|
||||
import backend.services.gdrive_service as gdrive_module
|
||||
|
||||
# Reset singleton
|
||||
gdrive_module._gdrive_service = None
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service1 = get_gdrive_service()
|
||||
service2 = get_gdrive_service()
|
||||
|
||||
assert service1 is service2
|
||||
|
||||
@@ -0,0 +1,448 @@
|
||||
"""
|
||||
Tests for storage_service.py - Google Cloud Storage operations.
|
||||
|
||||
These tests mock the GCS client to verify:
|
||||
- Upload operations (file, fileobj, JSON)
|
||||
- Download operations (file, JSON)
|
||||
- Signed URL generation
|
||||
- File operations (delete, list, exists)
|
||||
"""
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
from io import BytesIO
|
||||
|
||||
from backend.services.storage_service import StorageService
|
||||
|
||||
|
||||
class TestStorageServiceInit:
|
||||
"""Test StorageService initialization."""
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_init_creates_client_and_bucket(self, mock_settings, mock_client_class):
|
||||
"""Test initialization creates GCS client and gets bucket."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_client = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
|
||||
mock_client_class.assert_called_once_with(project="test-project")
|
||||
mock_client.bucket.assert_called_once_with("test-bucket")
|
||||
assert service.client == mock_client
|
||||
assert service.bucket == mock_bucket
|
||||
|
||||
|
||||
class TestStorageServiceUpload:
|
||||
"""Test upload operations."""
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_upload_file(self, mock_settings, mock_client_class):
|
||||
"""Test uploading a file to GCS."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
result = service.upload_file("/local/path/file.flac", "uploads/job123/file.flac")
|
||||
|
||||
mock_bucket.blob.assert_called_once_with("uploads/job123/file.flac")
|
||||
mock_blob.upload_from_filename.assert_called_once_with("/local/path/file.flac")
|
||||
assert result == "uploads/job123/file.flac"
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_upload_file_raises_on_error(self, mock_settings, mock_client_class):
|
||||
"""Test upload_file raises exception on GCS error."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.upload_from_filename.side_effect = Exception("GCS error")
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
service.upload_file("/local/path/file.flac", "dest/file.flac")
|
||||
|
||||
assert "GCS error" in str(exc_info.value)
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_upload_fileobj(self, mock_settings, mock_client_class):
|
||||
"""Test uploading a file object to GCS."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
file_obj = BytesIO(b"test content")
|
||||
|
||||
result = service.upload_fileobj(file_obj, "uploads/test.txt", content_type="text/plain")
|
||||
|
||||
mock_bucket.blob.assert_called_once_with("uploads/test.txt")
|
||||
assert mock_blob.content_type == "text/plain"
|
||||
mock_blob.upload_from_file.assert_called_once_with(file_obj, rewind=True)
|
||||
assert result == "uploads/test.txt"
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_upload_json(self, mock_settings, mock_client_class):
|
||||
"""Test uploading JSON data to GCS."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
data = {"key": "value", "nested": {"data": True}}
|
||||
|
||||
result = service.upload_json("data/config.json", data)
|
||||
|
||||
mock_bucket.blob.assert_called_once_with("data/config.json")
|
||||
assert mock_blob.content_type == "application/json"
|
||||
# Check the uploaded content
|
||||
call_args = mock_blob.upload_from_string.call_args
|
||||
uploaded_content = call_args[0][0]
|
||||
assert json.loads(uploaded_content) == data
|
||||
assert result == "data/config.json"
|
||||
|
||||
|
||||
class TestStorageServiceDownload:
|
||||
"""Test download operations."""
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_download_file(self, mock_settings, mock_client_class):
|
||||
"""Test downloading a file from GCS."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
result = service.download_file("uploads/job123/file.flac", "/local/path/file.flac")
|
||||
|
||||
mock_bucket.blob.assert_called_once_with("uploads/job123/file.flac")
|
||||
mock_blob.download_to_filename.assert_called_once_with("/local/path/file.flac")
|
||||
assert result == "/local/path/file.flac"
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_download_file_raises_on_error(self, mock_settings, mock_client_class):
|
||||
"""Test download_file raises exception on GCS error."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.download_to_filename.side_effect = Exception("File not found")
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
service.download_file("missing/file.flac", "/local/path")
|
||||
|
||||
assert "File not found" in str(exc_info.value)
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_download_json(self, mock_settings, mock_client_class):
|
||||
"""Test downloading and parsing JSON from GCS."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.download_as_text.return_value = '{"key": "value", "count": 42}'
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
result = service.download_json("data/config.json")
|
||||
|
||||
assert result == {"key": "value", "count": 42}
|
||||
|
||||
|
||||
class TestStorageServiceDelete:
|
||||
"""Test delete operations."""
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_delete_file(self, mock_settings, mock_client_class):
|
||||
"""Test deleting a single file from GCS."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
service.delete_file("uploads/job123/file.flac")
|
||||
|
||||
mock_bucket.blob.assert_called_once_with("uploads/job123/file.flac")
|
||||
mock_blob.delete.assert_called_once()
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_delete_folder(self, mock_settings, mock_client_class):
|
||||
"""Test deleting all files with a prefix (folder)."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
# Create mock blobs
|
||||
mock_blob1 = Mock()
|
||||
mock_blob1.name = "uploads/job123/file1.flac"
|
||||
mock_blob2 = Mock()
|
||||
mock_blob2.name = "uploads/job123/file2.flac"
|
||||
mock_blob3 = Mock()
|
||||
mock_blob3.name = "uploads/job123/subdir/file3.flac"
|
||||
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.list_blobs.return_value = [mock_blob1, mock_blob2, mock_blob3]
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
count = service.delete_folder("uploads/job123/")
|
||||
|
||||
mock_bucket.list_blobs.assert_called_once_with(prefix="uploads/job123/")
|
||||
mock_blob1.delete.assert_called_once()
|
||||
mock_blob2.delete.assert_called_once()
|
||||
mock_blob3.delete.assert_called_once()
|
||||
assert count == 3
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_delete_folder_handles_errors_gracefully(self, mock_settings, mock_client_class):
|
||||
"""Test delete_folder continues even if some deletes fail."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob1 = Mock()
|
||||
mock_blob1.name = "file1.flac"
|
||||
mock_blob2 = Mock()
|
||||
mock_blob2.name = "file2.flac"
|
||||
mock_blob2.delete.side_effect = Exception("Permission denied")
|
||||
mock_blob3 = Mock()
|
||||
mock_blob3.name = "file3.flac"
|
||||
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.list_blobs.return_value = [mock_blob1, mock_blob2, mock_blob3]
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
count = service.delete_folder("uploads/")
|
||||
|
||||
# Should have deleted 2 files (blob2 failed)
|
||||
assert count == 2
|
||||
|
||||
|
||||
class TestStorageServiceFileOperations:
|
||||
"""Test file listing and existence checks."""
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_list_files(self, mock_settings, mock_client_class):
|
||||
"""Test listing files with a prefix."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob1 = Mock()
|
||||
mock_blob1.name = "uploads/job123/audio.flac"
|
||||
mock_blob2 = Mock()
|
||||
mock_blob2.name = "uploads/job123/lyrics.txt"
|
||||
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.list_blobs.return_value = [mock_blob1, mock_blob2]
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
files = service.list_files("uploads/job123/")
|
||||
|
||||
assert files == ["uploads/job123/audio.flac", "uploads/job123/lyrics.txt"]
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_file_exists_returns_true(self, mock_settings, mock_client_class):
|
||||
"""Test file_exists returns True for existing file."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.exists.return_value = True
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
result = service.file_exists("uploads/job123/file.flac")
|
||||
|
||||
assert result is True
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_file_exists_returns_false(self, mock_settings, mock_client_class):
|
||||
"""Test file_exists returns False for missing file."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.exists.return_value = False
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
service = StorageService()
|
||||
result = service.file_exists("missing/file.flac")
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestStorageServiceSignedUrls:
|
||||
"""Test signed URL generation."""
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_generate_signed_url(self, mock_settings, mock_client_class):
|
||||
"""Test generating a signed download URL."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.generate_signed_url.return_value = "https://storage.googleapis.com/signed-url"
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Mock google.auth.default to return credentials without service_account_email
|
||||
with patch("google.auth.default") as mock_auth_default:
|
||||
mock_credentials = Mock(spec=[]) # No service_account_email attr
|
||||
mock_auth_default.return_value = (mock_credentials, "test-project")
|
||||
|
||||
service = StorageService()
|
||||
url = service.generate_signed_url("uploads/file.flac", expiration_minutes=30)
|
||||
|
||||
assert url == "https://storage.googleapis.com/signed-url"
|
||||
mock_blob.generate_signed_url.assert_called_once()
|
||||
call_kwargs = mock_blob.generate_signed_url.call_args.kwargs
|
||||
assert call_kwargs["method"] == "GET"
|
||||
assert call_kwargs["version"] == "v4"
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_generate_signed_upload_url(self, mock_settings, mock_client_class):
|
||||
"""Test generating a signed upload URL."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.generate_signed_url.return_value = "https://storage.googleapis.com/signed-upload-url"
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch("google.auth.default") as mock_auth_default:
|
||||
mock_credentials = Mock(spec=[])
|
||||
mock_auth_default.return_value = (mock_credentials, "test-project")
|
||||
|
||||
service = StorageService()
|
||||
url = service.generate_signed_upload_url(
|
||||
"uploads/file.flac",
|
||||
content_type="audio/flac",
|
||||
expiration_minutes=15
|
||||
)
|
||||
|
||||
assert url == "https://storage.googleapis.com/signed-upload-url"
|
||||
call_kwargs = mock_blob.generate_signed_url.call_args.kwargs
|
||||
assert call_kwargs["method"] == "PUT"
|
||||
assert call_kwargs["headers"] == {"Content-Type": "audio/flac"}
|
||||
|
||||
@patch("backend.services.storage_service.storage.Client")
|
||||
@patch("backend.services.storage_service.settings")
|
||||
def test_signed_url_with_service_account(self, mock_settings, mock_client_class):
|
||||
"""Test signed URL generation with service account credentials."""
|
||||
mock_settings.google_cloud_project = "test-project"
|
||||
mock_settings.gcs_bucket_name = "test-bucket"
|
||||
|
||||
mock_blob = Mock()
|
||||
mock_blob.generate_signed_url.return_value = "https://storage.googleapis.com/signed-url"
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
mock_client = Mock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch("google.auth.default") as mock_auth_default:
|
||||
with patch("google.auth.transport.requests.Request") as mock_request:
|
||||
# Mock credentials with service_account_email
|
||||
mock_credentials = Mock()
|
||||
mock_credentials.service_account_email = "sa@project.iam.gserviceaccount.com"
|
||||
mock_credentials.token = "access-token-123"
|
||||
mock_auth_default.return_value = (mock_credentials, "test-project")
|
||||
|
||||
service = StorageService()
|
||||
url = service.generate_signed_url("uploads/file.flac")
|
||||
|
||||
# Should have refreshed credentials and used IAM signing
|
||||
mock_credentials.refresh.assert_called_once()
|
||||
call_kwargs = mock_blob.generate_signed_url.call_args.kwargs
|
||||
assert call_kwargs["service_account_email"] == "sa@project.iam.gserviceaccount.com"
|
||||
assert call_kwargs["access_token"] == "access-token-123"
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
"""
|
||||
Tests for youtube_service.py - YouTube credential management.
|
||||
|
||||
These tests mock Secret Manager to verify:
|
||||
- Credential loading and parsing
|
||||
- Validation of required fields
|
||||
- Error handling for missing/invalid credentials
|
||||
"""
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
class TestYouTubeServiceInit:
|
||||
"""Test YouTubeService initialization."""
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_init_creates_service(self, mock_get_settings):
|
||||
"""Test initialization creates service with settings."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
|
||||
assert service.settings == mock_settings
|
||||
assert service._credentials is None
|
||||
assert service._loaded is False
|
||||
|
||||
|
||||
class TestLoadCredentials:
|
||||
"""Test load_credentials method."""
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_load_credentials_success(self, mock_get_settings):
|
||||
"""Test successful credential loading from Secret Manager."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"token": "access-token-123",
|
||||
"refresh_token": "refresh-token-456",
|
||||
"token_uri": "https://oauth2.googleapis.com/token",
|
||||
"client_id": "client-id.apps.googleusercontent.com",
|
||||
"client_secret": "client-secret-789",
|
||||
"scopes": ["https://www.googleapis.com/auth/youtube.upload"],
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
result = service.load_credentials()
|
||||
|
||||
assert result is True
|
||||
assert service._credentials is not None
|
||||
assert service._credentials["refresh_token"] == "refresh-token-456"
|
||||
mock_settings.get_secret.assert_called_once_with("youtube-oauth-credentials")
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_load_credentials_not_found(self, mock_get_settings):
|
||||
"""Test handling when credentials are not in Secret Manager."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
result = service.load_credentials()
|
||||
|
||||
assert result is False
|
||||
assert service._credentials is None
|
||||
assert service._loaded is True
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_load_credentials_missing_required_fields(self, mock_get_settings):
|
||||
"""Test handling when credentials are missing required fields."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"token": "access-token",
|
||||
# Missing: refresh_token, token_uri, client_id, client_secret
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
result = service.load_credentials()
|
||||
|
||||
assert result is False
|
||||
assert service._credentials is None
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_load_credentials_invalid_json(self, mock_get_settings):
|
||||
"""Test handling of invalid JSON in credentials."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = "not valid json {"
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
result = service.load_credentials()
|
||||
|
||||
assert result is False
|
||||
assert service._credentials is None
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_load_credentials_exception(self, mock_get_settings):
|
||||
"""Test handling of exceptions during credential loading."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.side_effect = Exception("Secret Manager error")
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
result = service.load_credentials()
|
||||
|
||||
assert result is False
|
||||
assert service._loaded is True
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_load_credentials_cached(self, mock_get_settings):
|
||||
"""Test that credentials are cached after first load."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"token_uri": "uri",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
|
||||
# First call
|
||||
result1 = service.load_credentials()
|
||||
# Second call
|
||||
result2 = service.load_credentials()
|
||||
|
||||
assert result1 is True
|
||||
assert result2 is True
|
||||
# Secret Manager should only be called once
|
||||
assert mock_settings.get_secret.call_count == 1
|
||||
|
||||
|
||||
class TestGetCredentialsDict:
|
||||
"""Test get_credentials_dict method."""
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_get_credentials_dict_loads_if_needed(self, mock_get_settings):
|
||||
"""Test get_credentials_dict loads credentials if not already loaded."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"token_uri": "uri",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
|
||||
# Don't explicitly call load_credentials
|
||||
result = service.get_credentials_dict()
|
||||
|
||||
assert result is not None
|
||||
assert result["refresh_token"] == "token"
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_get_credentials_dict_returns_none_if_not_configured(
|
||||
self, mock_get_settings
|
||||
):
|
||||
"""Test get_credentials_dict returns None when not configured."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
result = service.get_credentials_dict()
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestIsConfigured:
|
||||
"""Test is_configured property."""
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_is_configured_true(self, mock_get_settings):
|
||||
"""Test is_configured returns True when credentials exist."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token",
|
||||
"token_uri": "uri",
|
||||
"client_id": "id",
|
||||
"client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
|
||||
assert service.is_configured is True
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_is_configured_false(self, mock_get_settings):
|
||||
"""Test is_configured returns False when credentials missing."""
|
||||
from backend.services.youtube_service import YouTubeService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = YouTubeService()
|
||||
|
||||
assert service.is_configured is False
|
||||
|
||||
|
||||
class TestGetYouTubeService:
|
||||
"""Test get_youtube_service singleton."""
|
||||
|
||||
@patch("backend.services.youtube_service.get_settings")
|
||||
def test_get_youtube_service_singleton(self, mock_get_settings):
|
||||
"""Test get_youtube_service returns singleton instance."""
|
||||
from backend.services.youtube_service import get_youtube_service
|
||||
import backend.services.youtube_service as youtube_module
|
||||
|
||||
# Reset singleton
|
||||
youtube_module._youtube_service = None
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service1 = get_youtube_service()
|
||||
service2 = get_youtube_service()
|
||||
|
||||
assert service1 is service2
|
||||
|
||||
+119
-86
@@ -7,11 +7,17 @@ Handles the audio processing track of parallel processing:
|
||||
3. Post-processing: Combine instrumentals, normalize
|
||||
|
||||
Re-uses karaoke_gen.audio_processor.AudioProcessor for remote GPU separation.
|
||||
|
||||
Observability:
|
||||
- All operations wrapped in tracing spans for Cloud Trace visibility
|
||||
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
||||
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Optional, Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
@@ -20,6 +26,8 @@ from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.config import get_settings
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
from backend.services.metrics import metrics
|
||||
|
||||
# Import from karaoke_gen package
|
||||
from karaoke_gen.audio_processor import AudioProcessor
|
||||
@@ -215,6 +223,7 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
start_time = time.time()
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
settings = get_settings()
|
||||
@@ -222,7 +231,8 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
# Create job logger for remote debugging FIRST
|
||||
job_log = create_job_logger(job_id, "audio")
|
||||
|
||||
# Log immediately to verify logging is working
|
||||
# Log with structured markers for easy Cloud Logging queries
|
||||
logger.info(f"[job:{job_id}] WORKER_START worker=audio")
|
||||
job_log.info("=== AUDIO WORKER STARTED ===")
|
||||
job_log.info(f"Job ID: {job_id}")
|
||||
|
||||
@@ -232,7 +242,7 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
@@ -241,91 +251,114 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
|
||||
try:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (karaoke_gen.audio_processor) are
|
||||
# only captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job_log.info(f"Starting audio separation for {job.artist} - {job.title}")
|
||||
logger.info(f"Starting audio separation for job {job_id}")
|
||||
|
||||
# Ensure AUDIO_SEPARATOR_API_URL is set
|
||||
api_url = os.environ.get("AUDIO_SEPARATOR_API_URL")
|
||||
if not api_url:
|
||||
raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. "
|
||||
"Cannot perform audio separation without remote API access.")
|
||||
job_log.info(f"Audio separator API: {api_url}")
|
||||
|
||||
# Download audio file from GCS or URL
|
||||
job_log.info("Downloading audio file...")
|
||||
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager)
|
||||
if not audio_path:
|
||||
raise Exception("Failed to download audio file")
|
||||
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
# The status is managed at a higher level - workers just track their progress
|
||||
job_manager.update_state_data(job_id, 'audio_progress', {
|
||||
'stage': 'separating_stage1',
|
||||
'progress': 10,
|
||||
'message': 'Starting audio separation (Stage 1: Clean instrumental)'
|
||||
})
|
||||
|
||||
# Create AudioProcessor instance (reuses karaoke_gen code)
|
||||
# Use model configuration from job if provided, otherwise use defaults
|
||||
job_log.info("Creating AudioProcessor instance...")
|
||||
if job.clean_instrumental_model:
|
||||
job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}")
|
||||
if job.backing_vocals_models:
|
||||
job_log.info(f" Using backing vocals models: {job.backing_vocals_models}")
|
||||
if job.other_stems_models:
|
||||
job_log.info(f" Using other stems models: {job.other_stems_models}")
|
||||
|
||||
audio_processor = create_audio_processor(
|
||||
temp_dir,
|
||||
clean_instrumental_model=job.clean_instrumental_model,
|
||||
backing_vocals_models=job.backing_vocals_models,
|
||||
other_stems_models=job.other_stems_models
|
||||
)
|
||||
|
||||
# Format artist-title for file naming (matches CLI behavior)
|
||||
artist_title = f"{job.artist} - {job.title}"
|
||||
|
||||
# Run audio separation (calls Modal API internally)
|
||||
# This returns a dict with paths to all separated stems
|
||||
job_log.info("Starting audio separation (this may take 5-10 minutes)...")
|
||||
job_log.info(" Stage 1: Clean instrumental separation (MDX models)")
|
||||
job_log.info(" Stage 2: Backing vocals separation (Demucs model)")
|
||||
logger.info(f"Job {job_id}: Calling audio_processor.process_audio_separation()")
|
||||
separation_result = audio_processor.process_audio_separation(
|
||||
audio_file=audio_path,
|
||||
artist_title=artist_title,
|
||||
track_output_dir=temp_dir
|
||||
)
|
||||
|
||||
job_log.info("Audio separation complete!")
|
||||
job_log.info(f" Generated {len(separation_result)} stem files")
|
||||
logger.info(f"Job {job_id}: Audio separation complete, organizing results")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
job_manager.update_state_data(job_id, 'audio_progress', {
|
||||
'stage': 'audio_complete',
|
||||
'progress': 45,
|
||||
'message': 'Audio separation complete, uploading stems'
|
||||
})
|
||||
|
||||
# Upload all stems to GCS
|
||||
await upload_separation_results(job_id, separation_result, storage, job_manager)
|
||||
|
||||
logger.info(f"Job {job_id}: All stems uploaded successfully")
|
||||
|
||||
# Mark audio processing complete
|
||||
# This will check if lyrics are also complete and transition to next stage if so
|
||||
job_manager.mark_audio_complete(job_id)
|
||||
|
||||
return True
|
||||
# Wrap entire worker in a tracing span
|
||||
with job_span("audio-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (karaoke_gen.audio_processor) are
|
||||
# only captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job_log.info(f"Starting audio separation for {job.artist} - {job.title}")
|
||||
logger.info(f"[job:{job_id}] Starting audio separation for {job.artist} - {job.title}")
|
||||
|
||||
# Ensure AUDIO_SEPARATOR_API_URL is set
|
||||
api_url = os.environ.get("AUDIO_SEPARATOR_API_URL")
|
||||
if not api_url:
|
||||
raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. "
|
||||
"Cannot perform audio separation without remote API access.")
|
||||
job_log.info(f"Audio separator API: {api_url}")
|
||||
add_span_attribute("audio_separator_api", api_url)
|
||||
|
||||
# Download audio file from GCS or URL
|
||||
with job_span("download-audio", job_id) as download_span:
|
||||
job_log.info("Downloading audio file...")
|
||||
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager)
|
||||
if not audio_path:
|
||||
raise Exception("Failed to download audio file")
|
||||
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
|
||||
download_span.set_attribute("audio_file", os.path.basename(audio_path))
|
||||
download_span.set_attribute("source", "url" if job.url else "gcs")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
# The status is managed at a higher level - workers just track their progress
|
||||
job_manager.update_state_data(job_id, 'audio_progress', {
|
||||
'stage': 'separating_stage1',
|
||||
'progress': 10,
|
||||
'message': 'Starting audio separation (Stage 1: Clean instrumental)'
|
||||
})
|
||||
|
||||
# Create AudioProcessor instance (reuses karaoke_gen code)
|
||||
# Use model configuration from job if provided, otherwise use defaults
|
||||
job_log.info("Creating AudioProcessor instance...")
|
||||
if job.clean_instrumental_model:
|
||||
job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}")
|
||||
add_span_attribute("clean_model", job.clean_instrumental_model)
|
||||
if job.backing_vocals_models:
|
||||
job_log.info(f" Using backing vocals models: {job.backing_vocals_models}")
|
||||
if job.other_stems_models:
|
||||
job_log.info(f" Using other stems models: {job.other_stems_models}")
|
||||
|
||||
audio_processor = create_audio_processor(
|
||||
temp_dir,
|
||||
clean_instrumental_model=job.clean_instrumental_model,
|
||||
backing_vocals_models=job.backing_vocals_models,
|
||||
other_stems_models=job.other_stems_models
|
||||
)
|
||||
|
||||
# Format artist-title for file naming (matches CLI behavior)
|
||||
artist_title = f"{job.artist} - {job.title}"
|
||||
|
||||
# Run audio separation (calls Modal API internally)
|
||||
# This returns a dict with paths to all separated stems
|
||||
with job_span("modal-separation", job_id) as sep_span:
|
||||
sep_start = time.time()
|
||||
job_log.info("Starting audio separation (this may take 5-10 minutes)...")
|
||||
job_log.info(" Stage 1: Clean instrumental separation (MDX models)")
|
||||
job_log.info(" Stage 2: Backing vocals separation (Demucs model)")
|
||||
add_span_event("separation_started")
|
||||
logger.info(f"[job:{job_id}] Calling Modal API for audio separation")
|
||||
|
||||
with metrics.time_external_api("modal", job_id):
|
||||
separation_result = audio_processor.process_audio_separation(
|
||||
audio_file=audio_path,
|
||||
artist_title=artist_title,
|
||||
track_output_dir=temp_dir
|
||||
)
|
||||
|
||||
sep_duration = time.time() - sep_start
|
||||
sep_span.set_attribute("duration_seconds", sep_duration)
|
||||
sep_span.set_attribute("stem_count", len(separation_result))
|
||||
add_span_event("separation_completed", {"duration_seconds": sep_duration})
|
||||
|
||||
job_log.info("Audio separation complete!")
|
||||
job_log.info(f" Generated {len(separation_result)} stem files")
|
||||
logger.info(f"[job:{job_id}] Audio separation complete, organizing results")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
job_manager.update_state_data(job_id, 'audio_progress', {
|
||||
'stage': 'audio_complete',
|
||||
'progress': 45,
|
||||
'message': 'Audio separation complete, uploading stems'
|
||||
})
|
||||
|
||||
# Upload all stems to GCS
|
||||
with job_span("upload-stems", job_id) as upload_span:
|
||||
await upload_separation_results(job_id, separation_result, storage, job_manager)
|
||||
upload_span.set_attribute("stem_count", len(separation_result))
|
||||
|
||||
logger.info(f"[job:{job_id}] All stems uploaded successfully")
|
||||
|
||||
# Mark audio processing complete
|
||||
# This will check if lyrics are also complete and transition to next stage if so
|
||||
job_manager.mark_audio_complete(job_id)
|
||||
|
||||
duration = time.time() - start_time
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=audio status=success duration={duration:.1f}s")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Audio separation failed: {e}", exc_info=True)
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}")
|
||||
job_manager.mark_job_failed(
|
||||
job_id=job_id,
|
||||
error_message=f"Audio separation failed: {str(e)}",
|
||||
@@ -344,7 +377,7 @@ async def process_audio_separation(job_id: str) -> bool:
|
||||
# Cleanup temporary directory
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
logger.debug(f"Cleaned up temp directory: {temp_dir}")
|
||||
logger.debug(f"[job:{job_id}] Cleaned up temp directory: {temp_dir}")
|
||||
|
||||
|
||||
async def download_audio(
|
||||
|
||||
+207
-162
@@ -12,12 +12,18 @@ Handles the lyrics processing track of parallel processing:
|
||||
Re-uses:
|
||||
- karaoke_gen.lyrics_processor.LyricsProcessor for lyrics fetching and orchestration
|
||||
- lyrics_transcriber library (submodule) for transcription and correction
|
||||
|
||||
Observability:
|
||||
- All operations wrapped in tracing spans for Cloud Trace visibility
|
||||
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
||||
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
from pathlib import Path
|
||||
@@ -28,6 +34,8 @@ from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.workers.style_helper import load_style_config
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
from backend.services.metrics import metrics
|
||||
|
||||
# Import from karaoke_gen package
|
||||
from karaoke_gen.lyrics_processor import LyricsProcessor
|
||||
@@ -108,13 +116,15 @@ async def process_lyrics_transcription(job_id: str) -> bool:
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
start_time = time.time()
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
|
||||
# Create job logger for remote debugging FIRST
|
||||
job_log = create_job_logger(job_id, "lyrics")
|
||||
|
||||
# Log immediately to verify logging is working
|
||||
# Log with structured markers for easy Cloud Logging queries
|
||||
logger.info(f"[job:{job_id}] WORKER_START worker=lyrics")
|
||||
job_log.info("=== LYRICS WORKER STARTED ===")
|
||||
job_log.info(f"Job ID: {job_id}")
|
||||
|
||||
@@ -127,166 +137,188 @@ async def process_lyrics_transcription(job_id: str) -> bool:
|
||||
temp_dir = None
|
||||
|
||||
try:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (lyrics_transcriber) are only captured
|
||||
# by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
# Create temporary working directory
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_lyrics_{job_id}_")
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
job_log.info(f"Starting lyrics transcription for {job.artist} - {job.title}")
|
||||
job_log.info(f"Log capture enabled for: {', '.join(LYRICS_WORKER_LOGGERS)}")
|
||||
logger.info(f"Starting lyrics transcription for job {job_id}")
|
||||
|
||||
# Log environment configuration
|
||||
job_log.info("Checking API configuration...")
|
||||
if os.environ.get("GENIUS_API_TOKEN"):
|
||||
job_log.info(" Genius API: configured")
|
||||
else:
|
||||
job_log.warning(" Genius API: NOT configured")
|
||||
if os.environ.get("SPOTIFY_COOKIE_SP_DC"):
|
||||
job_log.info(" Spotify: configured")
|
||||
else:
|
||||
job_log.warning(" Spotify: NOT configured")
|
||||
if os.environ.get("RAPIDAPI_KEY"):
|
||||
job_log.info(" Musixmatch (RapidAPI): configured")
|
||||
else:
|
||||
job_log.warning(" Musixmatch (RapidAPI): NOT configured")
|
||||
|
||||
# Ensure required environment variables are set
|
||||
if not os.environ.get("AUDIOSHAKE_API_TOKEN"):
|
||||
job_log.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
|
||||
logger.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
|
||||
|
||||
# Download audio file from GCS (waits for audio worker if URL job)
|
||||
job_log.info("Downloading audio file from GCS...")
|
||||
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager)
|
||||
if not audio_path:
|
||||
raise Exception("Failed to download audio file")
|
||||
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
# The status is managed at a higher level - workers just track their progress
|
||||
job_manager.update_state_data(job_id, 'lyrics_progress', {
|
||||
'stage': 'transcribing',
|
||||
'progress': 10,
|
||||
'message': 'Starting lyrics transcription via AudioShake'
|
||||
})
|
||||
|
||||
# Load style configuration (downloads assets from GCS if available)
|
||||
# This is needed for max_line_length and other style settings in LyricsTranscriber
|
||||
job_log.info("Loading style configuration for lyrics processing...")
|
||||
style_config = await load_style_config(job, storage, temp_dir)
|
||||
style_params_json_path = style_config.get_style_params_path() if style_config.has_custom_styles() else None
|
||||
|
||||
if style_params_json_path:
|
||||
job_log.info(f"Using custom style params: {style_params_json_path}")
|
||||
# Log the contents of the style params for debugging
|
||||
try:
|
||||
with open(style_params_json_path, 'r') as f:
|
||||
style_content = json.load(f)
|
||||
job_log.info(f"Style params sections: {list(style_content.keys())}")
|
||||
if 'karaoke' in style_content:
|
||||
karaoke_style = style_content['karaoke']
|
||||
job_log.info(f" karaoke.background_image: {karaoke_style.get('background_image', 'NOT SET')}")
|
||||
job_log.info(f" karaoke.font_path: {karaoke_style.get('font_path', 'NOT SET')}")
|
||||
except Exception as e:
|
||||
job_log.warning(f"Could not read style params for logging: {e}")
|
||||
else:
|
||||
job_log.info("No custom style params found, using defaults")
|
||||
|
||||
# Download user-provided lyrics file if available
|
||||
lyrics_file_path = None
|
||||
if hasattr(job, 'lyrics_file_gcs_path') and job.lyrics_file_gcs_path:
|
||||
lyrics_file_path = os.path.join(temp_dir, "user_lyrics.txt")
|
||||
job_log.info(f"Downloading user-provided lyrics file: {job.lyrics_file_gcs_path}")
|
||||
storage.download_file(job.lyrics_file_gcs_path, lyrics_file_path)
|
||||
job_log.info(f"User lyrics file downloaded to: {lyrics_file_path}")
|
||||
|
||||
# Get lyrics configuration from job
|
||||
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
|
||||
if subtitle_offset != 0:
|
||||
job_log.info(f"Subtitle offset: {subtitle_offset}ms")
|
||||
|
||||
# Create LyricsProcessor instance (reuses karaoke_gen code)
|
||||
job_log.info("Creating LyricsProcessor instance...")
|
||||
job_log.info(f" style_params_json: {style_params_json_path}")
|
||||
job_log.info(f" lyrics_file: {lyrics_file_path}")
|
||||
job_log.info(f" subtitle_offset_ms: {subtitle_offset}")
|
||||
lyrics_processor = create_lyrics_processor(
|
||||
style_params_json=style_params_json_path,
|
||||
lyrics_file=lyrics_file_path,
|
||||
subtitle_offset_ms=subtitle_offset
|
||||
)
|
||||
|
||||
# Run transcription + correction
|
||||
# This will:
|
||||
# 1. Fetch reference lyrics from Genius/Spotify/Musixmatch/LRCLib
|
||||
# 2. Transcribe audio via AudioShake API
|
||||
# 3. Run automatic correction with lyrics_transcriber
|
||||
# 4. Generate corrections JSON (no interactive review)
|
||||
# 5. Skip video generation (render_video=False)
|
||||
|
||||
# Use lyrics_artist/lyrics_title overrides if provided, else fall back to job artist/title
|
||||
lyrics_search_artist = getattr(job, 'lyrics_artist', None) or job.artist
|
||||
lyrics_search_title = getattr(job, 'lyrics_title', None) or job.title
|
||||
|
||||
job_log.info("Starting LyricsTranscriber processing...")
|
||||
job_log.info(f" Artist: {job.artist}")
|
||||
job_log.info(f" Title: {job.title}")
|
||||
if lyrics_search_artist != job.artist:
|
||||
job_log.info(f" Lyrics search artist override: {lyrics_search_artist}")
|
||||
if lyrics_search_title != job.title:
|
||||
job_log.info(f" Lyrics search title override: {lyrics_search_title}")
|
||||
logger.info(f"Job {job_id}: Calling lyrics_processor.transcribe_lyrics()")
|
||||
|
||||
# Run transcription in thread pool to avoid blocking the event loop
|
||||
# (transcribe_lyrics is synchronous and takes 1-2 minutes)
|
||||
# Note: We pass the original artist/title for file naming, but the LyricsProcessor
|
||||
# will use lyrics_artist/lyrics_title for the search (set via constructor args)
|
||||
result = await asyncio.to_thread(
|
||||
lyrics_processor.transcribe_lyrics,
|
||||
input_audio_wav=audio_path,
|
||||
artist=job.artist, # Original artist for file naming
|
||||
title=job.title, # Original title for file naming
|
||||
track_output_dir=temp_dir,
|
||||
lyrics_artist=lyrics_search_artist, # Override for lyrics search
|
||||
lyrics_title=lyrics_search_title # Override for lyrics search
|
||||
)
|
||||
|
||||
job_log.info("Transcription processing complete")
|
||||
logger.info(f"Job {job_id}: Transcription complete, uploading results")
|
||||
|
||||
# Upload lyrics results to GCS
|
||||
job_log.info("Uploading lyrics results to GCS...")
|
||||
await upload_lyrics_results(job_id, temp_dir, result, storage, job_manager, job_log)
|
||||
|
||||
job_log.info("All lyrics data uploaded successfully")
|
||||
logger.info(f"Job {job_id}: All lyrics data uploaded successfully")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
job_manager.update_state_data(job_id, 'lyrics_progress', {
|
||||
'stage': 'lyrics_complete',
|
||||
'progress': 45,
|
||||
'message': 'Lyrics transcription complete'
|
||||
})
|
||||
|
||||
# Mark lyrics processing complete
|
||||
# This will check if audio is also complete and transition to next stage if so
|
||||
job_log.info("Lyrics worker complete, checking if audio is also done...")
|
||||
job_manager.mark_lyrics_complete(job_id)
|
||||
|
||||
return True
|
||||
# Wrap entire worker in a tracing span
|
||||
with job_span("lyrics-worker", job_id) as root_span:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (lyrics_transcriber) are only captured
|
||||
# by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
add_span_attribute("artist", job.artist)
|
||||
add_span_attribute("title", job.title)
|
||||
|
||||
# Create temporary working directory
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_lyrics_{job_id}_")
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
job_log.info(f"Starting lyrics transcription for {job.artist} - {job.title}")
|
||||
job_log.info(f"Log capture enabled for: {', '.join(LYRICS_WORKER_LOGGERS)}")
|
||||
logger.info(f"[job:{job_id}] Starting lyrics transcription for {job.artist} - {job.title}")
|
||||
|
||||
# Log environment configuration
|
||||
with job_span("check-api-config", job_id):
|
||||
job_log.info("Checking API configuration...")
|
||||
apis_configured = []
|
||||
if os.environ.get("GENIUS_API_TOKEN"):
|
||||
job_log.info(" Genius API: configured")
|
||||
apis_configured.append("genius")
|
||||
else:
|
||||
job_log.warning(" Genius API: NOT configured")
|
||||
if os.environ.get("SPOTIFY_COOKIE_SP_DC"):
|
||||
job_log.info(" Spotify: configured")
|
||||
apis_configured.append("spotify")
|
||||
else:
|
||||
job_log.warning(" Spotify: NOT configured")
|
||||
if os.environ.get("RAPIDAPI_KEY"):
|
||||
job_log.info(" Musixmatch (RapidAPI): configured")
|
||||
apis_configured.append("musixmatch")
|
||||
else:
|
||||
job_log.warning(" Musixmatch (RapidAPI): NOT configured")
|
||||
add_span_attribute("lyrics_apis", ",".join(apis_configured))
|
||||
|
||||
# Ensure required environment variables are set
|
||||
if not os.environ.get("AUDIOSHAKE_API_TOKEN"):
|
||||
job_log.warning("AUDIOSHAKE_API_TOKEN not set - transcription may fail")
|
||||
logger.warning(f"[job:{job_id}] AUDIOSHAKE_API_TOKEN not set - transcription may fail")
|
||||
|
||||
# Download audio file from GCS (waits for audio worker if URL job)
|
||||
with job_span("download-audio", job_id) as download_span:
|
||||
job_log.info("Downloading audio file from GCS...")
|
||||
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager)
|
||||
if not audio_path:
|
||||
raise Exception("Failed to download audio file")
|
||||
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
|
||||
download_span.set_attribute("audio_file", os.path.basename(audio_path))
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
# The status is managed at a higher level - workers just track their progress
|
||||
job_manager.update_state_data(job_id, 'lyrics_progress', {
|
||||
'stage': 'transcribing',
|
||||
'progress': 10,
|
||||
'message': 'Starting lyrics transcription via AudioShake'
|
||||
})
|
||||
|
||||
# Load style configuration (downloads assets from GCS if available)
|
||||
# This is needed for max_line_length and other style settings in LyricsTranscriber
|
||||
with job_span("load-style-config", job_id):
|
||||
job_log.info("Loading style configuration for lyrics processing...")
|
||||
style_config = await load_style_config(job, storage, temp_dir)
|
||||
style_params_json_path = style_config.get_style_params_path() if style_config.has_custom_styles() else None
|
||||
|
||||
if style_params_json_path:
|
||||
job_log.info(f"Using custom style params: {style_params_json_path}")
|
||||
# Log the contents of the style params for debugging
|
||||
try:
|
||||
with open(style_params_json_path, 'r') as f:
|
||||
style_content = json.load(f)
|
||||
job_log.info(f"Style params sections: {list(style_content.keys())}")
|
||||
if 'karaoke' in style_content:
|
||||
karaoke_style = style_content['karaoke']
|
||||
job_log.info(f" karaoke.background_image: {karaoke_style.get('background_image', 'NOT SET')}")
|
||||
job_log.info(f" karaoke.font_path: {karaoke_style.get('font_path', 'NOT SET')}")
|
||||
except Exception as e:
|
||||
job_log.warning(f"Could not read style params for logging: {e}")
|
||||
else:
|
||||
job_log.info("No custom style params found, using defaults")
|
||||
|
||||
# Download user-provided lyrics file if available
|
||||
lyrics_file_path = None
|
||||
if hasattr(job, 'lyrics_file_gcs_path') and job.lyrics_file_gcs_path:
|
||||
with job_span("download-user-lyrics", job_id):
|
||||
lyrics_file_path = os.path.join(temp_dir, "user_lyrics.txt")
|
||||
job_log.info(f"Downloading user-provided lyrics file: {job.lyrics_file_gcs_path}")
|
||||
storage.download_file(job.lyrics_file_gcs_path, lyrics_file_path)
|
||||
job_log.info(f"User lyrics file downloaded to: {lyrics_file_path}")
|
||||
|
||||
# Get lyrics configuration from job
|
||||
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
|
||||
if subtitle_offset != 0:
|
||||
job_log.info(f"Subtitle offset: {subtitle_offset}ms")
|
||||
add_span_attribute("subtitle_offset_ms", subtitle_offset)
|
||||
|
||||
# Create LyricsProcessor instance (reuses karaoke_gen code)
|
||||
job_log.info("Creating LyricsProcessor instance...")
|
||||
job_log.info(f" style_params_json: {style_params_json_path}")
|
||||
job_log.info(f" lyrics_file: {lyrics_file_path}")
|
||||
job_log.info(f" subtitle_offset_ms: {subtitle_offset}")
|
||||
lyrics_processor = create_lyrics_processor(
|
||||
style_params_json=style_params_json_path,
|
||||
lyrics_file=lyrics_file_path,
|
||||
subtitle_offset_ms=subtitle_offset
|
||||
)
|
||||
|
||||
# Use lyrics_artist/lyrics_title overrides if provided, else fall back to job artist/title
|
||||
lyrics_search_artist = getattr(job, 'lyrics_artist', None) or job.artist
|
||||
lyrics_search_title = getattr(job, 'lyrics_title', None) or job.title
|
||||
|
||||
job_log.info("Starting LyricsTranscriber processing...")
|
||||
job_log.info(f" Artist: {job.artist}")
|
||||
job_log.info(f" Title: {job.title}")
|
||||
if lyrics_search_artist != job.artist:
|
||||
job_log.info(f" Lyrics search artist override: {lyrics_search_artist}")
|
||||
if lyrics_search_title != job.title:
|
||||
job_log.info(f" Lyrics search title override: {lyrics_search_title}")
|
||||
logger.info(f"[job:{job_id}] Calling lyrics_processor.transcribe_lyrics()")
|
||||
|
||||
# Run transcription + correction
|
||||
with job_span("audioshake-transcription", job_id) as trans_span:
|
||||
trans_start = time.time()
|
||||
add_span_event("transcription_started")
|
||||
|
||||
# Run transcription in thread pool to avoid blocking the event loop
|
||||
# (transcribe_lyrics is synchronous and takes 1-2 minutes)
|
||||
# Note: transcribe_lyrics internally calls AudioShake and lyrics APIs
|
||||
with metrics.time_external_api("audioshake", job_id):
|
||||
result = await asyncio.to_thread(
|
||||
lyrics_processor.transcribe_lyrics,
|
||||
input_audio_wav=audio_path,
|
||||
artist=job.artist, # Original artist for file naming
|
||||
title=job.title, # Original title for file naming
|
||||
track_output_dir=temp_dir,
|
||||
lyrics_artist=lyrics_search_artist, # Override for lyrics search
|
||||
lyrics_title=lyrics_search_title # Override for lyrics search
|
||||
)
|
||||
|
||||
trans_duration = time.time() - trans_start
|
||||
trans_span.set_attribute("duration_seconds", trans_duration)
|
||||
add_span_event("transcription_completed", {"duration_seconds": trans_duration})
|
||||
|
||||
job_log.info("Transcription processing complete")
|
||||
logger.info(f"[job:{job_id}] Transcription complete, uploading results")
|
||||
|
||||
# Upload lyrics results to GCS
|
||||
with job_span("upload-lyrics-results", job_id):
|
||||
job_log.info("Uploading lyrics results to GCS...")
|
||||
await upload_lyrics_results(job_id, temp_dir, result, storage, job_manager, job_log)
|
||||
|
||||
job_log.info("All lyrics data uploaded successfully")
|
||||
logger.info(f"[job:{job_id}] All lyrics data uploaded successfully")
|
||||
|
||||
# Update progress using state_data (don't change status during parallel processing)
|
||||
job_manager.update_state_data(job_id, 'lyrics_progress', {
|
||||
'stage': 'lyrics_complete',
|
||||
'progress': 45,
|
||||
'message': 'Lyrics transcription complete'
|
||||
})
|
||||
|
||||
# Mark lyrics processing complete
|
||||
# This will check if audio is also complete and transition to next stage if so
|
||||
job_log.info("Lyrics worker complete, checking if audio is also done...")
|
||||
job_manager.mark_lyrics_complete(job_id)
|
||||
|
||||
duration = time.time() - start_time
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=lyrics status=success duration={duration:.1f}s")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
job_log.error(f"Lyrics transcription failed: {str(e)}", exc_info=True)
|
||||
logger.error(f"Job {job_id}: Lyrics transcription failed: {e}", exc_info=True)
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=lyrics status=error duration={duration:.1f}s error={e}")
|
||||
job_manager.mark_job_failed(
|
||||
job_id=job_id,
|
||||
error_message=f"Lyrics transcription failed: {str(e)}",
|
||||
@@ -305,7 +337,7 @@ async def process_lyrics_transcription(job_id: str) -> bool:
|
||||
# Cleanup temporary directory (only if it was created)
|
||||
if temp_dir and os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
logger.debug(f"Cleaned up temp directory: {temp_dir}")
|
||||
logger.debug(f"[job:{job_id}] Cleaned up temp directory: {temp_dir}")
|
||||
|
||||
|
||||
async def download_audio(
|
||||
@@ -447,11 +479,24 @@ async def upload_lyrics_results(
|
||||
corrections_data = json.load(f)
|
||||
|
||||
# Store metadata in state_data
|
||||
job_manager.update_state_data(job_id, 'lyrics_metadata', {
|
||||
# Include countdown padding info for instrumental synchronization
|
||||
lyrics_metadata = {
|
||||
'segment_count': len(corrections_data.get('corrected_segments', [])),
|
||||
'has_corrections': True,
|
||||
'ready_for_review': True
|
||||
})
|
||||
'ready_for_review': True,
|
||||
}
|
||||
|
||||
# Add countdown padding info from transcription result
|
||||
# This is needed by video_worker to pad instrumentals for sync
|
||||
if transcription_result.get("countdown_padding_added"):
|
||||
lyrics_metadata['has_countdown_padding'] = True
|
||||
lyrics_metadata['countdown_padding_seconds'] = transcription_result.get("countdown_padding_seconds", 3.0)
|
||||
logger.info(f"Job {job_id}: Countdown padding detected: {lyrics_metadata['countdown_padding_seconds']}s")
|
||||
else:
|
||||
lyrics_metadata['has_countdown_padding'] = False
|
||||
lyrics_metadata['countdown_padding_seconds'] = 0.0
|
||||
|
||||
job_manager.update_state_data(job_id, 'lyrics_metadata', lyrics_metadata)
|
||||
except Exception as e:
|
||||
logger.warning(f"Job {job_id}: Could not parse corrections JSON: {e}")
|
||||
else:
|
||||
|
||||
@@ -14,10 +14,16 @@ This worker:
|
||||
Key insight: We use OutputGenerator from lyrics_transcriber library
|
||||
WITHOUT using its blocking ReviewServer. This allows async operation
|
||||
in Cloud Run.
|
||||
|
||||
Observability:
|
||||
- All operations wrapped in tracing spans for Cloud Trace visibility
|
||||
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
||||
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
from dataclasses import dataclass
|
||||
@@ -27,6 +33,7 @@ from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.config import get_settings
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
|
||||
# Import from lyrics_transcriber (submodule)
|
||||
from lyrics_transcriber.output.generator import OutputGenerator
|
||||
@@ -62,12 +69,16 @@ async def process_render_video(job_id: str) -> bool:
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
start_time = time.time()
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
settings = get_settings()
|
||||
|
||||
# Create job logger for remote debugging FIRST
|
||||
job_log = create_job_logger(job_id, "render_video")
|
||||
|
||||
# Log with structured markers for easy Cloud Logging queries
|
||||
logger.info(f"[job:{job_id}] WORKER_START worker=render-video")
|
||||
job_log.info("=== RENDER VIDEO WORKER STARTED ===")
|
||||
job_log.info(f"Job ID: {job_id}")
|
||||
|
||||
@@ -77,30 +88,32 @@ async def process_render_video(job_id: str) -> bool:
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
logger.error(f"[job:{job_id}] Job not found in Firestore")
|
||||
job_log.error(f"Job {job_id} not found in Firestore!")
|
||||
return False
|
||||
|
||||
job_log.info(f"Starting video render for {job.artist} - {job.title}")
|
||||
logger.info(f"Job {job_id}: Starting video render (post-review)")
|
||||
logger.info(f"[job:{job_id}] Starting video render (post-review)")
|
||||
|
||||
try:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (lyrics_transcriber.output) are only
|
||||
# captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
# Transition to RENDERING_VIDEO
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.RENDERING_VIDEO,
|
||||
progress=75,
|
||||
message="Rendering karaoke video with corrected lyrics"
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
# 1. Download corrected corrections data
|
||||
corrections_path = os.path.join(temp_dir, "corrections.json")
|
||||
# Wrap entire worker in a tracing span
|
||||
with job_span("render-video-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (lyrics_transcriber.output) are only
|
||||
# captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
# Transition to RENDERING_VIDEO
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.RENDERING_VIDEO,
|
||||
progress=75,
|
||||
message="Rendering karaoke video with corrected lyrics"
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
job_log.info(f"Created temp directory: {temp_dir}")
|
||||
# 1. Download corrected corrections data
|
||||
corrections_path = os.path.join(temp_dir, "corrections.json")
|
||||
|
||||
# Try updated corrections first (from human review), fall back to original
|
||||
corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json"
|
||||
@@ -280,7 +293,10 @@ async def process_render_video(job_id: str) -> bool:
|
||||
message="Prep phase complete - download outputs to continue locally"
|
||||
)
|
||||
job_log.info("=== RENDER VIDEO WORKER COMPLETE (PREP ONLY) ===")
|
||||
logger.info(f"Job {job_id}: Prep-only video render complete")
|
||||
duration = time.time() - start_time
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
root_span.set_attribute("prep_only", True)
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s prep_only=true")
|
||||
else:
|
||||
# Normal mode: proceed to instrumental selection
|
||||
job_manager.transition_to_state(
|
||||
@@ -290,12 +306,15 @@ async def process_render_video(job_id: str) -> bool:
|
||||
message="Video rendered - select your instrumental"
|
||||
)
|
||||
job_log.info("=== RENDER VIDEO WORKER COMPLETE ===")
|
||||
logger.info(f"Job {job_id}: Video render complete, awaiting instrumental selection")
|
||||
duration = time.time() - start_time
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
job_log.error(f"Video render failed: {e}")
|
||||
logger.error(f"Job {job_id}: Video render failed: {e}", exc_info=True)
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=render-video status=error duration={duration:.1f}s error={e}")
|
||||
job_manager.fail_job(job_id, f"Video render failed: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
+112
-102
@@ -16,17 +16,16 @@ processing complete (via mark_audio_complete/mark_lyrics_complete coordination).
|
||||
|
||||
Integrates with karaoke_gen.video_generator.VideoGenerator.
|
||||
|
||||
SOLID Principles:
|
||||
- Single Responsibility: Only generates title/end screens
|
||||
- Open/Closed: Extensible for different screen styles
|
||||
- Liskov Substitution: Can swap different video generators
|
||||
- Interface Segregation: Focused interface for screen generation
|
||||
- Dependency Inversion: Depends on abstractions (VideoGenerator interface)
|
||||
Observability:
|
||||
- All operations wrapped in tracing spans for Cloud Trace visibility
|
||||
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
||||
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Optional, Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
@@ -36,6 +35,7 @@ from backend.services.storage_service import StorageService
|
||||
from backend.config import get_settings
|
||||
from backend.workers.style_helper import load_style_config, StyleConfig
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
|
||||
# Import from karaoke_gen package
|
||||
from karaoke_gen.video_generator import VideoGenerator
|
||||
@@ -64,6 +64,7 @@ async def generate_screens(job_id: str) -> bool:
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
start_time = time.time()
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
settings = get_settings()
|
||||
@@ -71,118 +72,127 @@ async def generate_screens(job_id: str) -> bool:
|
||||
# Create job logger for remote debugging
|
||||
job_log = create_job_logger(job_id, "screens")
|
||||
|
||||
# Log with structured markers for easy Cloud Logging queries
|
||||
logger.info(f"[job:{job_id}] WORKER_START worker=screens")
|
||||
|
||||
# Set up log capture for VideoGenerator and style_helper
|
||||
log_handler = setup_job_logging(job_id, "screens", *SCREENS_WORKER_LOGGERS)
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
logger.error(f"[job:{job_id}] Job not found")
|
||||
return False
|
||||
|
||||
# Validate both audio and lyrics are complete
|
||||
if not _validate_prerequisites(job):
|
||||
logger.error(f"Job {job_id}: Prerequisites not met for screen generation")
|
||||
logger.error(f"[job:{job_id}] Prerequisites not met for screen generation")
|
||||
return False
|
||||
|
||||
# Create temporary working directory
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_screens_{job_id}_")
|
||||
|
||||
try:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (karaoke_gen.video_generator) are only
|
||||
# captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job_log.info(f"Starting screen generation for {job.artist} - {job.title}")
|
||||
logger.info(f"Starting screen generation for job {job_id}")
|
||||
|
||||
# Transition to GENERATING_SCREENS state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.GENERATING_SCREENS,
|
||||
progress=50,
|
||||
message="Generating title and end screens"
|
||||
)
|
||||
|
||||
# Log style assets info
|
||||
style_assets = getattr(job, 'style_assets', {}) or {}
|
||||
job_log.info(f"Style assets from job: {list(style_assets.keys()) if style_assets else 'None'}")
|
||||
if style_assets:
|
||||
for key, path in style_assets.items():
|
||||
job_log.info(f" {key}: {path}")
|
||||
|
||||
# Load style configuration (downloads assets from GCS if available)
|
||||
job_log.info("Loading style configuration from GCS...")
|
||||
style_config = await load_style_config(job, storage, temp_dir)
|
||||
if style_config.has_custom_styles():
|
||||
job_log.info("Using CUSTOM style configuration")
|
||||
logger.info(f"Job {job_id}: Using custom style configuration")
|
||||
else:
|
||||
job_log.warning("Using DEFAULT style configuration (no custom styles found)")
|
||||
logger.info(f"Job {job_id}: Using default style configuration")
|
||||
|
||||
# Initialize video generator
|
||||
video_generator = _create_video_generator(temp_dir)
|
||||
|
||||
# Generate title screen with style config
|
||||
job_log.info("Generating title screen...")
|
||||
title_screen_path = await _generate_title_screen(
|
||||
job_id=job_id,
|
||||
job=job,
|
||||
video_generator=video_generator,
|
||||
style_config=style_config,
|
||||
temp_dir=temp_dir,
|
||||
job_log=job_log
|
||||
)
|
||||
|
||||
if not title_screen_path:
|
||||
raise Exception("Title screen generation failed")
|
||||
job_log.info(f"Title screen generated: {title_screen_path}")
|
||||
|
||||
# Generate end screen with style config
|
||||
job_log.info("Generating end screen...")
|
||||
end_screen_path = await _generate_end_screen(
|
||||
job_id=job_id,
|
||||
job=job,
|
||||
video_generator=video_generator,
|
||||
style_config=style_config,
|
||||
temp_dir=temp_dir,
|
||||
job_log=job_log
|
||||
)
|
||||
|
||||
if not end_screen_path:
|
||||
raise Exception("End screen generation failed")
|
||||
job_log.info(f"End screen generated: {end_screen_path}")
|
||||
|
||||
# Upload screens to GCS
|
||||
await _upload_screens(
|
||||
job_id=job_id,
|
||||
job_manager=job_manager,
|
||||
storage=storage,
|
||||
title_screen_path=title_screen_path,
|
||||
end_screen_path=end_screen_path
|
||||
)
|
||||
|
||||
# Apply countdown padding if needed
|
||||
# (This is handled automatically by checking lyrics metadata)
|
||||
await _apply_countdown_padding_if_needed(job_id, job_manager, job)
|
||||
|
||||
# Transition to AWAITING_REVIEW
|
||||
# Human must review lyrics before video can be rendered
|
||||
logger.info(f"Job {job_id}: Screens generated, awaiting lyrics review")
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.AWAITING_REVIEW,
|
||||
progress=55,
|
||||
message="Ready for lyrics review. Please review and correct lyrics."
|
||||
)
|
||||
|
||||
# TODO: Send notification to user that selection is needed
|
||||
# await notify_user(job_id, "instrumental_selection_ready")
|
||||
|
||||
return True
|
||||
# Wrap entire worker in a tracing span
|
||||
with job_span("screens-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
with job_logging_context(job_id):
|
||||
job_log.info(f"Starting screen generation for {job.artist} - {job.title}")
|
||||
logger.info(f"[job:{job_id}] Starting screen generation for {job.artist} - {job.title}")
|
||||
|
||||
# Transition to GENERATING_SCREENS state
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.GENERATING_SCREENS,
|
||||
progress=50,
|
||||
message="Generating title and end screens"
|
||||
)
|
||||
|
||||
# Log style assets info
|
||||
style_assets = getattr(job, 'style_assets', {}) or {}
|
||||
job_log.info(f"Style assets from job: {list(style_assets.keys()) if style_assets else 'None'}")
|
||||
if style_assets:
|
||||
for key, path in style_assets.items():
|
||||
job_log.info(f" {key}: {path}")
|
||||
|
||||
# Load style configuration (downloads assets from GCS if available)
|
||||
with job_span("load-style-config", job_id):
|
||||
job_log.info("Loading style configuration from GCS...")
|
||||
style_config = await load_style_config(job, storage, temp_dir)
|
||||
if style_config.has_custom_styles():
|
||||
job_log.info("Using CUSTOM style configuration")
|
||||
logger.info(f"[job:{job_id}] Using custom style configuration")
|
||||
add_span_attribute("style_type", "custom")
|
||||
else:
|
||||
job_log.warning("Using DEFAULT style configuration (no custom styles found)")
|
||||
logger.info(f"[job:{job_id}] Using default style configuration")
|
||||
add_span_attribute("style_type", "default")
|
||||
|
||||
# Initialize video generator
|
||||
video_generator = _create_video_generator(temp_dir)
|
||||
|
||||
# Generate title screen with style config
|
||||
with job_span("generate-title-screen", job_id):
|
||||
job_log.info("Generating title screen...")
|
||||
title_screen_path = await _generate_title_screen(
|
||||
job_id=job_id,
|
||||
job=job,
|
||||
video_generator=video_generator,
|
||||
style_config=style_config,
|
||||
temp_dir=temp_dir,
|
||||
job_log=job_log
|
||||
)
|
||||
|
||||
if not title_screen_path:
|
||||
raise Exception("Title screen generation failed")
|
||||
job_log.info(f"Title screen generated: {title_screen_path}")
|
||||
|
||||
# Generate end screen with style config
|
||||
with job_span("generate-end-screen", job_id):
|
||||
job_log.info("Generating end screen...")
|
||||
end_screen_path = await _generate_end_screen(
|
||||
job_id=job_id,
|
||||
job=job,
|
||||
video_generator=video_generator,
|
||||
style_config=style_config,
|
||||
temp_dir=temp_dir,
|
||||
job_log=job_log
|
||||
)
|
||||
|
||||
if not end_screen_path:
|
||||
raise Exception("End screen generation failed")
|
||||
job_log.info(f"End screen generated: {end_screen_path}")
|
||||
|
||||
# Upload screens to GCS
|
||||
with job_span("upload-screens", job_id):
|
||||
await _upload_screens(
|
||||
job_id=job_id,
|
||||
job_manager=job_manager,
|
||||
storage=storage,
|
||||
title_screen_path=title_screen_path,
|
||||
end_screen_path=end_screen_path
|
||||
)
|
||||
|
||||
# Apply countdown padding if needed
|
||||
await _apply_countdown_padding_if_needed(job_id, job_manager, job)
|
||||
|
||||
# Transition to AWAITING_REVIEW
|
||||
# Human must review lyrics before video can be rendered
|
||||
logger.info(f"[job:{job_id}] Screens generated, awaiting lyrics review")
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.AWAITING_REVIEW,
|
||||
progress=55,
|
||||
message="Ready for lyrics review. Please review and correct lyrics."
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=screens status=success duration={duration:.1f}s")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Screen generation failed: {e}", exc_info=True)
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=screens status=error duration={duration:.1f}s error={e}")
|
||||
job_manager.mark_job_failed(
|
||||
job_id=job_id,
|
||||
error_message=f"Screen generation failed: {str(e)}",
|
||||
@@ -201,7 +211,7 @@ async def generate_screens(job_id: str) -> bool:
|
||||
# Cleanup temporary directory
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
logger.debug(f"Cleaned up temp directory: {temp_dir}")
|
||||
logger.debug(f"[job:{job_id}] Cleaned up temp directory: {temp_dir}")
|
||||
|
||||
|
||||
def _validate_prerequisites(job) -> bool:
|
||||
|
||||
@@ -15,11 +15,17 @@ The worker's job is simply to:
|
||||
4. Upload results back to GCS
|
||||
|
||||
This reuses 100% of the existing karaoke_gen functionality.
|
||||
|
||||
Observability:
|
||||
- All operations wrapped in tracing spans for Cloud Trace visibility
|
||||
- Logs include [job:ID] prefix for easy filtering in Cloud Logging
|
||||
- Worker start/end timing logged with WORKER_START/WORKER_END markers
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
from pathlib import Path
|
||||
@@ -32,6 +38,7 @@ from backend.services.youtube_service import get_youtube_service
|
||||
from backend.config import get_settings
|
||||
from backend.workers.style_helper import load_style_config
|
||||
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
||||
from backend.services.tracing import job_span, add_span_event, add_span_attribute
|
||||
|
||||
# Import from karaoke_gen package - reuse existing implementation
|
||||
from karaoke_gen.karaoke_finalise.karaoke_finalise import KaraokeFinalise
|
||||
@@ -61,6 +68,7 @@ async def generate_video(job_id: str) -> bool:
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
start_time = time.time()
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
settings = get_settings()
|
||||
@@ -68,17 +76,20 @@ async def generate_video(job_id: str) -> bool:
|
||||
# Create job logger for remote debugging
|
||||
job_log = create_job_logger(job_id, "video")
|
||||
|
||||
# Log with structured markers for easy Cloud Logging queries
|
||||
logger.info(f"[job:{job_id}] WORKER_START worker=video")
|
||||
|
||||
# Set up log capture for KaraokeFinalise
|
||||
log_handler = setup_job_logging(job_id, "video", *VIDEO_WORKER_LOGGERS)
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
logger.error(f"[job:{job_id}] Job not found")
|
||||
return False
|
||||
|
||||
# Validate prerequisites
|
||||
if not _validate_prerequisites(job):
|
||||
logger.error(f"Job {job_id}: Prerequisites not met for video generation")
|
||||
logger.error(f"[job:{job_id}] Prerequisites not met for video generation")
|
||||
return False
|
||||
|
||||
# Create temporary working directory
|
||||
@@ -105,12 +116,14 @@ async def generate_video(job_id: str) -> bool:
|
||||
job_log.warning("YouTube credentials not available - upload will be skipped")
|
||||
|
||||
try:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (karaoke_gen.karaoke_finalise) are only
|
||||
# captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job_log.info(f"Starting video finalization for {job.artist} - {job.title}")
|
||||
logger.info(f"Starting video generation for job {job_id}")
|
||||
# Wrap entire worker in a tracing span
|
||||
with job_span("video-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
|
||||
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
||||
# This ensures logs from third-party libraries (karaoke_gen.karaoke_finalise) are only
|
||||
# captured by this job's handler, not handlers from other concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
job_log.info(f"Starting video finalization for {job.artist} - {job.title}")
|
||||
logger.info(f"[job:{job_id}] Starting video generation for {job.artist} - {job.title}")
|
||||
|
||||
# Transition to GENERATING_VIDEO state
|
||||
job_manager.transition_to_state(
|
||||
@@ -122,9 +135,10 @@ async def generate_video(job_id: str) -> bool:
|
||||
|
||||
# Download and set up files in the format KaraokeFinalise expects
|
||||
base_name = f"{job.artist} - {job.title}"
|
||||
job_log.info("Downloading files from GCS (this may take a few minutes for large files)...")
|
||||
await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log)
|
||||
job_log.info("All files downloaded successfully")
|
||||
with job_span("download-files", job_id):
|
||||
job_log.info("Downloading files from GCS (this may take a few minutes for large files)...")
|
||||
await _setup_working_directory(job_id, job, storage, temp_dir, base_name, job_log)
|
||||
job_log.info("All files downloaded successfully")
|
||||
|
||||
# Load style config for CDG styles
|
||||
job_log.info("Loading CDG style configuration...")
|
||||
@@ -162,6 +176,14 @@ async def generate_video(job_id: str) -> bool:
|
||||
message="Encoding videos (15-20 min)"
|
||||
)
|
||||
|
||||
# Get countdown padding info from lyrics metadata
|
||||
# This ensures instrumental is padded to match vocals if countdown was added
|
||||
lyrics_metadata = job.state_data.get('lyrics_metadata', {})
|
||||
countdown_padding_seconds = None
|
||||
if lyrics_metadata.get('has_countdown_padding'):
|
||||
countdown_padding_seconds = lyrics_metadata.get('countdown_padding_seconds', 3.0)
|
||||
job_log.info(f"Countdown padding detected: {countdown_padding_seconds}s - instrumental will be padded if needed")
|
||||
|
||||
# Log finalization parameters
|
||||
job_log.info("Creating KaraokeFinalise with parameters:")
|
||||
job_log.info(f" enable_cdg: {getattr(job, 'enable_cdg', False)}")
|
||||
@@ -169,6 +191,7 @@ async def generate_video(job_id: str) -> bool:
|
||||
job_log.info(f" brand_prefix: {getattr(job, 'brand_prefix', None)}")
|
||||
job_log.info(f" discord_webhook: {'configured' if getattr(job, 'discord_webhook_url', None) else 'not configured'}")
|
||||
job_log.info(f" instrumental source: {instrumental_source}")
|
||||
job_log.info(f" countdown_padding_seconds: {countdown_padding_seconds}")
|
||||
if existing_instrumental_path:
|
||||
job_log.info(f" using user-provided instrumental (selection was: {instrumental_selection})")
|
||||
|
||||
@@ -209,6 +232,8 @@ async def generate_video(job_id: str) -> bool:
|
||||
non_interactive=True,
|
||||
server_side_mode=True,
|
||||
selected_instrumental_file=instrumental_file,
|
||||
# Audio synchronization - ensure instrumental matches vocal padding
|
||||
countdown_padding_seconds=countdown_padding_seconds,
|
||||
)
|
||||
|
||||
# Call process() - this does ALL the work:
|
||||
@@ -216,25 +241,34 @@ async def generate_video(job_id: str) -> bool:
|
||||
# - Generates CDG/TXT packages if enabled
|
||||
# - Posts Discord notification if configured
|
||||
# - Handles brand code generation
|
||||
job_log.info("Starting KaraokeFinalise.process() - this may take 15-20 minutes...")
|
||||
logger.info(f"Job {job_id}: Starting KaraokeFinalise.process()")
|
||||
result = finalise.process()
|
||||
with job_span("karaoke-finalise", job_id) as finalise_span:
|
||||
finalise_start = time.time()
|
||||
job_log.info("Starting KaraokeFinalise.process() - this may take 15-20 minutes...")
|
||||
logger.info(f"[job:{job_id}] Starting KaraokeFinalise.process()")
|
||||
add_span_event("finalise_started")
|
||||
|
||||
result = finalise.process()
|
||||
|
||||
finalise_duration = time.time() - finalise_start
|
||||
finalise_span.set_attribute("duration_seconds", finalise_duration)
|
||||
add_span_event("finalise_completed", {"duration_seconds": finalise_duration})
|
||||
|
||||
job_log.info("KaraokeFinalise.process() complete!")
|
||||
if result.get('brand_code'):
|
||||
job_log.info(f"Brand code: {result.get('brand_code')}")
|
||||
logger.info(f"Job {job_id}: KaraokeFinalise.process() complete")
|
||||
logger.info(f"Job {job_id}: Brand code: {result.get('brand_code')}")
|
||||
logger.info(f"[job:{job_id}] KaraokeFinalise.process() complete in {finalise_duration:.1f}s")
|
||||
logger.info(f"[job:{job_id}] Brand code: {result.get('brand_code')}")
|
||||
|
||||
# Native API distribution uploads (used by remote CLI instead of rclone)
|
||||
await _handle_native_distribution(
|
||||
job_id=job_id,
|
||||
job=job,
|
||||
job_log=job_log,
|
||||
job_manager=job_manager,
|
||||
temp_dir=temp_dir,
|
||||
result=result,
|
||||
)
|
||||
with job_span("distribution", job_id):
|
||||
await _handle_native_distribution(
|
||||
job_id=job_id,
|
||||
job=job,
|
||||
job_log=job_log,
|
||||
job_manager=job_manager,
|
||||
temp_dir=temp_dir,
|
||||
result=result,
|
||||
)
|
||||
|
||||
# Upload generated files to GCS
|
||||
job_manager.transition_to_state(
|
||||
@@ -244,10 +278,11 @@ async def generate_video(job_id: str) -> bool:
|
||||
message="Uploading final files"
|
||||
)
|
||||
|
||||
await _upload_results(job_id, job_manager, storage, temp_dir, result)
|
||||
with job_span("upload-results", job_id):
|
||||
await _upload_results(job_id, job_manager, storage, temp_dir, result)
|
||||
|
||||
# Mark job as complete
|
||||
logger.info(f"Job {job_id}: Video generation complete")
|
||||
logger.info(f"[job:{job_id}] Video generation complete")
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.COMPLETE,
|
||||
@@ -264,10 +299,15 @@ async def generate_video(job_id: str) -> bool:
|
||||
}
|
||||
})
|
||||
|
||||
duration = time.time() - start_time
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
root_span.set_attribute("brand_code", result.get('brand_code', ''))
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=video status=success duration={duration:.1f}s")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Video generation failed: {e}", exc_info=True)
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=video status=error duration={duration:.1f}s error={e}")
|
||||
job_manager.mark_job_failed(
|
||||
job_id=job_id,
|
||||
error_message=f"Video generation failed: {str(e)}",
|
||||
|
||||
@@ -1,22 +1,23 @@
|
||||
# Cloud Build configuration for base image
|
||||
#
|
||||
# This builds the base image with all Python dependencies pre-installed.
|
||||
# Run this manually when poetry.lock changes:
|
||||
# Run this manually when poetry.lock OR Dockerfile.base changes:
|
||||
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
|
||||
#
|
||||
# The base image is used by the main Dockerfile for fast app builds.
|
||||
# The poetry.lock hash is stored as a label for change detection.
|
||||
# A combined hash of poetry.lock + Dockerfile.base is stored as a label for change detection.
|
||||
|
||||
steps:
|
||||
# Calculate poetry.lock hash for change detection
|
||||
# Calculate combined hash for change detection (poetry.lock + Dockerfile.base)
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
entrypoint: 'bash'
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
POETRY_LOCK_HASH=$$(sha256sum poetry.lock | cut -d' ' -f1)
|
||||
echo "Poetry.lock hash: $$POETRY_LOCK_HASH"
|
||||
echo "$$POETRY_LOCK_HASH" > /workspace/POETRY_LOCK_HASH
|
||||
# Hash both files to detect changes in either
|
||||
CONTENT_HASH=$$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
|
||||
echo "Combined content hash (poetry.lock + Dockerfile.base): $$CONTENT_HASH"
|
||||
echo "$$CONTENT_HASH" > /workspace/CONTENT_HASH
|
||||
|
||||
# Build base image with all dependencies
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
@@ -24,11 +25,11 @@ steps:
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
POETRY_LOCK_HASH=$$(cat /workspace/POETRY_LOCK_HASH)
|
||||
CONTENT_HASH=$$(cat /workspace/CONTENT_HASH)
|
||||
docker build \
|
||||
--build-arg BUILD_DATE=$BUILD_ID \
|
||||
--build-arg BUILD_ID=$BUILD_ID \
|
||||
--build-arg POETRY_LOCK_HASH=$$POETRY_LOCK_HASH \
|
||||
--build-arg BASE_CONTENT_HASH=$$CONTENT_HASH \
|
||||
-t us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest \
|
||||
-t us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID \
|
||||
-f backend/Dockerfile.base \
|
||||
|
||||
+1
-1
@@ -76,7 +76,7 @@ steps:
|
||||
--max-instances 50 \
|
||||
--min-instances 0 \
|
||||
--set-env-vars "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,GCS_BUCKET_NAME=karaoke-gen-storage-$PROJECT_ID,FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run,KARAOKE_GEN_VERSION=$$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1" \
|
||||
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest"
|
||||
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,REDACTED_API_KEY=redacted-api-key:latest,OPS_API_KEY=ops-api-key:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest"
|
||||
|
||||
images:
|
||||
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# Codecov Configuration
|
||||
# Docs: https://docs.codecov.com/docs/codecovyml-reference
|
||||
|
||||
coverage:
|
||||
# Coverage value precision (2 decimal places)
|
||||
precision: 2
|
||||
round: down
|
||||
# Green range starts at 70% (our target)
|
||||
range: "70...100"
|
||||
|
||||
status:
|
||||
# Project-level coverage status check
|
||||
project:
|
||||
default:
|
||||
target: 70% # Pass if coverage >= 70%
|
||||
threshold: 2% # Allow 2% drop without failing
|
||||
if_ci_failed: error
|
||||
|
||||
# Patch coverage (new/changed lines only)
|
||||
patch:
|
||||
default:
|
||||
target: 70% # New code should also meet 70%
|
||||
threshold: 5% # More lenient for patches
|
||||
|
||||
# Disable PR comments entirely since we're satisfied with 70%+ coverage
|
||||
# Status checks will still appear on the PR
|
||||
comment: false
|
||||
|
||||
# Alternative: Uncomment below to only comment when coverage changes
|
||||
# comment:
|
||||
# layout: "diff, flags, files"
|
||||
# behavior: default
|
||||
# require_changes: true # Only comment if coverage actually changes
|
||||
# require_base: true # Require base report to exist
|
||||
|
||||
@@ -681,6 +681,8 @@ The karaoke-gen system supports multiple interfaces to the same core functionali
|
||||
| `dropbox-oauth-credentials` | ✅ | Dropbox API OAuth tokens |
|
||||
| `gdrive-service-account` | ✅ | Google Drive service account |
|
||||
| `discord-webhook-url` | ✅ | Default Discord webhook |
|
||||
| `redacted-api-key` | ⏳ | Redacted tracker API key (flacfetch audio search) |
|
||||
| `ops-api-key` | ⏳ | OPS/Orpheus tracker API key (flacfetch audio search) |
|
||||
|
||||
### Default Distribution Settings
|
||||
|
||||
|
||||
@@ -0,0 +1,608 @@
|
||||
# Observability & Tracing Plan for Karaoke Generator
|
||||
|
||||
**Created:** 2025-12-22
|
||||
**Status:** Planning
|
||||
**Priority:** High
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The karaoke-gen backend has **partial observability infrastructure** but **no rich tracing implemented**. While the code includes OpenTelemetry setup and tracing utilities, actual job processing is minimally instrumented. This means debugging job failures requires manually querying Cloud Logging with grep-style searches rather than clicking through a trace waterfall.
|
||||
|
||||
### Current State: 🟡 Partial
|
||||
|
||||
| Capability | Status | Notes |
|
||||
|------------|--------|-------|
|
||||
| **Logs exported to Cloud Logging** | ✅ Working | Standard Python logs visible |
|
||||
| **OpenTelemetry SDK installed** | ✅ Working | Dependencies in pyproject.toml |
|
||||
| **Cloud Trace exporter configured** | 🟡 Broken | 403 permission errors (fixed in infra today) |
|
||||
| **FastAPI auto-instrumentation** | ✅ Working | HTTP requests create spans |
|
||||
| **HTTPX instrumentation** | ✅ Working | Outgoing HTTP calls traced |
|
||||
| **Job ID correlation in logs** | ❌ Missing | Can't filter logs by job ID |
|
||||
| **Worker tracing spans** | ❌ Missing | No spans in audio/lyrics workers |
|
||||
| **Custom metrics** | ❌ Missing | No Cloud Monitoring metrics |
|
||||
| **Dashboard** | ❌ Missing | No pre-built observability view |
|
||||
| **Alerting** | ❌ Missing | No alerts for job failures |
|
||||
|
||||
### What You Can Do Today
|
||||
|
||||
```bash
|
||||
# Query Cloud Logging for a specific job (manual grep)
|
||||
gcloud logging read 'resource.type="cloud_run_revision" AND textPayload:"JOB_ID"' \
|
||||
--project=nomadkaraoke --limit=100
|
||||
|
||||
# View traces (once permission is fixed)
|
||||
# Go to: https://console.cloud.google.com/traces/list?project=nomadkaraoke
|
||||
```
|
||||
|
||||
### What You Should Be Able To Do
|
||||
|
||||
1. Go to a dashboard, enter job ID → see entire job lifecycle
|
||||
2. Click trace → see waterfall of audio worker, lyrics worker, Cloud Tasks
|
||||
3. See metrics: jobs/hour, success rate, p95 latency by stage
|
||||
4. Get alerted when job failure rate exceeds threshold
|
||||
|
||||
---
|
||||
|
||||
## Current Implementation Analysis
|
||||
|
||||
### 1. Tracing Infrastructure (Exists, Partially Working)
|
||||
|
||||
**File: `backend/services/tracing.py`**
|
||||
|
||||
```python
|
||||
# Good: SDK is set up, exporter is configured
|
||||
setup_tracing(service_name="karaoke-backend", service_version=VERSION)
|
||||
|
||||
# Good: Utility functions exist
|
||||
@traced("my-function") # Decorator for automatic spans
|
||||
with create_span("operation", {"job_id": job_id}): # Context manager
|
||||
add_span_attribute("key", "value") # Add to current span
|
||||
```
|
||||
|
||||
**Problem:** These utilities are barely used! Only `backend/api/routes/review.py` actually creates spans for business logic. The critical workers (`audio_worker.py`, `lyrics_worker.py`, etc.) have zero tracing.
|
||||
|
||||
### 2. Logging Infrastructure (Exists, Not Correlated)
|
||||
|
||||
**File: `backend/workers/worker_logging.py`**
|
||||
|
||||
```python
|
||||
# Good: Logs go to Firestore for CLI streaming
|
||||
class JobLogHandler(logging.Handler):
|
||||
"""Forwards logs to Firestore job document"""
|
||||
|
||||
# Good: Context vars prevent log mixing in concurrent jobs
|
||||
with job_logging_context(job_id):
|
||||
# logs isolated per job
|
||||
```
|
||||
|
||||
**Problem:** Logs go to Firestore for CLI display, but Cloud Logging has no job_id label. You can't filter Cloud Logging by job ID.
|
||||
|
||||
### 3. Permission Issues (Fixed Today)
|
||||
|
||||
The Cloud Trace exporter was failing with 403 errors because the Cloud Run service account lacked `roles/cloudtrace.agent`. This was just fixed in `infrastructure/__main__.py`:
|
||||
|
||||
```python
|
||||
# NEW: Added today
|
||||
cloud_trace_iam = gcp.projects.IAMMember(
|
||||
"karaoke-backend-cloudtrace-agent",
|
||||
role="roles/cloudtrace.agent",
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gap Analysis: What's Missing
|
||||
|
||||
### Gap 1: No Trace Context Propagation to Workers
|
||||
|
||||
When Cloud Tasks triggers a worker (audio, lyrics), a new trace is started. The parent trace from the initial API call is lost.
|
||||
|
||||
```
|
||||
Current:
|
||||
POST /uploads-complete → Trace A
|
||||
↓ (Cloud Task)
|
||||
POST /internal/workers/audio → NEW Trace B (no link to A!)
|
||||
```
|
||||
|
||||
**Should be:**
|
||||
```
|
||||
POST /uploads-complete → Trace A
|
||||
↓ (Cloud Task - propagates trace context)
|
||||
POST /internal/workers/audio → Trace A, Span B (child of A)
|
||||
```
|
||||
|
||||
### Gap 2: No Job ID on Traces
|
||||
|
||||
Even when traces exist, they don't have `job_id` as a searchable attribute. You can't search Cloud Trace for "all spans for job 8f03f0ac".
|
||||
|
||||
### Gap 3: No Custom Spans in Workers
|
||||
|
||||
The workers just log text. There are no spans for:
|
||||
- Audio download duration
|
||||
- Modal API call duration
|
||||
- GCS upload duration
|
||||
- Lyrics transcription duration
|
||||
- Each processing stage
|
||||
|
||||
### Gap 4: No Structured Logging with Job Context
|
||||
|
||||
Cloud Logging receives unstructured text logs. Should be structured JSON with `job_id`, `worker`, `stage` fields for filtering.
|
||||
|
||||
### Gap 5: No Metrics
|
||||
|
||||
No custom Cloud Monitoring metrics for:
|
||||
- Jobs started/completed/failed per hour
|
||||
- Processing time by stage (p50, p95, p99)
|
||||
- Queue depth
|
||||
- Error rates
|
||||
|
||||
### Gap 6: No Dashboard
|
||||
|
||||
No Cloud Console dashboard showing:
|
||||
- Active jobs
|
||||
- Success/failure rates
|
||||
- Latency distribution
|
||||
- Worker health
|
||||
|
||||
### Gap 7: No Alerting
|
||||
|
||||
No alerts for:
|
||||
- Job stuck in status > X minutes
|
||||
- Error rate spike
|
||||
- Worker health degradation
|
||||
|
||||
---
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Fix Core Tracing (2-3 hours) ⭐ High Priority
|
||||
|
||||
**Goal:** Every job should have a single trace that spans its entire lifecycle.
|
||||
|
||||
#### 1.1 Propagate Trace Context Through Cloud Tasks
|
||||
|
||||
**File: `backend/services/worker_service.py`**
|
||||
|
||||
```python
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry import trace
|
||||
|
||||
def _enqueue_cloud_task(self, queue: str, endpoint: str, payload: Dict, ...):
|
||||
# Get current trace context
|
||||
propagator = TraceContextTextMapPropagator()
|
||||
carrier = {}
|
||||
propagator.inject(carrier) # Injects traceparent header
|
||||
|
||||
# Add trace context to task headers
|
||||
headers = {
|
||||
"Authorization": f"Bearer {admin_token}",
|
||||
"Content-Type": "application/json",
|
||||
**carrier, # Add traceparent, tracestate
|
||||
}
|
||||
|
||||
task = {
|
||||
"http_request": {
|
||||
"http_method": "POST",
|
||||
"url": url,
|
||||
"headers": headers, # Now includes trace context
|
||||
"body": base64.b64encode(json.dumps(payload).encode()).decode(),
|
||||
},
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
**File: `backend/api/routes/internal.py`**
|
||||
|
||||
```python
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry import trace
|
||||
|
||||
@router.post("/workers/audio")
|
||||
async def trigger_audio_worker(request: Request, ...):
|
||||
# Extract trace context from headers
|
||||
propagator = TraceContextTextMapPropagator()
|
||||
context = propagator.extract(request.headers)
|
||||
|
||||
# Create span as child of propagated context
|
||||
with tracer.start_as_current_span(
|
||||
"audio-worker",
|
||||
context=context,
|
||||
attributes={"job_id": payload.job_id}
|
||||
):
|
||||
await process_audio_separation(payload.job_id)
|
||||
```
|
||||
|
||||
#### 1.2 Add Job ID to All Spans
|
||||
|
||||
Create a helper that adds job_id as a span attribute:
|
||||
|
||||
```python
|
||||
# backend/services/tracing.py
|
||||
|
||||
def job_span(name: str, job_id: str, **extra_attributes):
|
||||
"""Create a span with job_id attribute for easy filtering."""
|
||||
return create_span(name, {
|
||||
"job_id": job_id,
|
||||
"service.operation": name,
|
||||
**extra_attributes
|
||||
})
|
||||
```
|
||||
|
||||
Usage in workers:
|
||||
|
||||
```python
|
||||
# backend/workers/audio_worker.py
|
||||
|
||||
async def process_audio_separation(job_id: str):
|
||||
with job_span("audio-worker", job_id) as span:
|
||||
span.set_attribute("worker", "audio")
|
||||
|
||||
with job_span("download-audio", job_id):
|
||||
# download logic
|
||||
pass
|
||||
|
||||
with job_span("modal-separation", job_id):
|
||||
# Modal API call
|
||||
pass
|
||||
|
||||
with job_span("upload-stems", job_id):
|
||||
# GCS upload
|
||||
pass
|
||||
```
|
||||
|
||||
### Phase 2: Instrument All Workers (3-4 hours) ⭐ High Priority
|
||||
|
||||
Add spans to every worker following this pattern:
|
||||
|
||||
#### Audio Worker (`audio_worker.py`)
|
||||
|
||||
```python
|
||||
async def process_audio_separation(job_id: str):
|
||||
with job_span("audio-worker", job_id) as root:
|
||||
# Stage 1: Download/Get Audio
|
||||
with job_span("get-audio-file", job_id) as span:
|
||||
if job.audio_url:
|
||||
span.set_attribute("source", "url")
|
||||
with job_span("download-from-url", job_id):
|
||||
audio_path = await download_from_url(...)
|
||||
else:
|
||||
span.set_attribute("source", "gcs")
|
||||
with job_span("download-from-gcs", job_id):
|
||||
audio_path = storage.download_file(...)
|
||||
|
||||
# Stage 2: Audio Separation
|
||||
with job_span("audio-separation", job_id) as span:
|
||||
with job_span("clean-instrumental", job_id):
|
||||
# First separation pass
|
||||
pass
|
||||
with job_span("backing-vocals", job_id):
|
||||
# Second separation pass
|
||||
pass
|
||||
|
||||
# Stage 3: Upload Results
|
||||
with job_span("upload-stems", job_id) as span:
|
||||
span.set_attribute("stem_count", len(stems))
|
||||
for stem_name, stem_path in stems.items():
|
||||
with job_span("upload-stem", job_id, stem_name=stem_name):
|
||||
storage.upload_file(...)
|
||||
```
|
||||
|
||||
#### Lyrics Worker (`lyrics_worker.py`)
|
||||
|
||||
```python
|
||||
async def process_lyrics_transcription(job_id: str):
|
||||
with job_span("lyrics-worker", job_id) as root:
|
||||
with job_span("get-vocals", job_id):
|
||||
# Download vocals from GCS
|
||||
pass
|
||||
|
||||
with job_span("audioshake-transcription", job_id):
|
||||
# AudioShake API call
|
||||
pass
|
||||
|
||||
with job_span("fetch-genius-lyrics", job_id) as span:
|
||||
span.set_attribute("artist", artist)
|
||||
span.set_attribute("title", title)
|
||||
# Genius API call
|
||||
pass
|
||||
|
||||
with job_span("auto-correction", job_id):
|
||||
# LyricsTranscriber correction
|
||||
pass
|
||||
|
||||
with job_span("upload-corrections", job_id):
|
||||
# Upload corrections.json
|
||||
pass
|
||||
```
|
||||
|
||||
### Phase 3: Structured Logging with Job Context (1-2 hours)
|
||||
|
||||
Replace text logs with structured JSON that includes job context.
|
||||
|
||||
**File: `backend/main.py`**
|
||||
|
||||
```python
|
||||
import json
|
||||
import logging
|
||||
|
||||
class StructuredFormatter(logging.Formatter):
|
||||
"""JSON formatter with trace correlation."""
|
||||
|
||||
def format(self, record):
|
||||
from backend.services.tracing import get_current_trace_id, get_current_span_id
|
||||
|
||||
log_entry = {
|
||||
"timestamp": self.formatTime(record),
|
||||
"severity": record.levelname,
|
||||
"message": record.getMessage(),
|
||||
"logger": record.name,
|
||||
# Trace correlation
|
||||
"logging.googleapis.com/trace": f"projects/nomadkaraoke/traces/{get_current_trace_id()}" if get_current_trace_id() else None,
|
||||
"logging.googleapis.com/spanId": get_current_span_id(),
|
||||
# Custom fields (if available in log record)
|
||||
"job_id": getattr(record, 'job_id', None),
|
||||
"worker": getattr(record, 'worker', None),
|
||||
}
|
||||
return json.dumps({k: v for k, v in log_entry.items() if v is not None})
|
||||
|
||||
# Configure in Cloud Run
|
||||
if os.environ.get("K_SERVICE"):
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(StructuredFormatter())
|
||||
logging.root.handlers = [handler]
|
||||
```
|
||||
|
||||
Now logs in Cloud Logging can be filtered:
|
||||
```
|
||||
jsonPayload.job_id="8f03f0ac"
|
||||
```
|
||||
|
||||
### Phase 4: Custom Metrics (2-3 hours) 🟡 Medium Priority
|
||||
|
||||
Add Cloud Monitoring metrics for key indicators.
|
||||
|
||||
**File: `backend/services/metrics.py`**
|
||||
|
||||
```python
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.exporter.cloud_monitoring import CloudMonitoringMetricsExporter
|
||||
|
||||
def setup_metrics():
|
||||
exporter = CloudMonitoringMetricsExporter(project_id="nomadkaraoke")
|
||||
provider = MeterProvider(metric_readers=[...])
|
||||
metrics.set_meter_provider(provider)
|
||||
|
||||
meter = metrics.get_meter("karaoke-backend")
|
||||
|
||||
# Counter: Jobs by status
|
||||
jobs_counter = meter.create_counter(
|
||||
"jobs_total",
|
||||
description="Total jobs by status",
|
||||
unit="1"
|
||||
)
|
||||
|
||||
# Histogram: Processing duration by stage
|
||||
duration_histogram = meter.create_histogram(
|
||||
"job_stage_duration_seconds",
|
||||
description="Duration of job processing stages",
|
||||
unit="s"
|
||||
)
|
||||
|
||||
return {
|
||||
"jobs": jobs_counter,
|
||||
"duration": duration_histogram,
|
||||
}
|
||||
|
||||
# Usage
|
||||
metrics["jobs"].add(1, {"status": "completed", "worker": "audio"})
|
||||
metrics["duration"].record(45.2, {"stage": "audio-separation", "job_id": job_id})
|
||||
```
|
||||
|
||||
**Metrics to track:**
|
||||
|
||||
| Metric | Type | Labels |
|
||||
|--------|------|--------|
|
||||
| `jobs_total` | Counter | status, source (upload/url/search) |
|
||||
| `job_stage_duration_seconds` | Histogram | stage, worker |
|
||||
| `worker_invocations_total` | Counter | worker, success/failure |
|
||||
| `gcs_operations_total` | Counter | operation (upload/download), bucket |
|
||||
| `external_api_duration_seconds` | Histogram | api (modal/audioshake/genius) |
|
||||
|
||||
### Phase 5: Dashboard & Alerting (2-3 hours) 🟡 Medium Priority
|
||||
|
||||
#### Create Cloud Monitoring Dashboard
|
||||
|
||||
**File: `infrastructure/monitoring/dashboard.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"displayName": "Karaoke Backend Overview",
|
||||
"dashboardFilters": [],
|
||||
"mosaicLayout": {
|
||||
"columns": 12,
|
||||
"tiles": [
|
||||
{
|
||||
"title": "Active Jobs by Status",
|
||||
"scorecard": {
|
||||
"timeSeriesQuery": {
|
||||
"prometheusQuery": "sum by (status) (jobs_total)"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Job Processing Time (p95)",
|
||||
"xyChart": {
|
||||
"dataSets": [{
|
||||
"timeSeriesQuery": {
|
||||
"prometheusQuery": "histogram_quantile(0.95, rate(job_stage_duration_seconds_bucket[5m]))"
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Error Rate",
|
||||
"xyChart": {
|
||||
"dataSets": [{
|
||||
"timeSeriesQuery": {
|
||||
"prometheusQuery": "rate(worker_invocations_total{success=\"false\"}[5m]) / rate(worker_invocations_total[5m])"
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Cloud Tasks Queue Depth",
|
||||
"xyChart": {
|
||||
"dataSets": [{
|
||||
"timeSeriesQuery": {
|
||||
"prometheusQuery": "cloud_tasks_queue_depth"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Create Alerts
|
||||
|
||||
**File: `infrastructure/__main__.py`** (Pulumi)
|
||||
|
||||
```python
|
||||
# Alert: Job failure rate > 10%
|
||||
job_failure_alert = gcp.monitoring.AlertPolicy(
|
||||
"job-failure-rate-alert",
|
||||
display_name="High Job Failure Rate",
|
||||
conditions=[
|
||||
gcp.monitoring.AlertPolicyConditionArgs(
|
||||
display_name="Failure rate > 10%",
|
||||
condition_threshold=gcp.monitoring.AlertPolicyConditionConditionThresholdArgs(
|
||||
filter='metric.type="custom.googleapis.com/karaoke/job_failure_rate"',
|
||||
comparison="COMPARISON_GT",
|
||||
threshold_value=0.1,
|
||||
duration="300s",
|
||||
),
|
||||
),
|
||||
],
|
||||
notification_channels=[...], # Slack, email, etc.
|
||||
)
|
||||
|
||||
# Alert: Job stuck in processing > 30 min
|
||||
stuck_job_alert = gcp.monitoring.AlertPolicy(
|
||||
"stuck-job-alert",
|
||||
display_name="Job Stuck in Processing",
|
||||
conditions=[
|
||||
gcp.monitoring.AlertPolicyConditionArgs(
|
||||
display_name="Job > 30 min in processing state",
|
||||
condition_threshold=gcp.monitoring.AlertPolicyConditionConditionThresholdArgs(
|
||||
filter='metric.type="custom.googleapis.com/karaoke/job_age_seconds" AND metric.labels.status!="complete"',
|
||||
comparison="COMPARISON_GT",
|
||||
threshold_value=1800,
|
||||
duration="60s",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Wins (Do First)
|
||||
|
||||
These give the most debugging value with minimal effort:
|
||||
|
||||
### 1. Add Job ID to Worker Logs (30 min)
|
||||
|
||||
Simplest possible fix - add job_id to every log message:
|
||||
|
||||
```python
|
||||
# backend/workers/audio_worker.py
|
||||
logger.info(f"[job:{job_id}] Starting audio separation")
|
||||
logger.info(f"[job:{job_id}] Downloaded audio file: {path}")
|
||||
logger.info(f"[job:{job_id}] Modal separation complete")
|
||||
```
|
||||
|
||||
Now you can grep:
|
||||
```bash
|
||||
gcloud logging read 'textPayload:"[job:8f03f0ac]"' --project=nomadkaraoke
|
||||
```
|
||||
|
||||
### 2. Log Worker Start/End with Timing (30 min)
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
async def process_audio_separation(job_id: str):
|
||||
start = time.time()
|
||||
logger.info(f"[job:{job_id}] WORKER_START worker=audio")
|
||||
|
||||
try:
|
||||
# ... processing ...
|
||||
|
||||
duration = time.time() - start
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=audio status=success duration={duration:.1f}s")
|
||||
except Exception as e:
|
||||
duration = time.time() - start
|
||||
logger.error(f"[job:{job_id}] WORKER_END worker=audio status=error duration={duration:.1f}s error={e}")
|
||||
raise
|
||||
```
|
||||
|
||||
Now you can easily find failed workers:
|
||||
```bash
|
||||
gcloud logging read 'textPayload:"WORKER_END" AND textPayload:"status=error"' --project=nomadkaraoke
|
||||
```
|
||||
|
||||
### 3. Enable Cloud Trace Viewing (0 min - just fixed!)
|
||||
|
||||
With the IAM permission fix deployed today, Cloud Trace should start working. Visit:
|
||||
https://console.cloud.google.com/traces/list?project=nomadkaraoke
|
||||
|
||||
---
|
||||
|
||||
## Summary: Implementation Priority
|
||||
|
||||
| Phase | Effort | Value | Priority |
|
||||
|-------|--------|-------|----------|
|
||||
| Quick Win #1: Job ID in logs | 30 min | High | ⭐ Do now |
|
||||
| Quick Win #2: Worker timing | 30 min | High | ⭐ Do now |
|
||||
| Phase 1: Trace propagation | 2-3 hrs | Very High | ⭐ Do soon |
|
||||
| Phase 2: Instrument workers | 3-4 hrs | Very High | ⭐ Do soon |
|
||||
| Phase 3: Structured logging | 1-2 hrs | Medium | 🟡 Nice to have |
|
||||
| Phase 4: Custom metrics | 2-3 hrs | Medium | 🟡 Nice to have |
|
||||
| Phase 5: Dashboard & alerts | 2-3 hrs | Medium | 🟡 Nice to have |
|
||||
|
||||
**Total for full observability: ~12-15 hours**
|
||||
|
||||
**Minimum viable observability (Quick Wins + Phase 1-2): ~5-6 hours**
|
||||
|
||||
---
|
||||
|
||||
## Appendix: GCP Observability Resources
|
||||
|
||||
- **Cloud Trace:** https://console.cloud.google.com/traces/list?project=nomadkaraoke
|
||||
- **Cloud Logging:** https://console.cloud.google.com/logs?project=nomadkaraoke
|
||||
- **Cloud Monitoring:** https://console.cloud.google.com/monitoring?project=nomadkaraoke
|
||||
- **OpenTelemetry Python Docs:** https://opentelemetry.io/docs/instrumentation/python/
|
||||
- **Cloud Trace Python:** https://cloud.google.com/trace/docs/setup/python-ot
|
||||
|
||||
### Useful Log Queries
|
||||
|
||||
```
|
||||
# All logs for a specific job
|
||||
resource.type="cloud_run_revision" AND textPayload:"JOB_ID"
|
||||
|
||||
# All worker starts
|
||||
resource.type="cloud_run_revision" AND textPayload:"WORKER_START"
|
||||
|
||||
# All errors
|
||||
resource.type="cloud_run_revision" AND severity>=ERROR
|
||||
|
||||
# Cloud Tasks failures
|
||||
resource.type="cloud_tasks_queue"
|
||||
|
||||
# Trace-correlated logs (after structured logging)
|
||||
jsonPayload.job_id="JOB_ID"
|
||||
```
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
# Refactor: Unify Audio Fetcher Code
|
||||
|
||||
**Created:** 2025-12-20
|
||||
**Completed:** 2025-12-20
|
||||
**Priority:** MEDIUM (prevents future bugs like YouTubeProvider -> YoutubeProvider)
|
||||
**Status:** ✅ Completed
|
||||
|
||||
## Problem
|
||||
|
||||
The codebase has **two separate implementations** of flacfetch integration:
|
||||
1. `karaoke_gen/audio_fetcher.py` - Used by local CLI
|
||||
2. `backend/services/audio_search_service.py` - Used by cloud backend
|
||||
|
||||
This violates DRY and caused a bug where the backend's import wasn't updated when flacfetch renamed `YouTubeProvider` to `YoutubeProvider`.
|
||||
|
||||
## Current State
|
||||
|
||||
### Duplicated Code
|
||||
|
||||
| Component | Local (`audio_fetcher.py`) | Backend (`audio_search_service.py`) |
|
||||
|-----------|---------------------------|-------------------------------------|
|
||||
| `_get_manager()` | ✅ 40 lines | ❌ 35 lines (copy) |
|
||||
| Provider setup | ✅ Redacted, OPS, YouTube | ❌ Same, diverged |
|
||||
| `search()` | ✅ Full implementation | ❌ Similar, diverged |
|
||||
| `download()` | ✅ With downloaders | ⚠️ Different, simpler |
|
||||
| `AudioSearchResult` | ✅ Dataclass | ❌ Duplicate, different fields |
|
||||
| `AudioDownloadResult` | ✅ (`AudioFetchResult`) | ❌ Duplicate |
|
||||
| Exceptions | ✅ 4 types | ❌ 3 types, different names |
|
||||
|
||||
### Drift Examples
|
||||
- Backend missing `raw_result` field on `AudioSearchResult`
|
||||
- Backend has `index` field, local doesn't
|
||||
- Backend exceptions inherit from `AudioSearchError`, local from `AudioFetcherError`
|
||||
- Backend missing `UserCancelledError`
|
||||
- Backend missing downloader registration (TorrentDownloader, YoutubeDownloader)
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
### Option 1: Backend imports from karaoke_gen (Recommended)
|
||||
|
||||
The backend already depends on `karaoke_gen` (installed via `pip install -e .` in Dockerfile).
|
||||
|
||||
```python
|
||||
# backend/services/audio_search_service.py - REFACTORED
|
||||
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
FlacFetcher,
|
||||
AudioSearchResult,
|
||||
AudioFetchResult,
|
||||
AudioFetcherError,
|
||||
NoResultsError,
|
||||
DownloadError,
|
||||
)
|
||||
|
||||
class AudioSearchService:
|
||||
"""
|
||||
Thin wrapper around karaoke_gen.audio_fetcher.FlacFetcher.
|
||||
|
||||
Adds backend-specific functionality:
|
||||
- Firestore serialization (to_dict/from_dict)
|
||||
- GCS integration for downloads
|
||||
- Job state management
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redacted_api_key: Optional[str] = None,
|
||||
ops_api_key: Optional[str] = None,
|
||||
):
|
||||
self._fetcher = FlacFetcher(
|
||||
redacted_api_key=redacted_api_key,
|
||||
ops_api_key=ops_api_key,
|
||||
)
|
||||
|
||||
def search(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""Delegate to FlacFetcher.search()."""
|
||||
return self._fetcher.search(artist, title)
|
||||
|
||||
def select_best(self, results: List[AudioSearchResult]) -> int:
|
||||
"""Delegate to FlacFetcher.select_best()."""
|
||||
return self._fetcher.select_best(results)
|
||||
|
||||
def download(
|
||||
self,
|
||||
result: AudioSearchResult,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
) -> AudioFetchResult:
|
||||
"""Delegate to FlacFetcher.download()."""
|
||||
return self._fetcher.download(result, output_dir, output_filename)
|
||||
```
|
||||
|
||||
### Changes Required to karaoke_gen.audio_fetcher
|
||||
|
||||
1. **Add `to_dict()` method to `AudioSearchResult`** - For Firestore serialization
|
||||
2. **Add `from_dict()` classmethod** - For deserialization
|
||||
3. **Ensure `select_best()` is a standalone method** - Currently buried in interactive flow
|
||||
4. **Export all public classes from `__init__.py`**
|
||||
|
||||
### Changes Required to backend
|
||||
|
||||
1. **Remove duplicate dataclasses** - Use imports from karaoke_gen
|
||||
2. **Remove duplicate `_get_manager()`** - Delegate to FlacFetcher
|
||||
3. **Remove duplicate exception classes** - Use karaoke_gen exceptions
|
||||
4. **Update tests** - Point to real classes, not mocks
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **Single source of truth** - Bug fixes apply everywhere
|
||||
2. **No more drift** - Dataclasses, exceptions stay in sync
|
||||
3. **Better test coverage** - karaoke_gen has integration tests for flacfetch imports
|
||||
4. **Less code to maintain** - Backend becomes thin wrapper
|
||||
|
||||
## Migration Steps
|
||||
|
||||
1. [x] Add `to_dict()` / `from_dict()` to `AudioSearchResult` in karaoke_gen
|
||||
2. [x] Extract `select_best()` as standalone method in FlacFetcher
|
||||
3. [x] Update karaoke_gen `__init__.py` exports (with lazy imports)
|
||||
4. [x] Refactor backend to import from karaoke_gen
|
||||
5. [x] Update backend tests
|
||||
6. [x] Delete duplicate code from backend (~150 lines removed)
|
||||
7. [x] Add integration test that catches import divergence
|
||||
|
||||
## Related Files
|
||||
|
||||
- `karaoke_gen/audio_fetcher.py` - Source of truth (keep)
|
||||
- `backend/services/audio_search_service.py` - Refactor to thin wrapper
|
||||
- `backend/api/routes/audio_search.py` - Update imports
|
||||
- `backend/tests/test_audio_search.py` - Update tests
|
||||
|
||||
## Estimated Effort
|
||||
|
||||
- **Time:** 2-4 hours
|
||||
- **Risk:** Low (well-defined changes, good test coverage)
|
||||
- **Dependencies:** None
|
||||
|
||||
## Related Architecture Docs
|
||||
|
||||
- `docs/01-reference/ARCHITECTURE.md` - "LyricsTranscriber Integration" section shows similar pattern
|
||||
- `docs/00-current-plan/SCALABLE-ARCHITECTURE-PLAN.md` - "Shared pipeline architecture" goal
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
# Backend Testing Gaps Analysis
|
||||
|
||||
**Created:** 2025-12-22
|
||||
**Updated:** 2025-12-22
|
||||
**Status:** Mostly Complete ✅
|
||||
**Priority:** High
|
||||
|
||||
## Background
|
||||
|
||||
During end-to-end testing of the remote karaoke-gen CLI, multiple bugs were discovered:
|
||||
|
||||
### Bug #1: Missing `extra_info` Attribute (Fixed 2025-12-22)
|
||||
```
|
||||
Error: 'AudioSearchResult' object has no attribute 'extra_info'
|
||||
```
|
||||
|
||||
This bug existed in production code but wasn't caught by automated tests because the remote download code path had no test coverage.
|
||||
|
||||
### Bug #2: Missing `gcs_path` Parameter in API Route (Fixed 2025-12-22)
|
||||
```
|
||||
Error handling audio selection: Error selecting audio: {'detail': "Download failed: [Errno 2] No such file or directory: '/var/lib/transmission-daemon/downloads/Avril Lavigne - Unwanted (Album, 2002, CD, 16bit, Redacted).flac'"}
|
||||
```
|
||||
|
||||
The backend API route `_download_and_start_processing` in `audio_search.py` was not passing the `gcs_path` parameter to `audio_search_service.download()`. Without this parameter, the remote `flacfetch` VM would download the file locally but not upload it to GCS. The backend then tried to open the VM's local path (`/var/lib/transmission-daemon/downloads/...`) which doesn't exist on Cloud Run.
|
||||
|
||||
**Why this wasn't caught:**
|
||||
- Service-level tests for `AudioSearchService.download()` existed and tested `gcs_path` parameter
|
||||
- But the **API route caller** (`_download_and_start_processing`) that invokes this service was never tested
|
||||
- Tests validated the service layer but not the route layer's usage of the service
|
||||
|
||||
## Root Cause Analysis
|
||||
|
||||
1. **Missing remote code path tests**: The download routing logic for `provider in ["Redacted", "OPS"]` was never exercised
|
||||
2. **State not set up for remote scenario**: Tests never configured `_remote_search_id` and `_remote_client` together
|
||||
3. **`FlacfetchClient` had zero tests**: The entire HTTP client was untested
|
||||
4. **Tests mocked at wrong level**: Tests mocked `service._fetcher` completely, bypassing the routing logic
|
||||
5. **API route orchestration untested**: The `_download_and_start_processing` function in `api/routes/audio_search.py` orchestrates multiple services but had no dedicated tests verifying correct parameter passing
|
||||
|
||||
## Services Testing Status
|
||||
|
||||
### ✅ Fully Tested
|
||||
|
||||
| Service | Test File | Test Count | Notes |
|
||||
|---------|-----------|------------|-------|
|
||||
| `credential_manager.py` | `test_credential_manager.py` | 27 | Good coverage |
|
||||
| `job_manager.py` | `test_job_manager.py` | 15 | Good coverage |
|
||||
| `flacfetch_client.py` | `test_flacfetch_client.py` | 26 | **Added 2025-12-22** |
|
||||
| `audio_search_service.py` | `test_audio_search.py` | 43 | **+11 tests 2025-12-22** |
|
||||
| `api/routes/audio_search.py` | `test_audio_search.py` | 4 | **API route tests added 2025-12-22** |
|
||||
| `storage_service.py` | `test_storage_service.py` | 17 | **Added 2025-12-22** |
|
||||
| `audio_analysis_service.py` | `test_audio_analysis_service.py` | 9 | **Added 2025-12-22** |
|
||||
| `audio_editing_service.py` | `test_audio_editing_service.py` | 13 | **Added 2025-12-22** |
|
||||
| `youtube_service.py` | `test_youtube_service.py` | 12 | **Added 2025-12-22** |
|
||||
| `dropbox_service.py` | `test_dropbox_service.py` | 14 | **Added 2025-12-22** |
|
||||
| `gdrive_service.py` | `test_gdrive_service.py` | 13 | **Added 2025-12-22** |
|
||||
|
||||
### ❌ No Dedicated Test File (Lower Priority)
|
||||
|
||||
| Service | Priority | Complexity | Notes |
|
||||
|---------|----------|------------|-------|
|
||||
| `rclone_service.py` | Low | Low | Wrapper around rclone CLI |
|
||||
| `firestore_service.py` | Low | Medium | Firestore operations |
|
||||
| `auth_service.py` | Low | Low | Auth utilities |
|
||||
| `job_logging.py` | Low | Low | Logging utilities |
|
||||
| `tracing.py` | Low | Low | OpenTelemetry setup |
|
||||
| `worker_service.py` | Medium | High | Worker orchestration |
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Core Audio Services (High Priority) ✅
|
||||
|
||||
- [x] `flacfetch_client.py` - Remote audio download client
|
||||
- [x] `audio_search_service.py` - Remote download path tests
|
||||
|
||||
### Phase 2: Audio Processing Services (High Priority) ✅
|
||||
|
||||
- [x] `audio_analysis_service.py` - Waveform analysis, peak detection
|
||||
- [x] `audio_editing_service.py` - Mute regions, audio manipulation
|
||||
|
||||
### Phase 3: Distribution Services (Medium Priority) ✅
|
||||
|
||||
- [x] `youtube_service.py` - YouTube credential management
|
||||
- [x] `dropbox_service.py` - Dropbox upload API
|
||||
- [x] `gdrive_service.py` - Google Drive upload API
|
||||
- [x] `storage_service.py` - GCS operations
|
||||
|
||||
### Phase 4: Infrastructure Services (Low Priority)
|
||||
|
||||
- [ ] `worker_service.py` - Worker orchestration
|
||||
- [ ] `firestore_service.py` - Firestore operations
|
||||
- [ ] `rclone_service.py` - Rclone wrapper
|
||||
- [ ] `auth_service.py` - Auth utilities
|
||||
- [ ] `job_logging.py` - Logging
|
||||
- [ ] `tracing.py` - Tracing
|
||||
|
||||
## Test Coverage Goals
|
||||
|
||||
- **Target:** 80%+ line coverage for high-priority services
|
||||
- **Minimum:** Key code paths tested (happy path + error handling)
|
||||
- **Integration:** Mock external APIs (YouTube, Dropbox, GCS, etc.)
|
||||
|
||||
## Testing Patterns to Follow
|
||||
|
||||
### 1. Mock External Services
|
||||
|
||||
```python
|
||||
@patch("backend.services.youtube_service.build")
|
||||
def test_upload_video(self, mock_build):
|
||||
mock_service = Mock()
|
||||
mock_build.return_value = mock_service
|
||||
# ...
|
||||
```
|
||||
|
||||
### 2. Test Error Handling
|
||||
|
||||
```python
|
||||
def test_upload_handles_network_error(self):
|
||||
with pytest.raises(ServiceError) as exc_info:
|
||||
service.upload(...)
|
||||
assert "network" in str(exc_info.value).lower()
|
||||
```
|
||||
|
||||
### 3. Test State Transitions
|
||||
|
||||
```python
|
||||
def test_download_routes_to_remote_for_torrent_providers(self):
|
||||
service._remote_client = Mock()
|
||||
service._remote_search_id = "search_123"
|
||||
service._cached_results = [result_with_provider_redacted]
|
||||
|
||||
service.download(0, "/tmp")
|
||||
|
||||
service._download_remote.assert_called_once()
|
||||
```
|
||||
|
||||
### 4. Verify Request/Response Format
|
||||
|
||||
```python
|
||||
def test_search_sends_correct_payload(self):
|
||||
# ...
|
||||
call_args = mock_client.post.call_args
|
||||
assert call_args.kwargs["json"] == {"artist": "ABBA", "title": "Waterloo"}
|
||||
```
|
||||
|
||||
### 5. Test API Route Orchestration (CRITICAL)
|
||||
|
||||
**Key Lesson from Bug #2:** Service-level tests are not sufficient. API routes that orchestrate multiple services must also be tested to verify correct parameter passing.
|
||||
|
||||
```python
|
||||
class TestAudioSearchApiRouteDownload:
|
||||
"""Tests for the _download_and_start_processing function in audio_search.py."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remote_download_passes_gcs_path(self):
|
||||
"""Verify that gcs_path is passed to audio_search_service.download for torrent providers."""
|
||||
# Mock the audio_search_service.download to capture the call
|
||||
mock_audio_search_service = Mock()
|
||||
mock_audio_search_service.download.return_value = AudioDownloadResult(
|
||||
filepath="gs://bucket/path.flac", artist="A", title="T", provider="Redacted", quality="FLAC"
|
||||
)
|
||||
|
||||
result = await _download_and_start_processing(job_id="job1", ...)
|
||||
|
||||
# Assert gcs_path was passed
|
||||
call_kwargs = mock_audio_search_service.download.call_args.kwargs
|
||||
assert call_kwargs.get('gcs_path') is not None # CRITICAL check!
|
||||
assert call_kwargs['gcs_path'].startswith('uploads/')
|
||||
```
|
||||
|
||||
This pattern ensures that when service APIs change or new required parameters are added, API route tests will catch missing parameter passing.
|
||||
|
||||
## Files Changed
|
||||
|
||||
**Bug Fixes:**
|
||||
- `backend/services/audio_search_service.py` - Removed `extra_info` attribute access (Bug #1)
|
||||
- `backend/api/routes/audio_search.py` - Added `gcs_path` parameter to `audio_search_service.download()` call (Bug #2)
|
||||
|
||||
**Tests Added (2025-12-22):**
|
||||
- `backend/tests/test_audio_search.py` - +11 tests for service + 4 tests for API route orchestration
|
||||
- `backend/tests/test_flacfetch_client.py` - 26 new tests (new file)
|
||||
- `backend/tests/test_storage_service.py` - 17 new tests (new file)
|
||||
- `backend/tests/test_audio_analysis_service.py` - 9 new tests (new file)
|
||||
- `backend/tests/test_audio_editing_service.py` - 13 new tests (new file)
|
||||
- `backend/tests/test_youtube_service.py` - 12 new tests (new file)
|
||||
- `backend/tests/test_dropbox_service.py` - 14 new tests (new file)
|
||||
- `backend/tests/test_gdrive_service.py` - 13 new tests (new file)
|
||||
|
||||
**Total New Tests:** 119 tests added
|
||||
|
||||
**Version:** 0.74.3
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# All backend tests
|
||||
cd backend && python -m pytest tests/ -v
|
||||
|
||||
# Specific service tests
|
||||
python -m pytest tests/test_flacfetch_client.py -v
|
||||
python -m pytest tests/test_audio_search.py -v
|
||||
|
||||
# With coverage
|
||||
python -m pytest tests/ -v --cov=services --cov-report=term-missing
|
||||
```
|
||||
|
||||
## Notes for Future Sessions
|
||||
|
||||
When continuing this work:
|
||||
|
||||
1. **Test API route orchestration, not just services**: Bug #2 showed that service tests alone don't catch parameter passing bugs at the route level
|
||||
2. Distribution services need OAuth mocking - check existing patterns in `test_credential_manager.py`
|
||||
3. Use `pytest.mark.asyncio` for async service methods
|
||||
4. Follow existing test patterns in `test_flacfetch_client.py` for HTTP client mocking
|
||||
5. **For remote operations**: Always verify that `gcs_path` parameters are passed when code paths involve remote VM operations
|
||||
|
||||
## Lessons Learned
|
||||
|
||||
### Testing Layers Matter
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ API Routes (api/routes/*.py) │
|
||||
│ └─> These orchestrate services and MUST be tested separately │
|
||||
│ to verify correct parameter passing │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ Services (services/*.py) │
|
||||
│ └─> Business logic with proper unit tests │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ Clients (FlacfetchClient, etc.) │
|
||||
│ └─> HTTP/external API interactions │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Bug #2 was missed because:
|
||||
- `FlacfetchClient` tests ✅ verified `gcs_path` is sent in HTTP requests
|
||||
- `AudioSearchService` tests ✅ verified `gcs_path` is passed to client
|
||||
- **API route tests ❌ didn't exist** - so nobody verified the route passes `gcs_path` to the service
|
||||
|
||||
Now we have `TestAudioSearchApiRouteDownload` to catch this class of bugs.
|
||||
|
||||
@@ -0,0 +1,834 @@
|
||||
# Flacfetch HTTP API Service Implementation Plan
|
||||
|
||||
**Created:** 2025-12-20
|
||||
**Updated:** 2025-12-22
|
||||
**Status:** Complete ✅
|
||||
**Priority:** Medium
|
||||
**Estimated Effort:** 3-4 days (actual: ~3 days)
|
||||
|
||||
## Problem Statement
|
||||
|
||||
BitTorrent downloads don't work reliably in Cloud Run due to:
|
||||
- No inbound peer connections (NAT/firewall)
|
||||
- Ephemeral containers with no stable IP
|
||||
- DHT peer discovery blocked
|
||||
- Container timeout limits (max 60 min)
|
||||
|
||||
This prevents downloading lossless audio from private trackers (Redacted, OPS) via the remote backend.
|
||||
|
||||
## Solution Overview
|
||||
|
||||
Add an HTTP API mode to **flacfetch** (our audio fetching library) and deploy it on a dedicated GCP VM with:
|
||||
- Full BitTorrent peer connectivity via Transmission daemon
|
||||
- Static IP for tracker whitelist (seedbox-like)
|
||||
- Indefinite seeding of downloaded torrents
|
||||
- Automatic disk cleanup when space runs low
|
||||
- YouTube downloads also handled (single service for all audio acquisition)
|
||||
|
||||
This approach provides clean separation of concerns:
|
||||
- **flacfetch**: Handles ALL audio acquisition (torrents + YouTube)
|
||||
- **karaoke-gen backend**: Orchestrates karaoke generation pipeline, calls flacfetch API
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ FLACFETCH SERVICE ARCHITECTURE │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ Cloud Run Backend Flacfetch VM (e2-small) │
|
||||
│ ┌────────────────────┐ ┌─────────────────────────┐ │
|
||||
│ │ │ │ │ │
|
||||
│ │ karaoke-gen │ 1. Search │ flacfetch HTTP API │ │
|
||||
│ │ AudioSearchService │ ────────────▶ │ POST /search │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 2. Results │ Providers: │ │
|
||||
│ │ │ ◀──────────── │ • Redacted (FLAC) │ │
|
||||
│ │ │ │ • OPS (FLAC) │ │
|
||||
│ │ │ 3. Download │ • YouTube (fallback) │ │
|
||||
│ │ │ ────────────▶ │ │ │
|
||||
│ │ │ │ POST /download │ │
|
||||
│ │ │ 4. Status │ GET /download/{id} │ │
|
||||
│ │ │ ◀──────────── │ │ │
|
||||
│ │ │ (polling) │ Transmission Daemon │ │
|
||||
│ │ │ │ • Full peer connect │ │
|
||||
│ │ │ 5. Complete │ • DHT enabled │ │
|
||||
│ │ │ ◀──────────── │ • Indefinite seeding │ │
|
||||
│ │ │ (file path) │ │ │
|
||||
│ └────────────────────┘ └───────────┬─────────────┘ │
|
||||
│ │ │ │
|
||||
│ │ │ 6. Upload to GCS │
|
||||
│ │ │ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌─────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Google Cloud Storage │ │
|
||||
│ │ │ │
|
||||
│ │ uploads/{job_id}/audio/{filename}.flac │ │
|
||||
│ │ │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
### 1. HTTP API in flacfetch (not separate service)
|
||||
- **Why**: Clean separation of concerns, reusable by future projects, single source of truth
|
||||
- **How**: Add `flacfetch/api/` module with FastAPI, new CLI mode `flacfetch serve`
|
||||
|
||||
### 2. Indefinite Seeding
|
||||
- **Why**: VM is always-on anyway, helps maintain tracker ratio, acts like a seedbox
|
||||
- **Current behavior**: Torrent removed after download (line 317-321 in torrent.py)
|
||||
- **New behavior**: Torrent kept seeding, only removed by cleanup or explicit delete
|
||||
|
||||
### 3. Static IP
|
||||
- **Why**: Required for tracker account IP whitelist, consistent for firewall rules
|
||||
- **Cost**: ~$3/month additional
|
||||
|
||||
### 4. Disk Space Management
|
||||
- **Strategy**: Auto-cleanup oldest torrents when free space < 5GB threshold
|
||||
- **API**: Manual cleanup endpoints + automatic background cleanup
|
||||
|
||||
### 5. Instance Type: e2-small
|
||||
- **Specs**: 0.5 vCPU, 2GB RAM, ~$12/month
|
||||
- **Why**: Sufficient for I/O-bound workload, newer C3/N4 require 2+ vCPUs (overkill)
|
||||
- **Disk**: 30GB SSD (20GB for torrents + headroom)
|
||||
|
||||
### 6. Authentication
|
||||
- **Method**: Static API key in `X-API-Key` header
|
||||
- **Firewall**: Permissive (allows future web UI), auth enforced at app level
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Add HTTP API to flacfetch
|
||||
|
||||
**Location:** `./flacfetch/flacfetch/api/`
|
||||
|
||||
**New Directory Structure:**
|
||||
```
|
||||
flacfetch/
|
||||
├── flacfetch/
|
||||
│ ├── api/ # NEW
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── main.py # FastAPI app entry point
|
||||
│ │ ├── auth.py # API key authentication
|
||||
│ │ ├── models.py # Pydantic request/response models
|
||||
│ │ ├── routes/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── search.py # POST /search
|
||||
│ │ │ ├── download.py # POST /download, GET /download/{id}/status
|
||||
│ │ │ ├── torrents.py # GET /torrents, DELETE /torrents/{id}
|
||||
│ │ │ └── health.py # GET /health, GET /disk-usage
|
||||
│ │ └── services/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── download_manager.py # Track active downloads
|
||||
│ │ └── disk_manager.py # Disk cleanup logic
|
||||
│ ├── core/
|
||||
│ ├── downloaders/
|
||||
│ │ └── torrent.py # MODIFIED: optional keep-seeding mode
|
||||
│ ├── interface/
|
||||
│ │ └── cli.py # MODIFIED: add 'serve' subcommand
|
||||
│ └── providers/
|
||||
└── pyproject.toml # MODIFIED: add fastapi, uvicorn deps
|
||||
```
|
||||
|
||||
**API Endpoints:**
|
||||
|
||||
```yaml
|
||||
# All endpoints require X-API-Key header
|
||||
|
||||
# Search for audio
|
||||
POST /search
|
||||
Request:
|
||||
{ "artist": "ABBA", "title": "Waterloo" }
|
||||
Response:
|
||||
{
|
||||
"search_id": "abc123",
|
||||
"results": [
|
||||
{
|
||||
"index": 0,
|
||||
"title": "Waterloo",
|
||||
"artist": "ABBA",
|
||||
"provider": "Redacted",
|
||||
"quality": "FLAC 16bit CD",
|
||||
"seeders": 45,
|
||||
"size_bytes": 31457280,
|
||||
"target_file": "02. Waterloo.flac",
|
||||
...
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
# Start download
|
||||
POST /download
|
||||
Request:
|
||||
{
|
||||
"search_id": "abc123", # From previous search
|
||||
"result_index": 0, # Which result to download
|
||||
"output_filename": "ABBA - Waterloo", # Optional
|
||||
"upload_to_gcs": true, # Upload to GCS when complete
|
||||
"gcs_path": "uploads/job123/audio/" # Required if upload_to_gcs
|
||||
}
|
||||
Response:
|
||||
{
|
||||
"download_id": "dl_xyz789",
|
||||
"status": "queued"
|
||||
}
|
||||
|
||||
# Check download status
|
||||
GET /download/{download_id}/status
|
||||
Response:
|
||||
{
|
||||
"download_id": "dl_xyz789",
|
||||
"status": "downloading", # queued, downloading, uploading, seeding, complete, failed
|
||||
"progress": 45.2,
|
||||
"peers": 3,
|
||||
"download_speed_kbps": 1250.5,
|
||||
"eta_seconds": 120,
|
||||
"provider": "Redacted",
|
||||
"title": "ABBA - Waterloo",
|
||||
"output_path": null, # Set when complete (local path)
|
||||
"gcs_path": null, # Set when uploaded to GCS
|
||||
"error": null
|
||||
}
|
||||
|
||||
# List all torrents (for management)
|
||||
GET /torrents
|
||||
Response:
|
||||
{
|
||||
"torrents": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "ABBA - Waterloo [FLAC]",
|
||||
"status": "seeding",
|
||||
"progress": 100.0,
|
||||
"size_bytes": 314572800,
|
||||
"uploaded_bytes": 628145600, # Seeding stats
|
||||
"ratio": 2.0,
|
||||
"added_at": "2025-12-20T10:30:00Z"
|
||||
},
|
||||
...
|
||||
],
|
||||
"total_size_bytes": 5368709120,
|
||||
"count": 15
|
||||
}
|
||||
|
||||
# Delete a torrent (manual cleanup)
|
||||
DELETE /torrents/{torrent_id}
|
||||
Query params: ?delete_data=true # Also delete downloaded files
|
||||
Response:
|
||||
{ "status": "deleted" }
|
||||
|
||||
# Trigger cleanup (free disk space)
|
||||
POST /torrents/cleanup
|
||||
Request:
|
||||
{
|
||||
"strategy": "oldest", # oldest, largest, lowest_ratio
|
||||
"target_free_gb": 10 # Keep removing until this much free
|
||||
}
|
||||
Response:
|
||||
{
|
||||
"removed_count": 3,
|
||||
"freed_bytes": 1073741824,
|
||||
"free_space_gb": 10.5
|
||||
}
|
||||
|
||||
# Health check
|
||||
GET /health
|
||||
Response:
|
||||
{
|
||||
"status": "healthy",
|
||||
"transmission": {
|
||||
"available": true,
|
||||
"version": "4.0.5",
|
||||
"active_torrents": 5
|
||||
},
|
||||
"disk": {
|
||||
"total_gb": 30,
|
||||
"used_gb": 18,
|
||||
"free_gb": 12
|
||||
},
|
||||
"providers": {
|
||||
"redacted": true,
|
||||
"ops": true,
|
||||
"youtube": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Torrent Downloader Changes:**
|
||||
|
||||
```python
|
||||
# flacfetch/downloaders/torrent.py
|
||||
|
||||
class TorrentDownloader(Downloader):
|
||||
def __init__(self, ..., keep_seeding: bool = False):
|
||||
"""
|
||||
Args:
|
||||
keep_seeding: If True, don't remove torrent after download.
|
||||
Used in server mode for indefinite seeding.
|
||||
"""
|
||||
self.keep_seeding = keep_seeding
|
||||
|
||||
def download(self, release, output_path, output_filename=None):
|
||||
# ... existing download logic ...
|
||||
|
||||
# After download complete:
|
||||
if self.keep_seeding:
|
||||
# Don't remove, just return the file path
|
||||
logger.info(f"Download complete, keeping torrent for seeding: {torrent.name}")
|
||||
else:
|
||||
# Original behavior: stop and remove
|
||||
self.client.stop_torrent(torrent.id)
|
||||
self.client.remove_torrent(torrent.id, delete_data=True)
|
||||
```
|
||||
|
||||
**CLI Changes:**
|
||||
|
||||
```bash
|
||||
# New serve subcommand
|
||||
flacfetch serve --port 8080 --host 0.0.0.0
|
||||
|
||||
# Or via environment
|
||||
FLACFETCH_API_PORT=8080 flacfetch serve
|
||||
```
|
||||
|
||||
### Phase 2: Infrastructure (Pulumi)
|
||||
|
||||
**Location:** `./infrastructure/__main__.py`
|
||||
|
||||
**New Resources:**
|
||||
```python
|
||||
# Static IP for flacfetch service (for tracker whitelist)
|
||||
flacfetch_ip = gcp.compute.Address(
|
||||
"flacfetch-ip",
|
||||
name="flacfetch-static-ip",
|
||||
region="us-central1",
|
||||
)
|
||||
|
||||
# Service account for flacfetch VM
|
||||
flacfetch_sa = gcp.serviceaccount.Account(
|
||||
"flacfetch-sa",
|
||||
account_id="flacfetch-service",
|
||||
display_name="Flacfetch Service Account",
|
||||
)
|
||||
|
||||
# GCS write permissions for flacfetch
|
||||
gcp.storage.BucketIAMMember(
|
||||
"flacfetch-gcs-writer",
|
||||
bucket=bucket.name,
|
||||
role="roles/storage.objectCreator",
|
||||
member=pulumi.Output.concat("serviceAccount:", flacfetch_sa.email),
|
||||
)
|
||||
|
||||
# Flacfetch VM
|
||||
flacfetch_vm = gcp.compute.Instance(
|
||||
"flacfetch-service",
|
||||
machine_type="e2-small",
|
||||
zone="us-central1-a",
|
||||
boot_disk=gcp.compute.InstanceBootDiskArgs(
|
||||
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=30, # GB
|
||||
type="pd-ssd",
|
||||
),
|
||||
),
|
||||
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs(
|
||||
nat_ip=flacfetch_ip.address, # Static IP
|
||||
)],
|
||||
)],
|
||||
service_account=gcp.compute.InstanceServiceAccountArgs(
|
||||
email=flacfetch_sa.email,
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
metadata_startup_script=FLACFETCH_STARTUP_SCRIPT,
|
||||
tags=["flacfetch-service"],
|
||||
)
|
||||
|
||||
# Firewall for flacfetch
|
||||
flacfetch_firewall = gcp.compute.Firewall(
|
||||
"flacfetch-firewall",
|
||||
network="default",
|
||||
allows=[
|
||||
# BitTorrent peer connections
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["51413"]),
|
||||
gcp.compute.FirewallAllowArgs(protocol="udp", ports=["51413"]),
|
||||
# Flacfetch HTTP API (auth required at app level)
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["8080"]),
|
||||
],
|
||||
source_ranges=["0.0.0.0/0"], # Permissive, auth at app level
|
||||
target_tags=["flacfetch-service"],
|
||||
)
|
||||
|
||||
# Store API key in Secret Manager
|
||||
flacfetch_api_key = gcp.secretmanager.Secret(
|
||||
"flacfetch-api-key",
|
||||
secret_id="flacfetch-api-key",
|
||||
replication=gcp.secretmanager.SecretReplicationArgs(
|
||||
auto=gcp.secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Export for use
|
||||
pulumi.export("flacfetch_static_ip", flacfetch_ip.address)
|
||||
pulumi.export("flacfetch_service_url", pulumi.Output.concat("http://", flacfetch_ip.address, ":8080"))
|
||||
```
|
||||
|
||||
**Startup Script:**
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Install dependencies
|
||||
apt-get update
|
||||
apt-get install -y python3-pip python3-venv transmission-daemon ffmpeg git
|
||||
|
||||
# Configure Transmission
|
||||
systemctl stop transmission-daemon
|
||||
cat > /etc/transmission-daemon/settings.json << 'EOF'
|
||||
{
|
||||
"download-dir": "/var/lib/transmission-daemon/downloads",
|
||||
"incomplete-dir": "/var/lib/transmission-daemon/.incomplete",
|
||||
"incomplete-dir-enabled": true,
|
||||
"rpc-authentication-required": false,
|
||||
"rpc-bind-address": "127.0.0.1",
|
||||
"rpc-enabled": true,
|
||||
"rpc-port": 9091,
|
||||
"rpc-whitelist-enabled": false,
|
||||
"peer-port": 51413,
|
||||
"port-forwarding-enabled": false,
|
||||
"speed-limit-down": 0,
|
||||
"speed-limit-down-enabled": false,
|
||||
"speed-limit-up": 0,
|
||||
"speed-limit-up-enabled": false,
|
||||
"ratio-limit-enabled": false,
|
||||
"umask": 2
|
||||
}
|
||||
EOF
|
||||
systemctl start transmission-daemon
|
||||
|
||||
# Install flacfetch
|
||||
cd /opt
|
||||
git clone https://github.com/nomadkaraoke/flacfetch.git
|
||||
cd flacfetch
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -e ".[api]"
|
||||
|
||||
# Get secrets from metadata/Secret Manager
|
||||
FLACFETCH_API_KEY=$(gcloud secrets versions access latest --secret=flacfetch-api-key)
|
||||
REDACTED_API_KEY=$(gcloud secrets versions access latest --secret=redacted-api-key)
|
||||
OPS_API_KEY=$(gcloud secrets versions access latest --secret=ops-api-key)
|
||||
GCS_BUCKET=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcs-bucket" -H "Metadata-Flavor: Google")
|
||||
|
||||
# Create systemd service
|
||||
cat > /etc/systemd/system/flacfetch.service << EOF
|
||||
[Unit]
|
||||
Description=Flacfetch HTTP API Service
|
||||
After=network.target transmission-daemon.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/flacfetch
|
||||
Environment="FLACFETCH_API_KEY=${FLACFETCH_API_KEY}"
|
||||
Environment="REDACTED_API_KEY=${REDACTED_API_KEY}"
|
||||
Environment="OPS_API_KEY=${OPS_API_KEY}"
|
||||
Environment="GCS_BUCKET=${GCS_BUCKET}"
|
||||
Environment="FLACFETCH_KEEP_SEEDING=true"
|
||||
Environment="FLACFETCH_MIN_FREE_GB=5"
|
||||
ExecStart=/opt/flacfetch/venv/bin/flacfetch serve --host 0.0.0.0 --port 8080
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable flacfetch
|
||||
systemctl start flacfetch
|
||||
```
|
||||
|
||||
### Phase 3: Backend Integration
|
||||
|
||||
**New File:** `backend/services/flacfetch_client.py`
|
||||
|
||||
```python
|
||||
"""
|
||||
Client for the remote flacfetch HTTP API service.
|
||||
|
||||
Used when FLACFETCH_API_URL is configured, otherwise falls back to local flacfetch.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FlacfetchServiceError(Exception):
|
||||
"""Error communicating with flacfetch service."""
|
||||
pass
|
||||
|
||||
|
||||
class FlacfetchClient:
|
||||
"""Client for remote flacfetch HTTP API."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str, timeout: int = 30):
|
||||
self.base_url = base_url.rstrip('/')
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
return {"X-API-Key": self.api_key}
|
||||
|
||||
async def health_check(self) -> Dict[str, Any]:
|
||||
"""Check if service is healthy."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/health",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def search(self, artist: str, title: str) -> Dict[str, Any]:
|
||||
"""Search for audio."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/search",
|
||||
headers=self._headers(),
|
||||
json={"artist": artist, "title": title},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def download(
|
||||
self,
|
||||
search_id: str,
|
||||
result_index: int,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Start download and return download_id.
|
||||
|
||||
If gcs_path is provided, file will be uploaded to GCS.
|
||||
"""
|
||||
async with httpx.AsyncClient() as client:
|
||||
payload = {
|
||||
"search_id": search_id,
|
||||
"result_index": result_index,
|
||||
}
|
||||
if output_filename:
|
||||
payload["output_filename"] = output_filename
|
||||
if gcs_path:
|
||||
payload["upload_to_gcs"] = True
|
||||
payload["gcs_path"] = gcs_path
|
||||
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/download",
|
||||
headers=self._headers(),
|
||||
json=payload,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["download_id"]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Get download status."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/download/{download_id}/status",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def wait_for_download(
|
||||
self,
|
||||
download_id: str,
|
||||
timeout: int = 600,
|
||||
poll_interval: int = 2,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Wait for download to complete.
|
||||
|
||||
Returns final status dict with output_path or gcs_path.
|
||||
Raises FlacfetchServiceError on failure or timeout.
|
||||
"""
|
||||
elapsed = 0
|
||||
while elapsed < timeout:
|
||||
status = await self.get_download_status(download_id)
|
||||
|
||||
if status["status"] == "complete":
|
||||
return status
|
||||
elif status["status"] == "failed":
|
||||
raise FlacfetchServiceError(f"Download failed: {status.get('error')}")
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
raise FlacfetchServiceError(f"Download timed out after {timeout}s")
|
||||
|
||||
|
||||
# Singleton
|
||||
_client: Optional[FlacfetchClient] = None
|
||||
|
||||
|
||||
def get_flacfetch_client() -> Optional[FlacfetchClient]:
|
||||
"""Get flacfetch client if configured."""
|
||||
global _client
|
||||
if _client is None:
|
||||
from backend.config import get_settings
|
||||
settings = get_settings()
|
||||
if settings.flacfetch_api_url and settings.flacfetch_api_key:
|
||||
_client = FlacfetchClient(
|
||||
base_url=settings.flacfetch_api_url,
|
||||
api_key=settings.flacfetch_api_key,
|
||||
)
|
||||
return _client
|
||||
```
|
||||
|
||||
**Modify:** `backend/services/audio_search_service.py` to use remote client when available.
|
||||
|
||||
**Modify:** `backend/config.py` to add new settings:
|
||||
```python
|
||||
flacfetch_api_url: Optional[str] = None # e.g., http://10.0.0.5:8080
|
||||
flacfetch_api_key: Optional[str] = None
|
||||
```
|
||||
|
||||
### Phase 4: Monitoring & Reliability
|
||||
|
||||
**Tasks:**
|
||||
- [ ] Add Cloud Monitoring alert for VM CPU/memory
|
||||
- [ ] Add alert for disk space < 5GB
|
||||
- [ ] Add alert for flacfetch service down (health check fails)
|
||||
- [ ] Configure Cloud Logging export from VM
|
||||
- [ ] Add Grafana dashboard (optional)
|
||||
|
||||
**Automatic Disk Cleanup:**
|
||||
The disk manager runs as a background task in the flacfetch API service:
|
||||
```python
|
||||
# Check every 5 minutes, cleanup if free space < threshold
|
||||
async def background_cleanup_task():
|
||||
while True:
|
||||
await asyncio.sleep(300)
|
||||
free_gb = get_free_space_gb()
|
||||
if free_gb < MIN_FREE_GB:
|
||||
cleanup_oldest_torrents(target_free_gb=MIN_FREE_GB + 5)
|
||||
```
|
||||
|
||||
### Phase 5: Testing & Documentation
|
||||
|
||||
**Testing:**
|
||||
- [ ] Unit tests for flacfetch API endpoints
|
||||
- [ ] Unit tests for disk manager
|
||||
- [ ] Integration test: search → download → GCS upload
|
||||
- [ ] Mock tests for karaoke-gen backend client
|
||||
|
||||
**Documentation:**
|
||||
- [ ] Update `docs/01-reference/ARCHITECTURE.md` with flacfetch service
|
||||
- [ ] Add `flacfetch/README.md` HTTP API section
|
||||
- [ ] Document deployment process
|
||||
- [ ] Document environment variables
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Flacfetch Service (VM)
|
||||
```bash
|
||||
# API Configuration
|
||||
FLACFETCH_API_KEY=<secret> # Required for API auth
|
||||
FLACFETCH_API_PORT=8080 # Default: 8080
|
||||
FLACFETCH_API_HOST=0.0.0.0 # Default: 0.0.0.0
|
||||
|
||||
# Tracker API Keys
|
||||
REDACTED_API_KEY=<key> # For Redacted provider
|
||||
OPS_API_KEY=<key> # For OPS provider
|
||||
|
||||
# GCS Upload (optional)
|
||||
GCS_BUCKET=karaoke-uploads # For uploading completed downloads
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke
|
||||
|
||||
# Transmission
|
||||
TRANSMISSION_HOST=localhost # Default: localhost
|
||||
TRANSMISSION_PORT=9091 # Default: 9091
|
||||
|
||||
# Behavior
|
||||
FLACFETCH_KEEP_SEEDING=true # Keep torrents seeding after download
|
||||
FLACFETCH_MIN_FREE_GB=5 # Minimum free disk space before cleanup
|
||||
FLACFETCH_DOWNLOAD_DIR=/var/lib/transmission-daemon/downloads
|
||||
```
|
||||
|
||||
### Cloud Run Backend
|
||||
```bash
|
||||
# New variables
|
||||
FLACFETCH_API_URL=http://10.128.0.5:8080 # Internal IP of flacfetch VM
|
||||
FLACFETCH_API_KEY=<same-secret>
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **API Authentication**: Static API key in `X-API-Key` header
|
||||
2. **Firewall**: Permissive for future web UI, auth enforced at application level
|
||||
3. **GCS**: Service account with `storage.objectCreator` role only
|
||||
4. **Transmission**: RPC bound to localhost only, no external access
|
||||
5. **Static IP**: Can be whitelisted on tracker accounts
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
If flacfetch service fails:
|
||||
1. Set `FLACFETCH_API_URL=""` in Cloud Run (disables remote service)
|
||||
2. Backend falls back to local flacfetch (YouTube only in Cloud Run)
|
||||
3. Users see warning that lossless downloads may be slower/unavailable
|
||||
|
||||
## Cost Estimate
|
||||
|
||||
| Resource | Monthly Cost |
|
||||
|----------|--------------|
|
||||
| e2-small VM | ~$12 |
|
||||
| Static IP | ~$3 |
|
||||
| 30GB SSD | ~$5 |
|
||||
| Network egress | ~$2-5 |
|
||||
| **Total** | **~$22-25/month** |
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [x] Flacfetch HTTP API accepts search requests and returns results
|
||||
- [x] Downloads complete successfully for Redacted/OPS sources
|
||||
- [x] Downloaded files upload to GCS automatically
|
||||
- [x] Torrents continue seeding after download
|
||||
- [x] Disk cleanup triggers automatically when space is low
|
||||
- [x] Remote CLI shows download progress from flacfetch service
|
||||
- [x] Fallback to local flacfetch works when service unavailable
|
||||
- [x] Static IP can be whitelisted on tracker accounts
|
||||
|
||||
## Timeline
|
||||
|
||||
| Phase | Duration | Dependencies |
|
||||
|-------|----------|--------------|
|
||||
| Phase 1: Flacfetch HTTP API | 1.5 days | None |
|
||||
| Phase 2: Infrastructure | 0.5 day | Phase 1 |
|
||||
| Phase 3: Backend Integration | 0.5 day | Phase 1, 2 |
|
||||
| Phase 4: Monitoring | 0.5 day | Phase 2 |
|
||||
| Phase 5: Testing & Docs | 0.5 day | Phase 1-3 |
|
||||
| **Total** | **3.5 days** | |
|
||||
|
||||
## Implementation Status (2025-12-22)
|
||||
|
||||
### Completed Work
|
||||
|
||||
#### Phase 1: Flacfetch HTTP API ✅
|
||||
- Full HTTP API implemented in `flacfetch/flacfetch/api/`
|
||||
- Endpoints: `/search`, `/download`, `/download/{id}/status`, `/download/{id}/file`, `/torrents`, `/health`
|
||||
- API key authentication via `X-API-Key` header
|
||||
- Disk manager with automatic cleanup when space < 5GB
|
||||
- Download manager with search caching and progress tracking
|
||||
- Dynamic version reporting via `importlib.metadata`
|
||||
- **Current Version:** 0.7.3
|
||||
|
||||
#### Phase 2: Infrastructure ✅
|
||||
- Pulumi infrastructure in `infrastructure/__main__.py`
|
||||
- GCP VM: `flacfetch-service` (e2-small, us-central1-a)
|
||||
- Static IP: `104.198.214.26` (whitelisted with trackers)
|
||||
- Firewall rules: TCP/UDP 51413 (BitTorrent), TCP 8080 (API)
|
||||
- Secrets: `flacfetch-api-key`, `redacted-api-key`, `ops-api-key`, `flacfetch-api-url`
|
||||
- GitHub Actions deployer has `compute.admin` and `compute.osLogin` roles
|
||||
|
||||
#### Phase 3: Backend Integration ✅
|
||||
- `FlacfetchClient` in `backend/services/flacfetch_client.py`
|
||||
- `AudioSearchService` updated to use remote flacfetch when configured
|
||||
- Cloud Run configured with `FLACFETCH_API_URL` and `FLACFETCH_API_KEY` from Secret Manager
|
||||
- `--loop asyncio` added to uvicorn CMD (required for `nest_asyncio` compatibility)
|
||||
- `nest_asyncio` applied only when async loop is running (not at import time)
|
||||
|
||||
#### Phase 4: Monitoring ✅
|
||||
- Health endpoint reports Transmission status, disk usage, provider availability
|
||||
- Version verification in deployment workflows
|
||||
- Automatic deployment on release via GitHub Actions
|
||||
|
||||
#### Phase 5: CLI & Testing ✅
|
||||
- `flacfetch-remote` CLI added for testing remote API
|
||||
- Downloads file locally with descriptive filenames (e.g., `Artist - Title (Album, 2022, CD, 16bit, Redacted).flac`)
|
||||
- 70%+ unit test coverage requirement enforced
|
||||
- Integration tests for API routes
|
||||
|
||||
### Deployment Automation
|
||||
|
||||
#### Flacfetch VM (automatic)
|
||||
- `.github/workflows/release-on-version-bump.yml` in flacfetch repo
|
||||
- Triggers on version bump in `pyproject.toml`
|
||||
- Runs tests → Creates GitHub release → SSHs to VM → Pulls latest → Restarts service
|
||||
- Verifies deployed version matches release version
|
||||
|
||||
#### Karaoke-gen Backend (version-gated)
|
||||
- `.github/workflows/ci.yml` deploy-backend job
|
||||
- Only deploys if version tag doesn't exist (requires version bump)
|
||||
- Verifies deployed version via `/api/health/detailed` endpoint
|
||||
- Retries version check with backoff for Cloud Run routing delays
|
||||
|
||||
### Key Files Modified
|
||||
|
||||
**Flacfetch repo (`./flacfetch/`):**
|
||||
- `flacfetch/api/` - New HTTP API module
|
||||
- `flacfetch/interface/cli_remote.py` - New remote CLI
|
||||
- `flacfetch/core/manager.py` - Improved sorting (artist match, release type, seeders)
|
||||
- `flacfetch/downloaders/torrent.py` - Keep-seeding mode, file permissions fix
|
||||
- `.github/workflows/release-on-version-bump.yml` - Auto-deploy to VM
|
||||
|
||||
**Karaoke-gen repo:**
|
||||
- `backend/services/flacfetch_client.py` - Remote API client
|
||||
- `backend/services/audio_search_service.py` - Remote integration with nest_asyncio
|
||||
- `backend/Dockerfile` - `--loop asyncio` for uvicorn
|
||||
- `infrastructure/__main__.py` - VM, firewall, secrets, WIF bindings
|
||||
- `cloudbuild.yaml` - FLACFETCH_API_URL/KEY from Secret Manager
|
||||
- `.github/workflows/ci.yml` - Version-gated deployments
|
||||
|
||||
### Testing the Integration
|
||||
|
||||
```bash
|
||||
# Test flacfetch-remote CLI (requires env vars)
|
||||
export FLACFETCH_API_URL=http://104.198.214.26:8080
|
||||
export FLACFETCH_API_KEY=<from-secret-manager>
|
||||
flacfetch-remote "Avril Lavigne" "Unwanted"
|
||||
|
||||
# Test backend health
|
||||
curl -s https://api.nomadkaraoke.com/api/health/detailed | jq '.dependencies.flacfetch_remote'
|
||||
|
||||
# Test backend search
|
||||
curl -s -X POST https://api.nomadkaraoke.com/api/audio-search/search \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"artist": "Avril Lavigne", "title": "Unwanted"}' | jq '.results | length'
|
||||
```
|
||||
|
||||
### Known Issues / Future Work
|
||||
|
||||
1. **Transmission removal from Cloud Run**: Transmission daemon and startup script removed from Docker images as torrent downloads are now handled by dedicated VM.
|
||||
|
||||
2. **URL field in search results**: Made optional in `AudioSearchResultResponse` since remote results don't have direct download URLs.
|
||||
|
||||
3. **Sorting improvements**: Results now prioritize:
|
||||
- Lossless first
|
||||
- Artist name match (exact match wins)
|
||||
- Release type (Album > Single > EP > Compilation > Anthology > Remix)
|
||||
- Seeders (more = higher)
|
||||
- Quality (24bit > 16bit, FLAC > MP3)
|
||||
|
||||
4. **Future enhancements**:
|
||||
- Cloud Monitoring alerts for disk/CPU/memory
|
||||
- Grafana dashboard for torrent stats
|
||||
- Web UI for manual torrent management
|
||||
|
||||
## References
|
||||
|
||||
- [Transmission RPC Spec](https://github.com/transmission/transmission/blob/main/docs/rpc-spec.md)
|
||||
- [transmission-rpc Python library](https://github.com/Trim21/transmission-rpc)
|
||||
- [FastAPI Documentation](https://fastapi.tiangolo.com/)
|
||||
- [flacfetch source code](./flacfetch/)
|
||||
Submodule
+1
Submodule flacfetch added at 35ae8994b8.
@@ -94,6 +94,14 @@ echo -n "your-genius-key" | \
|
||||
# Add Audio Separator API URL
|
||||
echo -n "https://your-modal-url" | \
|
||||
gcloud secrets versions add audio-separator-api-url --data-file=-
|
||||
|
||||
# Add Redacted API key (for flacfetch audio search - private tracker)
|
||||
echo -n "your-redacted-api-key" | \
|
||||
gcloud secrets versions add redacted-api-key --data-file=-
|
||||
|
||||
# Add OPS API key (for flacfetch audio search - private tracker)
|
||||
echo -n "your-ops-api-key" | \
|
||||
gcloud secrets versions add ops-api-key --data-file=-
|
||||
```
|
||||
|
||||
Or use Pulumi config secrets:
|
||||
|
||||
@@ -98,6 +98,14 @@ secrets_write_iam = gcp.projects.IAMMember(
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Cloud Trace write permissions (for OpenTelemetry tracing)
|
||||
cloud_trace_iam = gcp.projects.IAMMember(
|
||||
"karaoke-backend-cloudtrace-agent",
|
||||
project=project_id,
|
||||
role="roles/cloudtrace.agent",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant Cloud Build service accounts access to Artifact Registry
|
||||
# Cloud Build uses multiple service accounts depending on the context
|
||||
cloudbuild_service_accounts = [
|
||||
@@ -123,6 +131,16 @@ cloudbuild_logging_iam = gcp.projects.IAMMember(
|
||||
member=f"serviceAccount:{project.number}-compute@developer.gserviceaccount.com",
|
||||
)
|
||||
|
||||
# Grant Cloud Build compute service account permission to deploy as karaoke-backend SA
|
||||
# This is required for Cloud Build to deploy Cloud Run services that use the backend service account
|
||||
# Without this, gcloud run deploy fails with: PERMISSION_DENIED: Permission 'iam.serviceaccounts.actAs' denied
|
||||
cloudbuild_deploy_as_backend = gcp.serviceaccount.IAMMember(
|
||||
"cloudbuild-deploy-as-backend",
|
||||
service_account_id=service_account.name,
|
||||
role="roles/iam.serviceAccountUser",
|
||||
member=f"serviceAccount:{project.number}-compute@developer.gserviceaccount.com",
|
||||
)
|
||||
|
||||
# ==================== GitHub Actions Deployer Service Account ====================
|
||||
# This service account is used by GitHub Actions CI/CD via Workload Identity Federation
|
||||
|
||||
@@ -348,6 +366,15 @@ cloud_tasks_enqueuer = gcp.projects.IAMMember(
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant backend service account permission to "act as" itself for Cloud Tasks OIDC authentication
|
||||
# This is required when creating Cloud Tasks with oidc_token that specifies this service account
|
||||
backend_sa_act_as_self = gcp.serviceaccount.IAMBinding(
|
||||
"karaoke-backend-act-as-self",
|
||||
service_account_id=service_account.name,
|
||||
role="roles/iam.serviceAccountUser",
|
||||
members=[service_account.email.apply(lambda email: f"serviceAccount:{email}")],
|
||||
)
|
||||
|
||||
# ==================== Cloud Run Job for Video Encoding (Phase 2 Scalability) ====================
|
||||
# Cloud Run Jobs provide up to 24-hour execution time for long-running video encoding.
|
||||
# This is used when video encoding might exceed the 30-minute Cloud Tasks timeout.
|
||||
@@ -428,6 +455,24 @@ audio_separator_secret = secretmanager.Secret(
|
||||
),
|
||||
)
|
||||
|
||||
# Redacted API Key Secret (for flacfetch audio search)
|
||||
redacted_api_key_secret = secretmanager.Secret(
|
||||
"redacted-api-key",
|
||||
secret_id="redacted-api-key",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# OPS API Key Secret (for flacfetch audio search)
|
||||
ops_api_key_secret = secretmanager.Secret(
|
||||
"ops-api-key",
|
||||
secret_id="ops-api-key",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Create Cloud Run Domain Mapping
|
||||
# Maps api.nomadkaraoke.com to the karaoke-backend service
|
||||
domain_mapping = cloudrun.DomainMapping(
|
||||
@@ -479,3 +524,396 @@ domain_mapping.statuses.apply(
|
||||
})
|
||||
)
|
||||
|
||||
# ==================== Flacfetch Service (Torrent Downloads) ====================
|
||||
# A dedicated VM running flacfetch with Transmission daemon for BitTorrent downloads.
|
||||
# This is required because Cloud Run doesn't support inbound peer connections for torrents.
|
||||
# The VM has a static IP for tracker whitelist and runs indefinitely for seeding.
|
||||
|
||||
# Static IP for flacfetch (for tracker account whitelist)
|
||||
flacfetch_ip = gcp.compute.Address(
|
||||
"flacfetch-ip",
|
||||
name="flacfetch-static-ip",
|
||||
region="us-central1",
|
||||
address_type="EXTERNAL",
|
||||
description="Static IP for flacfetch service (torrent downloads)",
|
||||
)
|
||||
|
||||
# Service account for flacfetch VM
|
||||
flacfetch_sa = serviceaccount.Account(
|
||||
"flacfetch-sa",
|
||||
account_id="flacfetch-service",
|
||||
display_name="Flacfetch Service Account",
|
||||
description="Service account for flacfetch torrent/audio download VM",
|
||||
)
|
||||
|
||||
# Grant flacfetch service account GCS write permissions
|
||||
flacfetch_storage_iam = gcp.storage.BucketIAMMember(
|
||||
"flacfetch-storage-writer",
|
||||
bucket=bucket.name,
|
||||
role="roles/storage.objectCreator",
|
||||
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant flacfetch service account GCS read permissions (for uploads folder)
|
||||
flacfetch_storage_reader_iam = gcp.storage.BucketIAMMember(
|
||||
"flacfetch-storage-reader",
|
||||
bucket=bucket.name,
|
||||
role="roles/storage.objectViewer",
|
||||
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant flacfetch service account Secret Manager access
|
||||
flacfetch_secrets_iam = gcp.projects.IAMMember(
|
||||
"flacfetch-secrets-access",
|
||||
project=project_id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Flacfetch API key secret (for authenticating API requests)
|
||||
flacfetch_api_key_secret = secretmanager.Secret(
|
||||
"flacfetch-api-key",
|
||||
secret_id="flacfetch-api-key",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Flacfetch API URL secret (the static IP address of the flacfetch VM)
|
||||
# This is stored as a secret to avoid committing infrastructure IPs to the repo
|
||||
flacfetch_api_url_secret = secretmanager.Secret(
|
||||
"flacfetch-api-url",
|
||||
secret_id="flacfetch-api-url",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Startup script for the flacfetch VM
|
||||
# This installs and configures Transmission daemon and the flacfetch service
|
||||
FLACFETCH_STARTUP_SCRIPT = """#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Log to a file for debugging
|
||||
exec > >(tee /var/log/flacfetch-startup.log) 2>&1
|
||||
echo "Starting flacfetch setup at $(date)"
|
||||
|
||||
# Install dependencies
|
||||
apt-get update
|
||||
apt-get install -y python3-pip python3-venv transmission-daemon ffmpeg git curl
|
||||
|
||||
# Configure Transmission
|
||||
echo "Configuring Transmission daemon..."
|
||||
systemctl stop transmission-daemon || true
|
||||
|
||||
# Create transmission directories
|
||||
mkdir -p /var/lib/transmission-daemon/downloads
|
||||
mkdir -p /var/lib/transmission-daemon/.config/transmission-daemon
|
||||
chown -R debian-transmission:debian-transmission /var/lib/transmission-daemon
|
||||
|
||||
# Configure transmission settings
|
||||
cat > /var/lib/transmission-daemon/.config/transmission-daemon/settings.json << 'SETTINGS'
|
||||
{
|
||||
"download-dir": "/var/lib/transmission-daemon/downloads",
|
||||
"incomplete-dir": "/var/lib/transmission-daemon/.incomplete",
|
||||
"incomplete-dir-enabled": true,
|
||||
"rpc-authentication-required": false,
|
||||
"rpc-bind-address": "127.0.0.1",
|
||||
"rpc-enabled": true,
|
||||
"rpc-port": 9091,
|
||||
"rpc-whitelist-enabled": false,
|
||||
"peer-port": 51413,
|
||||
"port-forwarding-enabled": false,
|
||||
"speed-limit-down": 0,
|
||||
"speed-limit-down-enabled": false,
|
||||
"speed-limit-up": 0,
|
||||
"speed-limit-up-enabled": false,
|
||||
"ratio-limit-enabled": false,
|
||||
"umask": 2,
|
||||
"encryption": 1
|
||||
}
|
||||
SETTINGS
|
||||
|
||||
chown debian-transmission:debian-transmission /var/lib/transmission-daemon/.config/transmission-daemon/settings.json
|
||||
systemctl start transmission-daemon
|
||||
echo "Transmission daemon started"
|
||||
|
||||
# Install flacfetch
|
||||
echo "Installing flacfetch..."
|
||||
cd /opt
|
||||
|
||||
# Clone flacfetch (we'll use the repo from GitHub)
|
||||
if [ ! -d "flacfetch" ]; then
|
||||
git clone https://github.com/nomadkaraoke/flacfetch.git
|
||||
fi
|
||||
cd flacfetch
|
||||
|
||||
# Create and activate virtual environment
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
|
||||
# Install flacfetch with API dependencies
|
||||
pip install -e ".[api]"
|
||||
|
||||
# Get secrets from Secret Manager
|
||||
echo "Fetching secrets from Secret Manager..."
|
||||
FLACFETCH_API_KEY=$(gcloud secrets versions access latest --secret=flacfetch-api-key 2>/dev/null || echo "")
|
||||
REDACTED_API_KEY=$(gcloud secrets versions access latest --secret=redacted-api-key 2>/dev/null || echo "")
|
||||
OPS_API_KEY=$(gcloud secrets versions access latest --secret=ops-api-key 2>/dev/null || echo "")
|
||||
|
||||
# Get bucket name from project metadata
|
||||
GCS_BUCKET=$(gcloud compute project-info describe --format='value(commonInstanceMetadata.items.gcs-bucket)' 2>/dev/null || echo "")
|
||||
if [ -z "$GCS_BUCKET" ]; then
|
||||
# Fallback: construct from project ID
|
||||
PROJECT_ID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
|
||||
GCS_BUCKET="karaoke-gen-storage-${PROJECT_ID}"
|
||||
fi
|
||||
|
||||
echo "Using GCS bucket: $GCS_BUCKET"
|
||||
|
||||
# Create systemd service for flacfetch
|
||||
echo "Creating systemd service..."
|
||||
cat > /etc/systemd/system/flacfetch.service << SYSTEMD
|
||||
[Unit]
|
||||
Description=Flacfetch HTTP API Service
|
||||
After=network.target transmission-daemon.service
|
||||
Requires=transmission-daemon.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/flacfetch
|
||||
Environment="FLACFETCH_API_KEY=${FLACFETCH_API_KEY}"
|
||||
Environment="REDACTED_API_KEY=${REDACTED_API_KEY}"
|
||||
Environment="OPS_API_KEY=${OPS_API_KEY}"
|
||||
Environment="GCS_BUCKET=${GCS_BUCKET}"
|
||||
Environment="FLACFETCH_KEEP_SEEDING=true"
|
||||
Environment="FLACFETCH_MIN_FREE_GB=5"
|
||||
Environment="FLACFETCH_DOWNLOAD_DIR=/var/lib/transmission-daemon/downloads"
|
||||
Environment="TRANSMISSION_HOST=localhost"
|
||||
Environment="TRANSMISSION_PORT=9091"
|
||||
ExecStart=/opt/flacfetch/venv/bin/flacfetch serve --host 0.0.0.0 --port 8080
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
SYSTEMD
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable flacfetch
|
||||
systemctl start flacfetch
|
||||
|
||||
echo "Flacfetch service started at $(date)"
|
||||
echo "Setup complete!"
|
||||
"""
|
||||
|
||||
# Flacfetch VM instance
|
||||
flacfetch_vm = gcp.compute.Instance(
|
||||
"flacfetch-service",
|
||||
name="flacfetch-service",
|
||||
machine_type="e2-small", # 0.5 vCPU, 2GB RAM - sufficient for I/O-bound workload
|
||||
zone="us-central1-a",
|
||||
boot_disk=gcp.compute.InstanceBootDiskArgs(
|
||||
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=30, # GB - for torrent storage + headroom
|
||||
type="pd-ssd", # SSD for faster I/O
|
||||
),
|
||||
),
|
||||
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs(
|
||||
nat_ip=flacfetch_ip.address, # Static IP
|
||||
)],
|
||||
)],
|
||||
service_account=gcp.compute.InstanceServiceAccountArgs(
|
||||
email=flacfetch_sa.email,
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
metadata_startup_script=FLACFETCH_STARTUP_SCRIPT,
|
||||
metadata={
|
||||
"gcs-bucket": bucket.name, # Pass bucket name to startup script
|
||||
},
|
||||
tags=["flacfetch-service"],
|
||||
allow_stopping_for_update=True,
|
||||
)
|
||||
|
||||
# Firewall rule for flacfetch
|
||||
# - Port 51413 TCP/UDP: BitTorrent peer connections
|
||||
# - Port 8080 TCP: Flacfetch HTTP API (auth required at app level)
|
||||
flacfetch_firewall = gcp.compute.Firewall(
|
||||
"flacfetch-firewall",
|
||||
name="flacfetch-firewall",
|
||||
network="default",
|
||||
allows=[
|
||||
# BitTorrent peer connections
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["51413"]),
|
||||
gcp.compute.FirewallAllowArgs(protocol="udp", ports=["51413"]),
|
||||
# Flacfetch HTTP API (auth required at app level)
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["8080"]),
|
||||
],
|
||||
source_ranges=["0.0.0.0/0"], # Permissive, auth enforced at app level
|
||||
target_tags=["flacfetch-service"],
|
||||
description="Firewall rules for flacfetch torrent/API service",
|
||||
)
|
||||
|
||||
# Export flacfetch values
|
||||
pulumi.export("flacfetch_static_ip", flacfetch_ip.address)
|
||||
pulumi.export("flacfetch_service_url", flacfetch_ip.address.apply(
|
||||
lambda ip: f"http://{ip}:8080"
|
||||
))
|
||||
pulumi.export("flacfetch_service_account", flacfetch_sa.email)
|
||||
|
||||
# ==================== Cloud Monitoring Alerting ====================
|
||||
# Alert policies for proactive monitoring and incident response
|
||||
|
||||
# Discord notification channel (webhook-based)
|
||||
# Note: You'll need to manually create this in Cloud Console and reference it,
|
||||
# or use the Discord webhook URL via Cloud Functions
|
||||
|
||||
# Alert: High Error Rate (>10% of requests returning errors)
|
||||
error_rate_alert = gcp.monitoring.AlertPolicy(
|
||||
"high-error-rate-alert",
|
||||
display_name="Karaoke Backend - High Error Rate",
|
||||
combiner="OR",
|
||||
conditions=[
|
||||
gcp.monitoring.AlertPolicyConditionArgs(
|
||||
display_name="Error rate > 10%",
|
||||
condition_threshold=gcp.monitoring.AlertPolicyConditionConditionThresholdArgs(
|
||||
filter='resource.type="cloud_run_revision" AND resource.labels.service_name="karaoke-backend" AND metric.type="run.googleapis.com/request_count" AND metric.labels.response_code_class!="2xx"',
|
||||
comparison="COMPARISON_GT",
|
||||
threshold_value=0.1,
|
||||
duration="300s", # Must be above threshold for 5 minutes
|
||||
aggregations=[
|
||||
gcp.monitoring.AlertPolicyConditionConditionThresholdAggregationArgs(
|
||||
alignment_period="60s",
|
||||
per_series_aligner="ALIGN_RATE",
|
||||
cross_series_reducer="REDUCE_SUM",
|
||||
),
|
||||
],
|
||||
trigger=gcp.monitoring.AlertPolicyConditionConditionThresholdTriggerArgs(
|
||||
count=1,
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
alert_strategy=gcp.monitoring.AlertPolicyAlertStrategyArgs(
|
||||
auto_close="3600s", # Auto-close after 1 hour if resolved
|
||||
),
|
||||
documentation=gcp.monitoring.AlertPolicyDocumentationArgs(
|
||||
content="High error rate detected on karaoke-backend. Check Cloud Run logs for details.\n\nDashboard: https://console.cloud.google.com/run/detail/us-central1/karaoke-backend/logs?project=nomadkaraoke",
|
||||
mime_type="text/markdown",
|
||||
),
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
# Alert: Cloud Tasks Queue Backlog (tasks piling up)
|
||||
queue_backlog_alert = gcp.monitoring.AlertPolicy(
|
||||
"queue-backlog-alert",
|
||||
display_name="Karaoke Backend - Queue Backlog",
|
||||
combiner="OR",
|
||||
conditions=[
|
||||
gcp.monitoring.AlertPolicyConditionArgs(
|
||||
display_name="Queue depth > 50 tasks",
|
||||
condition_threshold=gcp.monitoring.AlertPolicyConditionConditionThresholdArgs(
|
||||
filter='resource.type="cloud_tasks_queue" AND metric.type="cloudtasks.googleapis.com/queue/depth"',
|
||||
comparison="COMPARISON_GT",
|
||||
threshold_value=50,
|
||||
duration="600s", # Must be above threshold for 10 minutes
|
||||
aggregations=[
|
||||
gcp.monitoring.AlertPolicyConditionConditionThresholdAggregationArgs(
|
||||
alignment_period="60s",
|
||||
per_series_aligner="ALIGN_MEAN",
|
||||
cross_series_reducer="REDUCE_MAX",
|
||||
),
|
||||
],
|
||||
trigger=gcp.monitoring.AlertPolicyConditionConditionThresholdTriggerArgs(
|
||||
count=1,
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
alert_strategy=gcp.monitoring.AlertPolicyAlertStrategyArgs(
|
||||
auto_close="1800s", # Auto-close after 30 min if resolved
|
||||
),
|
||||
documentation=gcp.monitoring.AlertPolicyDocumentationArgs(
|
||||
content="Cloud Tasks queue backlog detected. Tasks are piling up faster than workers can process.\n\nPossible causes:\n- Worker errors (check logs)\n- External API issues (Modal, AudioShake)\n- Cloud Run scaling limits\n\nDashboard: https://console.cloud.google.com/cloudtasks?project=nomadkaraoke",
|
||||
mime_type="text/markdown",
|
||||
),
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
# Alert: High Memory Utilization (workers may be crashing)
|
||||
memory_alert = gcp.monitoring.AlertPolicy(
|
||||
"high-memory-alert",
|
||||
display_name="Karaoke Backend - High Memory Usage",
|
||||
combiner="OR",
|
||||
conditions=[
|
||||
gcp.monitoring.AlertPolicyConditionArgs(
|
||||
display_name="Memory > 85%",
|
||||
condition_threshold=gcp.monitoring.AlertPolicyConditionConditionThresholdArgs(
|
||||
filter='resource.type="cloud_run_revision" AND resource.labels.service_name="karaoke-backend" AND metric.type="run.googleapis.com/container/memory/utilizations"',
|
||||
comparison="COMPARISON_GT",
|
||||
threshold_value=0.85,
|
||||
duration="300s",
|
||||
aggregations=[
|
||||
gcp.monitoring.AlertPolicyConditionConditionThresholdAggregationArgs(
|
||||
alignment_period="60s",
|
||||
per_series_aligner="ALIGN_PERCENTILE_95",
|
||||
cross_series_reducer="REDUCE_MAX",
|
||||
),
|
||||
],
|
||||
trigger=gcp.monitoring.AlertPolicyConditionConditionThresholdTriggerArgs(
|
||||
count=1,
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
alert_strategy=gcp.monitoring.AlertPolicyAlertStrategyArgs(
|
||||
auto_close="1800s",
|
||||
),
|
||||
documentation=gcp.monitoring.AlertPolicyDocumentationArgs(
|
||||
content="High memory utilization on karaoke-backend. Workers may be running out of memory.\n\nActions:\n- Check for memory leaks in worker code\n- Consider increasing Cloud Run memory limits\n- Review temp file cleanup",
|
||||
mime_type="text/markdown",
|
||||
),
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
# Alert: Cloud Run Service Unavailable
|
||||
service_unavailable_alert = gcp.monitoring.AlertPolicy(
|
||||
"service-unavailable-alert",
|
||||
display_name="Karaoke Backend - Service Unavailable",
|
||||
combiner="OR",
|
||||
conditions=[
|
||||
gcp.monitoring.AlertPolicyConditionArgs(
|
||||
display_name="No healthy instances",
|
||||
condition_absent=gcp.monitoring.AlertPolicyConditionConditionAbsentArgs(
|
||||
filter='resource.type="cloud_run_revision" AND resource.labels.service_name="karaoke-backend" AND metric.type="run.googleapis.com/container/instance_count"',
|
||||
duration="300s", # Alert if no instances for 5 minutes
|
||||
aggregations=[
|
||||
gcp.monitoring.AlertPolicyConditionConditionAbsentAggregationArgs(
|
||||
alignment_period="60s",
|
||||
per_series_aligner="ALIGN_MEAN",
|
||||
cross_series_reducer="REDUCE_SUM",
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
alert_strategy=gcp.monitoring.AlertPolicyAlertStrategyArgs(
|
||||
auto_close="1800s", # Minimum 30 minutes required by GCP
|
||||
),
|
||||
documentation=gcp.monitoring.AlertPolicyDocumentationArgs(
|
||||
content="Karaoke backend service appears to be down - no healthy Cloud Run instances detected.\n\nImmediate actions:\n1. Check Cloud Run logs for crash reasons\n2. Verify recent deployments\n3. Check external dependencies (Modal, AudioShake)\n\nService URL: https://api.nomadkaraoke.com/api/health",
|
||||
mime_type="text/markdown",
|
||||
),
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
# Export alert policy IDs
|
||||
pulumi.export("error_rate_alert_id", error_rate_alert.name)
|
||||
pulumi.export("queue_backlog_alert_id", queue_backlog_alert.name)
|
||||
pulumi.export("memory_alert_id", memory_alert.name)
|
||||
pulumi.export("service_unavailable_alert_id", service_unavailable_alert.name)
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
# Monitoring Infrastructure
|
||||
|
||||
This directory contains Cloud Monitoring dashboard and alerting configurations for the karaoke-gen backend.
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Quick Start: Debug a Failed Job
|
||||
|
||||
**Bookmark this URL:**
|
||||
```
|
||||
https://console.cloud.google.com/logs/query;query=resource.type%3D%22cloud_run_revision%22%0Aresource.labels.service_name%3D%22karaoke-backend%22;duration=PT6H?project=nomadkaraoke
|
||||
```
|
||||
|
||||
### How to investigate a job:
|
||||
|
||||
1. **Open the bookmarked URL** (Logs Explorer)
|
||||
|
||||
2. **Add your job ID to the query.** In the query box, add a third line with your job ID:
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
"[job:YOUR_JOB_ID"
|
||||
```
|
||||
(Replace `YOUR_JOB_ID` with the actual ID, e.g. `"[job:8f03f0ac"`)
|
||||
|
||||
3. **Click "Run query"** or press `Shift+Enter`
|
||||
|
||||
4. **Enable "Stream logs"** (toggle at top right) for real-time updates
|
||||
|
||||
### What you'll see:
|
||||
- `WORKER_START` - When each worker (audio/lyrics/screens/video) begins
|
||||
- `WORKER_END status=success/error` - When it finishes with duration
|
||||
- All log messages from that job's processing
|
||||
- Any error tracebacks
|
||||
|
||||
### Example queries:
|
||||
|
||||
**All logs for job `8f03f0ac`:**
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
"[job:8f03f0ac"
|
||||
```
|
||||
|
||||
**Only errors for a job:**
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
"[job:8f03f0ac"
|
||||
severity>=ERROR
|
||||
```
|
||||
|
||||
**Only worker start/end events:**
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
"[job:8f03f0ac"
|
||||
("WORKER_START" OR "WORKER_END")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dashboard
|
||||
|
||||
The `dashboard.json` file defines a Cloud Monitoring dashboard with the following panels:
|
||||
|
||||
### Overview Metrics
|
||||
- **Jobs Created (Last Hour)**: Count of jobs created
|
||||
- **Cloud Run Request Count**: Request rate to the backend
|
||||
- **Cloud Run Error Rate**: Percentage of non-2xx responses
|
||||
|
||||
### Performance Metrics
|
||||
- **Request Latency (p95)**: 95th percentile response time
|
||||
- **Cloud Run Instance Count**: Active container instances
|
||||
- **CPU Utilization**: CPU usage across instances
|
||||
- **Memory Utilization**: Memory usage across instances
|
||||
|
||||
### Cloud Tasks
|
||||
- **Queue Depth**: Number of tasks waiting in each queue
|
||||
- **Execution Rate**: Task processing rate per queue
|
||||
|
||||
### Logs
|
||||
- **Worker Logs**: Searchable by `job_id`
|
||||
- **Error Logs**: All ERROR-level logs from the backend
|
||||
|
||||
## Importing the Dashboard
|
||||
|
||||
To import the dashboard into Cloud Monitoring:
|
||||
|
||||
```bash
|
||||
# Using gcloud CLI
|
||||
gcloud monitoring dashboards create --config-from-file=infrastructure/monitoring/dashboard.json --project=nomadkaraoke
|
||||
```
|
||||
|
||||
Or manually:
|
||||
1. Go to Cloud Console → Monitoring → Dashboards
|
||||
2. Click "Create Dashboard"
|
||||
3. Click "JSON Editor" and paste contents of `dashboard.json`
|
||||
|
||||
## Alert Policies
|
||||
|
||||
Alert policies are defined in `infrastructure/__main__.py` as Pulumi resources:
|
||||
|
||||
### High Error Rate
|
||||
- **Condition**: >10% of requests return errors for 5+ minutes
|
||||
- **Severity**: Warning
|
||||
- **Auto-close**: 1 hour after resolution
|
||||
|
||||
### Queue Backlog
|
||||
- **Condition**: >50 tasks in any queue for 10+ minutes
|
||||
- **Severity**: Warning
|
||||
- **Auto-close**: 30 minutes after resolution
|
||||
|
||||
### High Memory Usage
|
||||
- **Condition**: >85% memory utilization for 5+ minutes
|
||||
- **Severity**: Warning
|
||||
- **Auto-close**: 30 minutes after resolution
|
||||
|
||||
### Service Unavailable
|
||||
- **Condition**: No healthy Cloud Run instances for 5+ minutes
|
||||
- **Severity**: Critical
|
||||
- **Auto-close**: 10 minutes after resolution
|
||||
|
||||
## Adding Notification Channels
|
||||
|
||||
To receive alerts, you need to create notification channels:
|
||||
|
||||
### Discord Webhook
|
||||
|
||||
1. Go to Cloud Console → Monitoring → Alerting → Notification channels
|
||||
2. Click "Add New" → Webhook
|
||||
3. Enter your Discord webhook URL
|
||||
4. Save and note the channel ID
|
||||
|
||||
Then update the alert policies in `__main__.py`:
|
||||
|
||||
```python
|
||||
error_rate_alert = gcp.monitoring.AlertPolicy(
|
||||
# ...existing config...
|
||||
notification_channels=[discord_webhook_channel.name],
|
||||
)
|
||||
```
|
||||
|
||||
### Email
|
||||
|
||||
1. Go to Cloud Console → Monitoring → Alerting → Notification channels
|
||||
2. Click "Add New" → Email
|
||||
3. Enter email addresses to notify
|
||||
|
||||
## Log-Based Metrics
|
||||
|
||||
The structured logging in `backend/services/structured_logging.py` outputs metrics as JSON log entries. You can create log-based metrics to track:
|
||||
|
||||
1. Go to Cloud Console → Logging → Log-based metrics
|
||||
2. Create new metric with filter:
|
||||
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
jsonPayload.metric_name="jobs_total"
|
||||
```
|
||||
|
||||
3. Set extraction field to `jsonPayload.metric_value`
|
||||
|
||||
Repeat for other metrics:
|
||||
- `worker_invocations_total`
|
||||
- `job_stage_duration_seconds`
|
||||
- `external_api_calls_total`
|
||||
- `external_api_duration_seconds`
|
||||
- `gcs_operations_total`
|
||||
|
||||
## Useful Log Queries
|
||||
|
||||
### Find all logs for a specific job:
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
jsonPayload.job_id="YOUR_JOB_ID"
|
||||
```
|
||||
|
||||
### Find worker errors:
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
textPayload:"WORKER_END" AND textPayload:"status=error"
|
||||
```
|
||||
|
||||
### Find slow workers (>5 min):
|
||||
```
|
||||
resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="karaoke-backend"
|
||||
textPayload:"WORKER_END" AND textPayload:"duration="
|
||||
```
|
||||
|
||||
Then add duration filter in the UI.
|
||||
|
||||
## Cloud Trace Integration
|
||||
|
||||
Structured logs include trace correlation fields:
|
||||
- `logging.googleapis.com/trace`: Links to Cloud Trace
|
||||
- `logging.googleapis.com/spanId`: Current span ID
|
||||
|
||||
To view correlated traces:
|
||||
1. Find a log entry in Cloud Logging
|
||||
2. Click the "View in Trace" link in the log entry
|
||||
3. See the full distributed trace across services
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
{
|
||||
"displayName": "Karaoke Backend Overview",
|
||||
"mosaicLayout": {
|
||||
"columns": 12,
|
||||
"tiles": [
|
||||
{
|
||||
"xPos": 0,
|
||||
"yPos": 0,
|
||||
"width": 4,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Jobs Created (Last Hour)",
|
||||
"scorecard": {
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/log_entry_count\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "3600s",
|
||||
"perSeriesAligner": "ALIGN_SUM",
|
||||
"crossSeriesReducer": "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
},
|
||||
"sparkChartView": {
|
||||
"sparkChartType": "SPARK_LINE"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 4,
|
||||
"yPos": 0,
|
||||
"width": 4,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Cloud Run Request Count",
|
||||
"scorecard": {
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/request_count\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "3600s",
|
||||
"perSeriesAligner": "ALIGN_RATE",
|
||||
"crossSeriesReducer": "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
},
|
||||
"sparkChartView": {
|
||||
"sparkChartType": "SPARK_LINE"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 8,
|
||||
"yPos": 0,
|
||||
"width": 4,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Cloud Run Error Rate",
|
||||
"scorecard": {
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/request_count\" AND metric.labels.response_code_class!=\"2xx\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "3600s",
|
||||
"perSeriesAligner": "ALIGN_RATE",
|
||||
"crossSeriesReducer": "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
},
|
||||
"sparkChartView": {
|
||||
"sparkChartType": "SPARK_LINE"
|
||||
},
|
||||
"thresholds": [
|
||||
{
|
||||
"value": 0.05,
|
||||
"color": "YELLOW",
|
||||
"direction": "ABOVE"
|
||||
},
|
||||
{
|
||||
"value": 0.1,
|
||||
"color": "RED",
|
||||
"direction": "ABOVE"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 0,
|
||||
"yPos": 4,
|
||||
"width": 6,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Cloud Run Request Latency (p95)",
|
||||
"xyChart": {
|
||||
"dataSets": [
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/request_latencies\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_PERCENTILE_95",
|
||||
"crossSeriesReducer": "REDUCE_MEAN"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "LINE",
|
||||
"legendTemplate": "p95 latency"
|
||||
}
|
||||
],
|
||||
"yAxis": {
|
||||
"label": "Latency (ms)",
|
||||
"scale": "LINEAR"
|
||||
},
|
||||
"chartOptions": {
|
||||
"mode": "COLOR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 6,
|
||||
"yPos": 4,
|
||||
"width": 6,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Cloud Run Instance Count",
|
||||
"xyChart": {
|
||||
"dataSets": [
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/container/instance_count\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_MEAN",
|
||||
"crossSeriesReducer": "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "LINE",
|
||||
"legendTemplate": "instances"
|
||||
}
|
||||
],
|
||||
"yAxis": {
|
||||
"label": "Instances",
|
||||
"scale": "LINEAR"
|
||||
},
|
||||
"chartOptions": {
|
||||
"mode": "COLOR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 0,
|
||||
"yPos": 8,
|
||||
"width": 6,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Cloud Tasks Queue Depth",
|
||||
"xyChart": {
|
||||
"dataSets": [
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_tasks_queue\" AND metric.type=\"cloudtasks.googleapis.com/queue/depth\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_MEAN",
|
||||
"groupByFields": ["resource.labels.queue_id"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "STACKED_AREA",
|
||||
"legendTemplate": "${resource.labels.queue_id}"
|
||||
}
|
||||
],
|
||||
"yAxis": {
|
||||
"label": "Tasks in Queue",
|
||||
"scale": "LINEAR"
|
||||
},
|
||||
"chartOptions": {
|
||||
"mode": "COLOR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 6,
|
||||
"yPos": 8,
|
||||
"width": 6,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Cloud Tasks Execution Rate",
|
||||
"xyChart": {
|
||||
"dataSets": [
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_tasks_queue\" AND metric.type=\"cloudtasks.googleapis.com/queue/task_attempt_count\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_RATE",
|
||||
"groupByFields": ["resource.labels.queue_id"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "STACKED_BAR",
|
||||
"legendTemplate": "${resource.labels.queue_id}"
|
||||
}
|
||||
],
|
||||
"yAxis": {
|
||||
"label": "Tasks/sec",
|
||||
"scale": "LINEAR"
|
||||
},
|
||||
"chartOptions": {
|
||||
"mode": "COLOR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 0,
|
||||
"yPos": 12,
|
||||
"width": 12,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Worker Logs (Search by job_id)",
|
||||
"logsPanel": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND (textPayload=~\"\\[job:\" OR jsonPayload.job_id!=\"\")",
|
||||
"resourceNames": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 0,
|
||||
"yPos": 16,
|
||||
"width": 6,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "CPU Utilization",
|
||||
"xyChart": {
|
||||
"dataSets": [
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/container/cpu/utilizations\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_PERCENTILE_50",
|
||||
"crossSeriesReducer": "REDUCE_MEAN"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "LINE",
|
||||
"legendTemplate": "CPU (p50)"
|
||||
},
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/container/cpu/utilizations\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_PERCENTILE_95",
|
||||
"crossSeriesReducer": "REDUCE_MEAN"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "LINE",
|
||||
"legendTemplate": "CPU (p95)"
|
||||
}
|
||||
],
|
||||
"yAxis": {
|
||||
"label": "CPU Utilization",
|
||||
"scale": "LINEAR"
|
||||
},
|
||||
"chartOptions": {
|
||||
"mode": "COLOR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 6,
|
||||
"yPos": 16,
|
||||
"width": 6,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Memory Utilization",
|
||||
"xyChart": {
|
||||
"dataSets": [
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/container/memory/utilizations\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_PERCENTILE_50",
|
||||
"crossSeriesReducer": "REDUCE_MEAN"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "LINE",
|
||||
"legendTemplate": "Memory (p50)"
|
||||
},
|
||||
{
|
||||
"timeSeriesQuery": {
|
||||
"timeSeriesFilter": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND metric.type=\"run.googleapis.com/container/memory/utilizations\"",
|
||||
"aggregation": {
|
||||
"alignmentPeriod": "60s",
|
||||
"perSeriesAligner": "ALIGN_PERCENTILE_95",
|
||||
"crossSeriesReducer": "REDUCE_MEAN"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plotType": "LINE",
|
||||
"legendTemplate": "Memory (p95)"
|
||||
}
|
||||
],
|
||||
"yAxis": {
|
||||
"label": "Memory Utilization",
|
||||
"scale": "LINEAR"
|
||||
},
|
||||
"chartOptions": {
|
||||
"mode": "COLOR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"xPos": 0,
|
||||
"yPos": 20,
|
||||
"width": 12,
|
||||
"height": 4,
|
||||
"widget": {
|
||||
"title": "Error Logs",
|
||||
"logsPanel": {
|
||||
"filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"karaoke-backend\" AND severity>=ERROR",
|
||||
"resourceNames": []
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Karaoke Job Inspector</title>
|
||||
<style>
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
|
||||
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
|
||||
color: #eee;
|
||||
min-height: 100vh;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
}
|
||||
.container {
|
||||
max-width: 600px;
|
||||
margin: 0 auto;
|
||||
padding-top: 40px;
|
||||
}
|
||||
h1 {
|
||||
text-align: center;
|
||||
font-size: 2rem;
|
||||
margin-bottom: 0.5rem;
|
||||
background: linear-gradient(90deg, #00d4ff, #7b2cbf);
|
||||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
background-clip: text;
|
||||
}
|
||||
.subtitle {
|
||||
text-align: center;
|
||||
color: #888;
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
.input-group {
|
||||
background: rgba(255,255,255,0.05);
|
||||
border-radius: 16px;
|
||||
padding: 30px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
label {
|
||||
display: block;
|
||||
font-size: 0.9rem;
|
||||
color: #aaa;
|
||||
margin-bottom: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
input[type="text"] {
|
||||
width: 100%;
|
||||
padding: 16px 20px;
|
||||
font-size: 1.5rem;
|
||||
font-family: 'Monaco', 'Menlo', monospace;
|
||||
border: 2px solid #333;
|
||||
border-radius: 10px;
|
||||
background: #0d0d1a;
|
||||
color: #00d4ff;
|
||||
outline: none;
|
||||
transition: border-color 0.2s;
|
||||
}
|
||||
input[type="text"]:focus {
|
||||
border-color: #00d4ff;
|
||||
}
|
||||
input[type="text"]::placeholder {
|
||||
color: #444;
|
||||
}
|
||||
.buttons {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
margin-top: 20px;
|
||||
}
|
||||
button {
|
||||
flex: 1;
|
||||
padding: 16px 24px;
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
border: none;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
transition: transform 0.1s, box-shadow 0.2s;
|
||||
}
|
||||
button:hover {
|
||||
transform: translateY(-2px);
|
||||
}
|
||||
button:active {
|
||||
transform: translateY(0);
|
||||
}
|
||||
.btn-primary {
|
||||
background: linear-gradient(135deg, #00d4ff, #0099cc);
|
||||
color: #000;
|
||||
}
|
||||
.btn-primary:hover {
|
||||
box-shadow: 0 4px 20px rgba(0, 212, 255, 0.4);
|
||||
}
|
||||
.btn-secondary {
|
||||
background: rgba(255,255,255,0.1);
|
||||
color: #fff;
|
||||
border: 1px solid #444;
|
||||
}
|
||||
.btn-secondary:hover {
|
||||
background: rgba(255,255,255,0.15);
|
||||
box-shadow: 0 4px 20px rgba(255,255,255,0.1);
|
||||
}
|
||||
.quick-links {
|
||||
background: rgba(255,255,255,0.03);
|
||||
border-radius: 16px;
|
||||
padding: 20px 30px;
|
||||
}
|
||||
.quick-links h3 {
|
||||
font-size: 0.8rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
color: #666;
|
||||
margin: 0 0 15px 0;
|
||||
}
|
||||
.quick-links a {
|
||||
display: block;
|
||||
color: #00d4ff;
|
||||
text-decoration: none;
|
||||
padding: 10px 0;
|
||||
border-bottom: 1px solid #222;
|
||||
transition: color 0.2s;
|
||||
}
|
||||
.quick-links a:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.quick-links a:hover {
|
||||
color: #fff;
|
||||
}
|
||||
.hint {
|
||||
text-align: center;
|
||||
color: #555;
|
||||
font-size: 0.85rem;
|
||||
margin-top: 30px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>🎤 Karaoke Job Inspector</h1>
|
||||
<p class="subtitle">Debug failed jobs in seconds</p>
|
||||
|
||||
<div class="input-group">
|
||||
<label for="jobId">Enter Job ID</label>
|
||||
<input
|
||||
type="text"
|
||||
id="jobId"
|
||||
placeholder="e.g. 8f03f0ac-1234-5678-abcd"
|
||||
autofocus
|
||||
autocomplete="off"
|
||||
>
|
||||
<div class="buttons">
|
||||
<button class="btn-primary" onclick="openLogs()">
|
||||
📋 View Logs
|
||||
</button>
|
||||
<button class="btn-secondary" onclick="openTrace()">
|
||||
🔍 View Trace
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quick-links">
|
||||
<h3>Quick Links</h3>
|
||||
<a href="https://console.cloud.google.com/monitoring/dashboards/builder/0ea8ed34-409c-4fa4-b663-d8e282f3b3a2?project=nomadkaraoke" target="_blank">
|
||||
📊 System Dashboard
|
||||
</a>
|
||||
<a href="https://console.cloud.google.com/logs/query;query=resource.type%3D%22cloud_run_revision%22%0Aresource.labels.service_name%3D%22karaoke-backend%22%0Aseverity%3E%3DERROR;duration=PT1H?project=nomadkaraoke" target="_blank">
|
||||
🚨 Recent Errors
|
||||
</a>
|
||||
<a href="https://console.cloud.google.com/cloudtasks?project=nomadkaraoke" target="_blank">
|
||||
📥 Cloud Tasks Queues
|
||||
</a>
|
||||
<a href="https://console.cloud.google.com/run/detail/us-central1/karaoke-backend/metrics?project=nomadkaraoke" target="_blank">
|
||||
☁️ Cloud Run Service
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<p class="hint">
|
||||
Tip: Press Enter after typing the job ID to view logs
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const PROJECT = 'nomadkaraoke';
|
||||
const SERVICE = 'karaoke-backend';
|
||||
|
||||
function getJobId() {
|
||||
const jobId = document.getElementById('jobId').value.trim();
|
||||
if (!jobId) {
|
||||
alert('Please enter a Job ID');
|
||||
return null;
|
||||
}
|
||||
return jobId;
|
||||
}
|
||||
|
||||
function openLogs() {
|
||||
const jobId = getJobId();
|
||||
if (!jobId) return;
|
||||
|
||||
// Build the Logs Explorer query
|
||||
const query = `resource.type="cloud_run_revision"
|
||||
resource.labels.service_name="${SERVICE}"
|
||||
"[job:${jobId}"`;
|
||||
|
||||
const encodedQuery = encodeURIComponent(query);
|
||||
const url = `https://console.cloud.google.com/logs/query;query=${encodedQuery};duration=PT6H?project=${PROJECT}`;
|
||||
|
||||
window.open(url, '_blank');
|
||||
}
|
||||
|
||||
function openTrace() {
|
||||
const jobId = getJobId();
|
||||
if (!jobId) return;
|
||||
|
||||
// Cloud Trace search by job_id attribute
|
||||
const url = `https://console.cloud.google.com/traces/list?project=${PROJECT}&pageState=("traceFilter":("chips":"job_id:${jobId}"))`;
|
||||
|
||||
window.open(url, '_blank');
|
||||
}
|
||||
|
||||
// Handle Enter key
|
||||
document.getElementById('jobId').addEventListener('keypress', function(e) {
|
||||
if (e.key === 'Enter') {
|
||||
openLogs();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+32
-1
@@ -4,4 +4,35 @@ import warnings
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="pydub.*")
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="syrics.*")
|
||||
|
||||
from .karaoke_gen import KaraokePrep
|
||||
# Lazy imports to avoid loading heavy dependencies when not needed.
|
||||
# KaraokePrep has many dependencies (lyrics_transcriber, etc.) which may not
|
||||
# be available in all contexts (e.g., backend which only needs audio_fetcher).
|
||||
#
|
||||
# Use explicit imports instead:
|
||||
# from karaoke_gen.karaoke_gen import KaraokePrep
|
||||
# from karaoke_gen.audio_fetcher import FlacFetcher, AudioSearchResult, ...
|
||||
|
||||
__all__ = [
|
||||
"KaraokePrep",
|
||||
# Audio fetcher
|
||||
"FlacFetcher",
|
||||
"AudioSearchResult",
|
||||
"AudioFetchResult",
|
||||
"AudioFetcherError",
|
||||
"NoResultsError",
|
||||
"DownloadError",
|
||||
"UserCancelledError",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy import for heavy modules."""
|
||||
if name == "KaraokePrep":
|
||||
from .karaoke_gen import KaraokePrep
|
||||
return KaraokePrep
|
||||
elif name in ("FlacFetcher", "AudioSearchResult", "AudioFetchResult",
|
||||
"AudioFetcherError", "NoResultsError", "DownloadError",
|
||||
"UserCancelledError"):
|
||||
from . import audio_fetcher
|
||||
return getattr(audio_fetcher, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
+313
-53
@@ -13,8 +13,8 @@ import tempfile
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import List, Optional, Dict, Any
|
||||
|
||||
# Global flag to track if user requested cancellation via Ctrl+C
|
||||
_interrupt_requested = False
|
||||
@@ -22,7 +22,14 @@ _interrupt_requested = False
|
||||
|
||||
@dataclass
|
||||
class AudioSearchResult:
|
||||
"""Represents a single search result for audio."""
|
||||
"""Represents a single search result for audio.
|
||||
|
||||
Used by both local CLI and cloud backend. Supports serialization
|
||||
for Firestore storage via to_dict()/from_dict().
|
||||
|
||||
For rich display, this class can serialize the full flacfetch Release
|
||||
data so remote CLIs can use flacfetch's shared display functions.
|
||||
"""
|
||||
|
||||
title: str
|
||||
artist: str
|
||||
@@ -31,13 +38,75 @@ class AudioSearchResult:
|
||||
duration: Optional[int] = None # Duration in seconds
|
||||
quality: Optional[str] = None # e.g., "FLAC", "320kbps", etc.
|
||||
source_id: Optional[str] = None # Unique ID from the source
|
||||
# Raw result object from the provider (for download)
|
||||
raw_result: Optional[object] = None
|
||||
index: int = 0 # Index in the results list (for API selection)
|
||||
seeders: Optional[int] = None # Number of seeders (for torrent sources)
|
||||
target_file: Optional[str] = None # Target filename in the release
|
||||
# Raw result object from the provider (for download) - not serialized
|
||||
raw_result: Optional[object] = field(default=None, repr=False)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict for JSON/Firestore serialization.
|
||||
|
||||
Includes full flacfetch Release data if available, enabling
|
||||
remote CLIs to use flacfetch's shared display functions.
|
||||
"""
|
||||
result = {
|
||||
"title": self.title,
|
||||
"artist": self.artist,
|
||||
"url": self.url,
|
||||
"provider": self.provider,
|
||||
"duration": self.duration,
|
||||
"quality": self.quality,
|
||||
"source_id": self.source_id,
|
||||
"index": self.index,
|
||||
"seeders": self.seeders,
|
||||
"target_file": self.target_file,
|
||||
}
|
||||
|
||||
# If we have a raw_result (flacfetch Release), include its full data
|
||||
# This enables rich display on the remote CLI
|
||||
if self.raw_result:
|
||||
try:
|
||||
release_dict = self.raw_result.to_dict()
|
||||
# Merge Release fields into result (they may override basic fields)
|
||||
for key in ['year', 'label', 'edition_info', 'release_type', 'channel',
|
||||
'view_count', 'size_bytes', 'target_file_size', 'track_pattern',
|
||||
'match_score', 'formatted_size', 'formatted_duration',
|
||||
'formatted_views', 'is_lossless', 'quality_str', 'quality']:
|
||||
if key in release_dict:
|
||||
# Use 'quality_data' for the quality dict to avoid confusion with quality string
|
||||
result_key = 'quality_data' if key == 'quality' else key
|
||||
result[result_key] = release_dict[key]
|
||||
except AttributeError:
|
||||
pass # raw_result doesn't have to_dict() method
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AudioSearchResult":
|
||||
"""Create from dict (e.g., from Firestore)."""
|
||||
return cls(
|
||||
title=data.get("title", ""),
|
||||
artist=data.get("artist", ""),
|
||||
url=data.get("url", ""),
|
||||
provider=data.get("provider", "Unknown"),
|
||||
duration=data.get("duration"),
|
||||
quality=data.get("quality"),
|
||||
source_id=data.get("source_id"),
|
||||
index=data.get("index", 0),
|
||||
seeders=data.get("seeders"),
|
||||
target_file=data.get("target_file"),
|
||||
raw_result=None, # Not stored in serialized form
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioFetchResult:
|
||||
"""Result of an audio fetch operation."""
|
||||
"""Result of an audio fetch operation.
|
||||
|
||||
Used by both local CLI and cloud backend. Supports serialization
|
||||
for Firestore storage via to_dict()/from_dict().
|
||||
"""
|
||||
|
||||
filepath: str
|
||||
artist: str
|
||||
@@ -45,6 +114,22 @@ class AudioFetchResult:
|
||||
provider: str
|
||||
duration: Optional[int] = None
|
||||
quality: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict for JSON/Firestore serialization."""
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AudioFetchResult":
|
||||
"""Create from dict (e.g., from Firestore)."""
|
||||
return cls(
|
||||
filepath=data.get("filepath", ""),
|
||||
artist=data.get("artist", ""),
|
||||
title=data.get("title", ""),
|
||||
provider=data.get("provider", "Unknown"),
|
||||
duration=data.get("duration"),
|
||||
quality=data.get("quality"),
|
||||
)
|
||||
|
||||
|
||||
class AudioFetcherError(Exception):
|
||||
@@ -160,6 +245,8 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
|
||||
This provides access to multiple audio sources including private music trackers
|
||||
and YouTube, with intelligent prioritization of high-quality sources.
|
||||
|
||||
Also exported as FlacFetcher for shorter name.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -183,6 +270,47 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
self._ops_api_key = ops_api_key or os.environ.get("OPS_API_KEY")
|
||||
self._provider_priority = provider_priority
|
||||
self._manager = None
|
||||
self._transmission_available = None # Cached result of Transmission check
|
||||
|
||||
def _check_transmission_available(self) -> bool:
|
||||
"""
|
||||
Check if Transmission daemon is available for torrent downloads.
|
||||
|
||||
This prevents adding tracker providers (Redacted/OPS) when Transmission
|
||||
isn't running, which would result in search results that can't be downloaded.
|
||||
|
||||
Returns:
|
||||
True if Transmission is available and responsive, False otherwise.
|
||||
"""
|
||||
if self._transmission_available is not None:
|
||||
self.logger.info(f"[Transmission] Using cached status: available={self._transmission_available}")
|
||||
return self._transmission_available
|
||||
|
||||
host = os.environ.get("TRANSMISSION_HOST", "localhost")
|
||||
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
|
||||
self.logger.info(f"[Transmission] Checking availability at {host}:{port}")
|
||||
|
||||
try:
|
||||
import transmission_rpc
|
||||
self.logger.info(f"[Transmission] transmission_rpc imported successfully")
|
||||
|
||||
client = transmission_rpc.Client(host=host, port=port, timeout=5)
|
||||
self.logger.info(f"[Transmission] Client created, calling session_stats()...")
|
||||
|
||||
# Simple test to verify connection works
|
||||
stats = client.session_stats()
|
||||
self.logger.info(f"[Transmission] Connected! Download dir: {getattr(stats, 'download_dir', 'unknown')}")
|
||||
|
||||
self._transmission_available = True
|
||||
except ImportError as e:
|
||||
self._transmission_available = False
|
||||
self.logger.warning(f"[Transmission] transmission_rpc not installed: {e}")
|
||||
except Exception as e:
|
||||
self._transmission_available = False
|
||||
self.logger.warning(f"[Transmission] Connection failed to {host}:{port}: {type(e).__name__}: {e}")
|
||||
|
||||
self.logger.info(f"[Transmission] Final status: available={self._transmission_available}")
|
||||
return self._transmission_available
|
||||
|
||||
def _get_manager(self):
|
||||
"""Lazily initialize and return the FetchManager."""
|
||||
@@ -193,30 +321,46 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
from flacfetch.downloaders.youtube import YoutubeDownloader
|
||||
|
||||
# Try to import TorrentDownloader (has optional dependencies)
|
||||
TorrentDownloader = None
|
||||
try:
|
||||
from flacfetch.downloaders.torrent import TorrentDownloader
|
||||
except ImportError:
|
||||
TorrentDownloader = None
|
||||
self.logger.debug("TorrentDownloader not available (missing dependencies)")
|
||||
|
||||
self._manager = FetchManager()
|
||||
|
||||
# Only add tracker providers if we can actually download from them
|
||||
# This requires both TorrentDownloader and a running Transmission daemon
|
||||
has_torrent_downloader = TorrentDownloader is not None
|
||||
transmission_available = self._check_transmission_available()
|
||||
can_use_trackers = has_torrent_downloader and transmission_available
|
||||
|
||||
self.logger.info(
|
||||
f"[FlacFetcher] Provider setup: TorrentDownloader={has_torrent_downloader}, "
|
||||
f"Transmission={transmission_available}, can_use_trackers={can_use_trackers}"
|
||||
)
|
||||
|
||||
if not can_use_trackers and (self._redacted_api_key or self._ops_api_key):
|
||||
self.logger.warning(
|
||||
"[FlacFetcher] Tracker providers (Redacted/OPS) DISABLED: "
|
||||
f"TorrentDownloader={has_torrent_downloader}, Transmission={transmission_available}. "
|
||||
"Only YouTube sources will be used."
|
||||
)
|
||||
|
||||
# Add providers and downloaders based on available API keys
|
||||
if self._redacted_api_key:
|
||||
if self._redacted_api_key and can_use_trackers:
|
||||
from flacfetch.providers.redacted import RedactedProvider
|
||||
|
||||
self._manager.add_provider(RedactedProvider(api_key=self._redacted_api_key))
|
||||
if TorrentDownloader:
|
||||
self._manager.register_downloader("Redacted", TorrentDownloader())
|
||||
self.logger.debug("Added Redacted provider")
|
||||
self._manager.register_downloader("Redacted", TorrentDownloader())
|
||||
self.logger.info("[FlacFetcher] Added Redacted provider with TorrentDownloader")
|
||||
|
||||
if self._ops_api_key:
|
||||
if self._ops_api_key and can_use_trackers:
|
||||
from flacfetch.providers.ops import OPSProvider
|
||||
|
||||
self._manager.add_provider(OPSProvider(api_key=self._ops_api_key))
|
||||
if TorrentDownloader:
|
||||
self._manager.register_downloader("OPS", TorrentDownloader())
|
||||
self.logger.debug("Added OPS provider")
|
||||
self._manager.register_downloader("OPS", TorrentDownloader())
|
||||
self.logger.info("[FlacFetcher] Added OPS provider with TorrentDownloader")
|
||||
|
||||
# Always add YouTube as a fallback provider with its downloader
|
||||
self._manager.add_provider(YoutubeProvider())
|
||||
@@ -252,7 +396,7 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
|
||||
# Convert to our AudioSearchResult format
|
||||
search_results = []
|
||||
for result in results:
|
||||
for i, result in enumerate(results):
|
||||
# Get quality as string if it's a Quality object
|
||||
quality = getattr(result, "quality", None)
|
||||
quality_str = str(quality) if quality else None
|
||||
@@ -266,6 +410,9 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
duration=getattr(result, "duration_seconds", None),
|
||||
quality=quality_str,
|
||||
source_id=getattr(result, "info_hash", None),
|
||||
index=i, # Set index for API selection
|
||||
seeders=getattr(result, "seeders", None),
|
||||
target_file=getattr(result, "target_file", None),
|
||||
raw_result=result,
|
||||
)
|
||||
)
|
||||
@@ -329,6 +476,40 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
except Exception as e:
|
||||
raise DownloadError(f"Failed to download {result.artist} - {result.title}: {e}") from e
|
||||
|
||||
def select_best(self, results: List[AudioSearchResult]) -> int:
|
||||
"""
|
||||
Select the best result from a list of search results.
|
||||
|
||||
Uses flacfetch's built-in quality ranking to determine the best source.
|
||||
This is useful for automated/non-interactive usage.
|
||||
|
||||
Args:
|
||||
results: List of AudioSearchResult objects from search()
|
||||
|
||||
Returns:
|
||||
Index of the best result in the list
|
||||
"""
|
||||
if not results:
|
||||
return 0
|
||||
|
||||
manager = self._get_manager()
|
||||
|
||||
# Get raw results that have raw_result set
|
||||
raw_results = [r.raw_result for r in results if r.raw_result is not None]
|
||||
|
||||
if raw_results:
|
||||
try:
|
||||
best = manager.select_best(raw_results)
|
||||
# Find index of best result
|
||||
for i, r in enumerate(results):
|
||||
if r.raw_result == best:
|
||||
return i
|
||||
except Exception as e:
|
||||
self.logger.warning(f"select_best failed, using first result: {e}")
|
||||
|
||||
# Fallback: return first result
|
||||
return 0
|
||||
|
||||
def search_and_download(
|
||||
self,
|
||||
artist: str,
|
||||
@@ -396,13 +577,15 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
self.logger.info(f"Downloading from {getattr(selected, 'source_name', 'Unknown')}...")
|
||||
|
||||
try:
|
||||
filepath = manager.download(
|
||||
# Use interruptible download so Ctrl+C works during torrent downloads
|
||||
filepath = self._interruptible_download(
|
||||
manager,
|
||||
selected,
|
||||
output_path=output_dir,
|
||||
output_filename=output_filename,
|
||||
)
|
||||
|
||||
if filepath is None:
|
||||
if not filepath:
|
||||
raise DownloadError(f"Download returned no file path for: {artist} - {title}")
|
||||
|
||||
self.logger.info(f"Downloaded to: {filepath}")
|
||||
@@ -488,6 +671,111 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
signal.signal(signal.SIGINT, original_handler)
|
||||
_interrupt_requested = False
|
||||
|
||||
def _interruptible_download(self, manager, selected, output_path: str, output_filename: str) -> str:
|
||||
"""
|
||||
Run download in a way that can be interrupted by Ctrl+C.
|
||||
|
||||
The flacfetch/transmission download is a blocking operation that doesn't
|
||||
respond to SIGINT while running (especially for torrent downloads).
|
||||
This method runs it in a background thread and periodically checks for interrupts.
|
||||
|
||||
Args:
|
||||
manager: The FetchManager instance
|
||||
selected: The selected result to download
|
||||
output_path: Directory to save the file
|
||||
output_filename: Filename to save as
|
||||
|
||||
Returns:
|
||||
Path to the downloaded file
|
||||
|
||||
Raises:
|
||||
UserCancelledError: If user presses Ctrl+C during download
|
||||
DownloadError: If download fails
|
||||
"""
|
||||
global _interrupt_requested
|
||||
_interrupt_requested = False
|
||||
result_container = {"filepath": None, "error": None}
|
||||
was_cancelled = False
|
||||
|
||||
def do_download():
|
||||
try:
|
||||
result_container["filepath"] = manager.download(
|
||||
selected,
|
||||
output_path=output_path,
|
||||
output_filename=output_filename,
|
||||
)
|
||||
except Exception as e:
|
||||
result_container["error"] = e
|
||||
|
||||
# Set up signal handler for immediate response to Ctrl+C
|
||||
original_handler = signal.getsignal(signal.SIGINT)
|
||||
|
||||
def interrupt_handler(signum, frame):
|
||||
global _interrupt_requested
|
||||
_interrupt_requested = True
|
||||
# Print immediately so user knows it was received
|
||||
print("\nCancelling download... please wait (may take a few seconds)", file=sys.stderr)
|
||||
|
||||
signal.signal(signal.SIGINT, interrupt_handler)
|
||||
|
||||
try:
|
||||
# Start download in background thread
|
||||
download_thread = threading.Thread(target=do_download, daemon=True)
|
||||
download_thread.start()
|
||||
|
||||
# Wait for completion with periodic interrupt checks
|
||||
while download_thread.is_alive():
|
||||
download_thread.join(timeout=0.2) # Check every 200ms
|
||||
if _interrupt_requested:
|
||||
was_cancelled = True
|
||||
# Clean up any pending torrents before raising
|
||||
self._cleanup_transmission_torrents(selected)
|
||||
raise UserCancelledError("Download cancelled by user (Ctrl+C)")
|
||||
|
||||
# Check for errors from the download
|
||||
if result_container["error"] is not None:
|
||||
raise result_container["error"]
|
||||
|
||||
return result_container["filepath"]
|
||||
|
||||
finally:
|
||||
# Restore original signal handler
|
||||
signal.signal(signal.SIGINT, original_handler)
|
||||
_interrupt_requested = False
|
||||
|
||||
def _cleanup_transmission_torrents(self, selected) -> None:
|
||||
"""
|
||||
Clean up any torrents in Transmission that were started for this download.
|
||||
|
||||
Called when a download is cancelled to remove incomplete torrents and their data.
|
||||
|
||||
Args:
|
||||
selected: The selected result that was being downloaded
|
||||
"""
|
||||
try:
|
||||
import transmission_rpc
|
||||
host = os.environ.get("TRANSMISSION_HOST", "localhost")
|
||||
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
|
||||
client = transmission_rpc.Client(host=host, port=port, timeout=5)
|
||||
|
||||
# Get the release name to match against torrents
|
||||
release_name = getattr(selected, 'name', None) or getattr(selected, 'title', None)
|
||||
if not release_name:
|
||||
self.logger.debug("[Transmission] No release name to match for cleanup")
|
||||
return
|
||||
|
||||
# Find and remove matching incomplete torrents
|
||||
torrents = client.get_torrents()
|
||||
for torrent in torrents:
|
||||
# Match by name similarity and incomplete status
|
||||
if torrent.progress < 100 and release_name.lower() in torrent.name.lower():
|
||||
self.logger.info(f"[Transmission] Removing cancelled torrent: {torrent.name}")
|
||||
client.remove_torrent(torrent.id, delete_data=True)
|
||||
|
||||
except Exception as e:
|
||||
# Don't fail the cancellation if cleanup fails
|
||||
self.logger.debug(f"[Transmission] Cleanup failed (non-fatal): {e}")
|
||||
|
||||
def _interactive_select(self, results: list, artist: str, title: str) -> object:
|
||||
"""
|
||||
Present search results to the user for interactive selection.
|
||||
@@ -539,41 +827,9 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
Raises:
|
||||
UserCancelledError: If user cancels selection
|
||||
"""
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Search Results for: {artist} - {title}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
for i, result in enumerate(results, 1):
|
||||
# Use correct flacfetch attribute names
|
||||
source_name = getattr(result, "source_name", "Unknown")
|
||||
result_title = getattr(result, "title", "Unknown")
|
||||
result_artist = getattr(result, "artist", "Unknown")
|
||||
quality = getattr(result, "quality", None)
|
||||
duration_seconds = getattr(result, "duration_seconds", None)
|
||||
seeders = getattr(result, "seeders", None)
|
||||
target_file = getattr(result, "target_file", None)
|
||||
|
||||
# Format duration if available
|
||||
duration_str = ""
|
||||
if duration_seconds:
|
||||
minutes = duration_seconds // 60
|
||||
seconds = duration_seconds % 60
|
||||
duration_str = f" [{minutes}:{seconds:02d}]"
|
||||
|
||||
# Format quality - it's a Quality object with __str__
|
||||
quality_str = f" ({quality})" if quality else ""
|
||||
|
||||
# Format seeders for torrent sources
|
||||
seeders_str = f" Seeders: {seeders}" if seeders is not None else ""
|
||||
|
||||
# Format target file
|
||||
file_str = f' "{target_file}"' if target_file else ""
|
||||
|
||||
print(f" {i}. [{source_name}] {result_artist} - {result_title}{quality_str}{duration_str}{seeders_str}{file_str}")
|
||||
|
||||
print()
|
||||
print(" 0. Cancel")
|
||||
print()
|
||||
# Use flacfetch's shared display function
|
||||
from flacfetch import print_releases
|
||||
print_releases(results, target_artist=artist, use_colors=True)
|
||||
|
||||
while True:
|
||||
try:
|
||||
@@ -598,6 +854,10 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
raise UserCancelledError("Selection cancelled by user (Ctrl+C)")
|
||||
|
||||
|
||||
# Alias for shorter name - used by backend and other consumers
|
||||
FlacFetcher = FlacFetchAudioFetcher
|
||||
|
||||
|
||||
def create_audio_fetcher(
|
||||
logger: Optional[logging.Logger] = None,
|
||||
redacted_api_key: Optional[str] = None,
|
||||
|
||||
@@ -771,8 +771,16 @@ class AudioProcessor:
|
||||
padded_result["other_stems"] = separation_result.get("other_stems", {})
|
||||
padded_result["backing_vocals"] = separation_result.get("backing_vocals", {})
|
||||
|
||||
self.logger.info(
|
||||
f"✓ Countdown padding applied to {len(padded_result['combined_instrumentals']) + 1} instrumental file(s)"
|
||||
)
|
||||
# Count actual padded files (don't assume clean instrumental was padded)
|
||||
padded_count = (1 if padded_result["clean_instrumental"].get("instrumental") else 0) + len(padded_result["combined_instrumentals"])
|
||||
|
||||
if padded_count > 0:
|
||||
self.logger.info(
|
||||
f"✓ Countdown padding applied to {padded_count} instrumental file(s)"
|
||||
)
|
||||
else:
|
||||
self.logger.warning(
|
||||
"No instrumental files found to pad. Check that audio separation completed successfully."
|
||||
)
|
||||
|
||||
return padded_result
|
||||
@@ -11,6 +11,7 @@ Similar pattern to LyricsTranscriber's ReviewServer.
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import threading
|
||||
import webbrowser
|
||||
from typing import List, Optional
|
||||
@@ -358,23 +359,74 @@ class InstrumentalReviewServer:
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
@staticmethod
|
||||
def _is_port_available(host: str, port: int) -> bool:
|
||||
"""Check if a port is available for binding."""
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.bind((host, port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _find_available_port(host: str, preferred_port: int, max_attempts: int = 100) -> int:
|
||||
"""
|
||||
Find an available port, starting with the preferred port.
|
||||
|
||||
Args:
|
||||
host: Host to bind to
|
||||
preferred_port: The preferred port to try first
|
||||
max_attempts: Maximum number of ports to try
|
||||
|
||||
Returns:
|
||||
An available port number
|
||||
|
||||
Raises:
|
||||
RuntimeError: If no available port could be found
|
||||
"""
|
||||
# Try the preferred port first
|
||||
if InstrumentalReviewServer._is_port_available(host, preferred_port):
|
||||
return preferred_port
|
||||
|
||||
# Try subsequent ports
|
||||
for offset in range(1, max_attempts):
|
||||
port = preferred_port + offset
|
||||
if port > 65535:
|
||||
break
|
||||
if InstrumentalReviewServer._is_port_available(host, port):
|
||||
return port
|
||||
|
||||
# Last resort: let the OS assign a port
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind((host, 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
def start_and_open_browser(self, port: int = 8765) -> str:
|
||||
"""
|
||||
Start server, open browser, and block until selection is submitted.
|
||||
|
||||
Args:
|
||||
port: Port to run the server on
|
||||
port: Preferred port to run the server on. If unavailable, will
|
||||
automatically find an available port.
|
||||
|
||||
Returns:
|
||||
The user's selection ("clean", "with_backing", or "custom")
|
||||
"""
|
||||
self._app = self._create_app()
|
||||
|
||||
# Find an available port (handles concurrent CLI instances)
|
||||
host = "127.0.0.1"
|
||||
actual_port = self._find_available_port(host, port)
|
||||
if actual_port != port:
|
||||
logger.info(f"Port {port} in use, using port {actual_port} instead")
|
||||
|
||||
# Run uvicorn in a separate thread
|
||||
config = uvicorn.Config(
|
||||
self._app,
|
||||
host="127.0.0.1",
|
||||
port=port,
|
||||
host=host,
|
||||
port=actual_port,
|
||||
log_level="warning",
|
||||
)
|
||||
server = uvicorn.Server(config)
|
||||
@@ -389,7 +441,7 @@ class InstrumentalReviewServer:
|
||||
import time
|
||||
time.sleep(0.5)
|
||||
|
||||
url = f"http://localhost:{port}/"
|
||||
url = f"http://localhost:{actual_port}/"
|
||||
logger.info(f"Instrumental review server started at {url}")
|
||||
|
||||
# Open browser
|
||||
|
||||
@@ -1428,9 +1428,25 @@
|
||||
<div class="success-screen">
|
||||
<h2>✓ Selection Submitted</h2>
|
||||
<p>You selected: <strong>${escapeHtml(selectionLabel)}</strong></p>
|
||||
<p style="color: var(--text-muted);">You can close this window now.</p>
|
||||
<p id="close-msg" style="color: var(--text-muted);">Closing in <span id="countdown">2</span>s...</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
// Auto-close window after 2 seconds
|
||||
let countdown = 2;
|
||||
const countdownEl = document.getElementById('countdown');
|
||||
const countdownInterval = setInterval(() => {
|
||||
countdown--;
|
||||
if (countdownEl) countdownEl.textContent = countdown;
|
||||
if (countdown <= 0) {
|
||||
clearInterval(countdownInterval);
|
||||
// Try to close the window (works for windows opened by script)
|
||||
window.close();
|
||||
// If window.close() didn't work, update message
|
||||
const msg = document.getElementById('close-msg');
|
||||
if (msg) msg.textContent = 'You can close this window now.';
|
||||
}
|
||||
}, 1000);
|
||||
} catch (error) {
|
||||
showError(error.message);
|
||||
if (btn) {
|
||||
|
||||
@@ -47,6 +47,7 @@ class KaraokeFinalise:
|
||||
user_youtube_credentials=None, # Add support for pre-stored credentials
|
||||
server_side_mode=False, # New parameter for server-side deployment
|
||||
selected_instrumental_file=None, # Add support for pre-selected instrumental file
|
||||
countdown_padding_seconds=None, # Padding applied to vocals; instrumental must match
|
||||
):
|
||||
self.log_level = log_level
|
||||
self.log_formatter = log_formatter
|
||||
@@ -108,6 +109,7 @@ class KaraokeFinalise:
|
||||
self.user_youtube_credentials = user_youtube_credentials
|
||||
self.server_side_mode = server_side_mode
|
||||
self.selected_instrumental_file = selected_instrumental_file
|
||||
self.countdown_padding_seconds = countdown_padding_seconds
|
||||
|
||||
self.suffixes = {
|
||||
"title_mov": " (Title).mov",
|
||||
@@ -732,6 +734,32 @@ class KaraokeFinalise:
|
||||
artist, title = base_name.split(" - ", 1)
|
||||
return base_name, artist, title
|
||||
|
||||
def _pad_audio_file(self, input_audio, output_audio, padding_seconds):
|
||||
"""
|
||||
Pad an audio file by prepending silence at the beginning.
|
||||
|
||||
Uses the same ffmpeg approach as LyricsTranscriber's CountdownProcessor
|
||||
to ensure consistent padding behavior.
|
||||
|
||||
Args:
|
||||
input_audio: Path to input audio file
|
||||
output_audio: Path for the padded output file
|
||||
padding_seconds: Amount of silence to prepend (in seconds)
|
||||
"""
|
||||
self.logger.info(f"Padding audio file with {padding_seconds}s of silence")
|
||||
|
||||
# Use ffmpeg to prepend silence - this matches the approach in audio_processor.py
|
||||
# adelay filter adds delay in milliseconds
|
||||
delay_ms = int(padding_seconds * 1000)
|
||||
|
||||
ffmpeg_command = (
|
||||
f'{self.ffmpeg_base_command} -i "{input_audio}" '
|
||||
f'-af "adelay={delay_ms}|{delay_ms}" '
|
||||
f'"{output_audio}"'
|
||||
)
|
||||
|
||||
self.execute_command(ffmpeg_command, f"Padding audio with {padding_seconds}s silence")
|
||||
|
||||
def execute_command(self, command, description):
|
||||
"""Execute a shell command and log the output. For general commands (rclone, etc.)"""
|
||||
self.logger.info(f"{description}")
|
||||
@@ -776,11 +804,32 @@ class KaraokeFinalise:
|
||||
|
||||
def remux_with_instrumental(self, with_vocals_file, instrumental_audio, output_file):
|
||||
"""Remux the video with instrumental audio to create karaoke version"""
|
||||
# Safety net: If countdown padding was applied to vocals, ensure instrumental is padded too
|
||||
actual_instrumental = instrumental_audio
|
||||
if self.countdown_padding_seconds and self.countdown_padding_seconds > 0:
|
||||
# Check if the instrumental file is already padded (has "(Padded)" in name)
|
||||
if "(Padded)" not in instrumental_audio:
|
||||
self.logger.warning(
|
||||
f"Countdown padding ({self.countdown_padding_seconds}s) was applied to vocals, "
|
||||
f"but instrumental doesn't appear to be padded. Creating padded version..."
|
||||
)
|
||||
# Create a padded version of the instrumental
|
||||
base, ext = os.path.splitext(instrumental_audio)
|
||||
padded_instrumental = f"{base} (Padded){ext}"
|
||||
|
||||
if not os.path.exists(padded_instrumental):
|
||||
self._pad_audio_file(instrumental_audio, padded_instrumental, self.countdown_padding_seconds)
|
||||
self.logger.info(f"Created padded instrumental: {padded_instrumental}")
|
||||
|
||||
actual_instrumental = padded_instrumental
|
||||
else:
|
||||
self.logger.info(f"Using already-padded instrumental: {instrumental_audio}")
|
||||
|
||||
# This operation is primarily I/O bound (remuxing), so hardware acceleration doesn't provide significant benefit
|
||||
# Keep the existing approach but use the new execute method
|
||||
ffmpeg_command = (
|
||||
f'{self.ffmpeg_base_command} -an -i "{with_vocals_file}" '
|
||||
f'-vn -i "{instrumental_audio}" -c:v copy -c:a pcm_s16le "{output_file}"'
|
||||
f'-vn -i "{actual_instrumental}" -c:v copy -c:a pcm_s16le "{output_file}"'
|
||||
)
|
||||
self.execute_command(ffmpeg_command, "Remuxing video with instrumental audio")
|
||||
|
||||
|
||||
@@ -259,6 +259,101 @@ class KaraokePrep:
|
||||
self.artist = metadata_result["artist"]
|
||||
self.title = metadata_result["title"]
|
||||
|
||||
def _scan_directory_for_instrumentals(self, track_output_dir, artist_title):
|
||||
"""
|
||||
Scan the directory for existing instrumental files and build a separated_audio structure.
|
||||
|
||||
This is used when transcription was skipped (existing files found) but we need to
|
||||
pad instrumentals due to countdown padding.
|
||||
|
||||
Args:
|
||||
track_output_dir: The track output directory to scan
|
||||
artist_title: The "{artist} - {title}" string for matching files
|
||||
|
||||
Returns:
|
||||
Dictionary with separated_audio structure containing found instrumental paths
|
||||
"""
|
||||
self.logger.info(f"Scanning directory for existing instrumentals: {track_output_dir}")
|
||||
|
||||
separated_audio = {
|
||||
"clean_instrumental": {},
|
||||
"backing_vocals": {},
|
||||
"other_stems": {},
|
||||
"combined_instrumentals": {},
|
||||
}
|
||||
|
||||
# Search patterns for instrumental files
|
||||
# Files are named like: "{artist} - {title} (Instrumental {model}).flac"
|
||||
# Or with backing vocals: "{artist} - {title} (Instrumental +BV {model}).flac"
|
||||
|
||||
# Look for files in the track output directory
|
||||
search_dir = track_output_dir
|
||||
|
||||
# Find all instrumental files (not padded ones - we want the originals)
|
||||
instrumental_pattern = os.path.join(search_dir, f"{artist_title} (Instrumental*.flac")
|
||||
instrumental_files = glob.glob(instrumental_pattern)
|
||||
|
||||
# Also check for wav files
|
||||
instrumental_pattern_wav = os.path.join(search_dir, f"{artist_title} (Instrumental*.wav")
|
||||
instrumental_files.extend(glob.glob(instrumental_pattern_wav))
|
||||
|
||||
self.logger.debug(f"Found {len(instrumental_files)} instrumental files")
|
||||
|
||||
for filepath in instrumental_files:
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
# Skip already padded files
|
||||
if "(Padded)" in filename:
|
||||
self.logger.debug(f"Skipping already padded file: {filename}")
|
||||
continue
|
||||
|
||||
# Determine if it's a combined instrumental (+BV) or clean instrumental
|
||||
if "+BV" in filename or "+bv" in filename.lower():
|
||||
# Combined instrumental with backing vocals
|
||||
# Extract model name from filename
|
||||
# Pattern: "(Instrumental +BV {model}).flac"
|
||||
model_match = re.search(r'\(Instrumental \+BV ([^)]+)\)', filename)
|
||||
if model_match:
|
||||
model_name = model_match.group(1).strip()
|
||||
separated_audio["combined_instrumentals"][model_name] = filepath
|
||||
self.logger.info(f"Found combined instrumental: {filename}")
|
||||
else:
|
||||
# Clean instrumental (no backing vocals)
|
||||
# Pattern: "(Instrumental {model}).flac"
|
||||
model_match = re.search(r'\(Instrumental ([^)]+)\)', filename)
|
||||
if model_match:
|
||||
# Use as clean instrumental if we don't have one yet
|
||||
if not separated_audio["clean_instrumental"].get("instrumental"):
|
||||
separated_audio["clean_instrumental"]["instrumental"] = filepath
|
||||
self.logger.info(f"Found clean instrumental: {filename}")
|
||||
else:
|
||||
# Additional clean instrumentals go to combined_instrumentals for padding
|
||||
model_name = model_match.group(1).strip()
|
||||
separated_audio["combined_instrumentals"][model_name] = filepath
|
||||
self.logger.info(f"Found additional instrumental: {filename}")
|
||||
|
||||
# Also look for backing vocals files
|
||||
backing_vocals_pattern = os.path.join(search_dir, f"{artist_title} (Backing Vocals*.flac")
|
||||
backing_vocals_files = glob.glob(backing_vocals_pattern)
|
||||
backing_vocals_pattern_wav = os.path.join(search_dir, f"{artist_title} (Backing Vocals*.wav")
|
||||
backing_vocals_files.extend(glob.glob(backing_vocals_pattern_wav))
|
||||
|
||||
for filepath in backing_vocals_files:
|
||||
filename = os.path.basename(filepath)
|
||||
model_match = re.search(r'\(Backing Vocals ([^)]+)\)', filename)
|
||||
if model_match:
|
||||
model_name = model_match.group(1).strip()
|
||||
if model_name not in separated_audio["backing_vocals"]:
|
||||
separated_audio["backing_vocals"][model_name] = {"backing_vocals": filepath}
|
||||
self.logger.info(f"Found backing vocals: {filename}")
|
||||
|
||||
# Log summary
|
||||
clean_count = 1 if separated_audio["clean_instrumental"].get("instrumental") else 0
|
||||
combined_count = len(separated_audio["combined_instrumentals"])
|
||||
self.logger.info(f"Directory scan complete: {clean_count} clean instrumental, {combined_count} combined instrumentals")
|
||||
|
||||
return separated_audio
|
||||
|
||||
async def prep_single_track(self):
|
||||
# Add signal handler at the start
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -767,6 +862,18 @@ class KaraokePrep:
|
||||
f"Applying {padding_seconds}s padding to all instrumental files to sync with vocal countdown"
|
||||
)
|
||||
|
||||
# If separated_audio is empty (e.g., transcription was skipped but existing files have countdown),
|
||||
# scan the directory for existing instrumental files
|
||||
has_instrumentals = (
|
||||
processed_track["separated_audio"].get("clean_instrumental", {}).get("instrumental") or
|
||||
processed_track["separated_audio"].get("combined_instrumentals")
|
||||
)
|
||||
if not has_instrumentals:
|
||||
self.logger.info("No instrumentals in separated_audio, scanning directory for existing files...")
|
||||
processed_track["separated_audio"] = self._scan_directory_for_instrumentals(
|
||||
track_output_dir, artist_title
|
||||
)
|
||||
|
||||
# Apply padding using AudioProcessor
|
||||
padded_separation_result = self.audio_processor.apply_countdown_padding_to_instrumentals(
|
||||
separation_result=processed_track["separated_audio"],
|
||||
|
||||
@@ -11,6 +11,9 @@ from .utils import sanitize_filename
|
||||
|
||||
# Placeholder class or functions for lyrics processing
|
||||
class LyricsProcessor:
|
||||
# Standard countdown padding duration used by LyricsTranscriber
|
||||
COUNTDOWN_PADDING_SECONDS = 3.0
|
||||
|
||||
def __init__(
|
||||
self, logger, style_params_json, lyrics_file, skip_transcription, skip_transcription_review, render_video, subtitle_offset_ms
|
||||
):
|
||||
@@ -22,6 +25,60 @@ class LyricsProcessor:
|
||||
self.render_video = render_video
|
||||
self.subtitle_offset_ms = subtitle_offset_ms
|
||||
|
||||
def _detect_countdown_padding_from_lrc(self, lrc_filepath):
|
||||
"""
|
||||
Detect if countdown padding was applied by checking the first lyric timestamp in the LRC file.
|
||||
|
||||
LRC format timestamps look like: [mm:ss.xx] or [mm:ss.xxx]
|
||||
If the first lyric timestamp is >= 3.0 seconds, countdown padding was likely applied.
|
||||
|
||||
Args:
|
||||
lrc_filepath: Path to the LRC file
|
||||
|
||||
Returns:
|
||||
Tuple of (countdown_padding_added: bool, countdown_padding_seconds: float)
|
||||
"""
|
||||
try:
|
||||
with open(lrc_filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find all timestamp patterns in the LRC file
|
||||
# LRC timestamps: [mm:ss.xx] or [mm:ss.xxx]
|
||||
timestamp_pattern = r'\[(\d{1,2}):(\d{2})\.(\d{2,3})\]'
|
||||
matches = re.findall(timestamp_pattern, content)
|
||||
|
||||
if not matches:
|
||||
self.logger.debug("No timestamps found in LRC file")
|
||||
return (False, 0.0)
|
||||
|
||||
# Find the first non-metadata timestamp (metadata like [ar:Artist] doesn't have decimal)
|
||||
# We already filtered for decimal timestamps in our pattern
|
||||
first_timestamp = matches[0]
|
||||
minutes = int(first_timestamp[0])
|
||||
seconds = int(first_timestamp[1])
|
||||
# Handle both .xx and .xxx formats
|
||||
centiseconds = first_timestamp[2]
|
||||
if len(centiseconds) == 2:
|
||||
milliseconds = int(centiseconds) * 10
|
||||
else:
|
||||
milliseconds = int(centiseconds)
|
||||
|
||||
first_lyric_time = minutes * 60 + seconds + milliseconds / 1000.0
|
||||
|
||||
self.logger.debug(f"First lyric timestamp in LRC: {first_lyric_time:.3f}s")
|
||||
|
||||
# If first lyric is at or after 3 seconds, countdown padding was applied
|
||||
# Use a small buffer (2.5s) to account for songs that naturally start a bit late
|
||||
if first_lyric_time >= 2.5:
|
||||
self.logger.info(f"Detected countdown padding from LRC: first lyric at {first_lyric_time:.2f}s")
|
||||
return (True, self.COUNTDOWN_PADDING_SECONDS)
|
||||
|
||||
return (False, 0.0)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to detect countdown padding from LRC file: {e}")
|
||||
return (False, 0.0)
|
||||
|
||||
def find_best_split_point(self, line):
|
||||
"""
|
||||
Find the best split point in a line based on the specified criteria.
|
||||
@@ -138,23 +195,43 @@ class LyricsProcessor:
|
||||
lyrics_video_path = os.path.join(lyrics_dir, f"{sanitized_artist} - {sanitized_title} (With Vocals).mkv")
|
||||
lyrics_lrc_path = os.path.join(lyrics_dir, f"{sanitized_artist} - {sanitized_title} (Karaoke).lrc")
|
||||
|
||||
# If files exist in parent directory, return early
|
||||
# If files exist in parent directory, return early (but detect countdown padding first)
|
||||
if os.path.exists(parent_video_path) and os.path.exists(parent_lrc_path):
|
||||
self.logger.info(f"Found existing video and LRC files in parent directory, skipping transcription")
|
||||
self.logger.info("Found existing video and LRC files in parent directory, skipping transcription")
|
||||
|
||||
# Detect countdown padding from existing LRC file
|
||||
countdown_padding_added, countdown_padding_seconds = self._detect_countdown_padding_from_lrc(parent_lrc_path)
|
||||
|
||||
if countdown_padding_added:
|
||||
self.logger.info(f"Existing files have countdown padding: {countdown_padding_seconds}s")
|
||||
|
||||
return {
|
||||
"lrc_filepath": parent_lrc_path,
|
||||
"ass_filepath": parent_video_path,
|
||||
"countdown_padding_added": countdown_padding_added,
|
||||
"countdown_padding_seconds": countdown_padding_seconds,
|
||||
"padded_audio_filepath": None, # Original padded audio may not exist
|
||||
}
|
||||
|
||||
# If files exist in lyrics directory, copy to parent and return
|
||||
# If files exist in lyrics directory, copy to parent and return (but detect countdown padding first)
|
||||
if os.path.exists(lyrics_video_path) and os.path.exists(lyrics_lrc_path):
|
||||
self.logger.info(f"Found existing video and LRC files in lyrics directory, copying to parent")
|
||||
self.logger.info("Found existing video and LRC files in lyrics directory, copying to parent")
|
||||
os.makedirs(track_output_dir, exist_ok=True)
|
||||
shutil.copy2(lyrics_video_path, parent_video_path)
|
||||
shutil.copy2(lyrics_lrc_path, parent_lrc_path)
|
||||
|
||||
# Detect countdown padding from existing LRC file
|
||||
countdown_padding_added, countdown_padding_seconds = self._detect_countdown_padding_from_lrc(parent_lrc_path)
|
||||
|
||||
if countdown_padding_added:
|
||||
self.logger.info(f"Existing files have countdown padding: {countdown_padding_seconds}s")
|
||||
|
||||
return {
|
||||
"lrc_filepath": parent_lrc_path,
|
||||
"ass_filepath": parent_video_path,
|
||||
"countdown_padding_added": countdown_padding_added,
|
||||
"countdown_padding_seconds": countdown_padding_seconds,
|
||||
"padded_audio_filepath": None, # Original padded audio may not exist
|
||||
}
|
||||
|
||||
# Create lyrics directory if it doesn't exist
|
||||
|
||||
@@ -820,6 +820,12 @@ async def async_main():
|
||||
|
||||
logger.info(f"Selected instrumental file: {selected_instrumental_file}")
|
||||
|
||||
# Get countdown padding info from track (if vocals were padded, instrumental must match)
|
||||
countdown_padding_seconds = None
|
||||
if track.get("countdown_padding_added", False):
|
||||
countdown_padding_seconds = track.get("countdown_padding_seconds", 3.0)
|
||||
logger.info(f"Countdown padding detected: {countdown_padding_seconds}s (will be applied to instrumental if needed)")
|
||||
|
||||
# Load CDG styles if CDG generation is enabled
|
||||
cdg_styles = None
|
||||
if args.enable_cdg:
|
||||
@@ -864,6 +870,7 @@ async def async_main():
|
||||
keep_brand_code=getattr(args, 'keep_brand_code', False),
|
||||
non_interactive=args.yes,
|
||||
selected_instrumental_file=selected_instrumental_file,
|
||||
countdown_padding_seconds=countdown_padding_seconds,
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
+112
-29
@@ -36,6 +36,8 @@ from typing import Any, Dict, Optional
|
||||
import requests
|
||||
|
||||
from .cli_args import create_parser, process_style_overrides, is_url, is_file
|
||||
# Use flacfetch's shared display functions for consistent formatting
|
||||
from flacfetch import print_releases, Release
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
@@ -1329,6 +1331,54 @@ class JobMonitor:
|
||||
"""Get user-friendly description for a status."""
|
||||
return self.STATUS_DESCRIPTIONS.get(status, status)
|
||||
|
||||
def _show_download_progress(self, job_data: Dict[str, Any]) -> None:
|
||||
"""Show detailed download progress during audio download."""
|
||||
try:
|
||||
# Get provider from job state_data
|
||||
state_data = job_data.get('state_data', {})
|
||||
provider = state_data.get('selected_audio_provider', 'unknown')
|
||||
|
||||
# For non-torrent providers (YouTube), just show simple message
|
||||
if provider.lower() == 'youtube':
|
||||
self.logger.info(f" [Downloading from YouTube...]")
|
||||
return
|
||||
|
||||
# Query health endpoint for transmission status (torrent providers)
|
||||
health_url = f"{self.config.service_url}/api/health/detailed"
|
||||
response = requests.get(health_url, timeout=5)
|
||||
|
||||
if response.status_code == 200:
|
||||
health = response.json()
|
||||
transmission = health.get('dependencies', {}).get('transmission', {})
|
||||
|
||||
if transmission.get('available'):
|
||||
torrents = transmission.get('torrents', [])
|
||||
if torrents:
|
||||
# Show info about active torrents
|
||||
for t in torrents:
|
||||
progress = t.get('progress', 0)
|
||||
peers = t.get('peers', 0)
|
||||
speed = t.get('download_speed', 0)
|
||||
stalled = t.get('stalled', False)
|
||||
|
||||
if stalled:
|
||||
self.logger.info(f" [Downloading from {provider}] {progress:.1f}% - STALLED (no peers)")
|
||||
elif progress < 100:
|
||||
self.logger.info(f" [Downloading from {provider}] {progress:.1f}% @ {speed:.1f} KB/s ({peers} peers)")
|
||||
else:
|
||||
self.logger.info(f" [Downloading from {provider}] Complete, processing...")
|
||||
else:
|
||||
# No torrents - might be starting or YouTube download
|
||||
self.logger.info(f" [Downloading from {provider}] Starting download...")
|
||||
else:
|
||||
self.logger.info(f" [Downloading from {provider}] Transmission not available - download may fail")
|
||||
else:
|
||||
self.logger.info(f" [Downloading from {provider}]...")
|
||||
|
||||
except Exception as e:
|
||||
# Fall back to simple message
|
||||
self.logger.info(f" [Downloading audio...]")
|
||||
|
||||
def open_browser(self, url: str) -> None:
|
||||
"""Open URL in the default browser."""
|
||||
system = platform.system()
|
||||
@@ -1517,6 +1567,47 @@ class JobMonitor:
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error submitting selection: {e}")
|
||||
|
||||
def _convert_api_result_to_release_dict(self, result: dict) -> dict:
|
||||
"""
|
||||
Convert API search result to a dict compatible with flacfetch's Release.from_dict().
|
||||
|
||||
This enables using flacfetch's shared display functions for consistent,
|
||||
rich formatting between local and remote CLIs.
|
||||
"""
|
||||
# Build quality dict from API response
|
||||
quality_data = result.get('quality_data') or {
|
||||
"format": "OTHER",
|
||||
"media": "OTHER",
|
||||
}
|
||||
|
||||
return {
|
||||
"title": result.get('title', ''),
|
||||
"artist": result.get('artist', ''),
|
||||
"source_name": result.get('provider', 'Unknown'),
|
||||
"download_url": result.get('url'),
|
||||
"info_hash": result.get('source_id'),
|
||||
"size_bytes": result.get('size_bytes'),
|
||||
"year": result.get('year'),
|
||||
"edition_info": result.get('edition_info'),
|
||||
"label": result.get('label'),
|
||||
"release_type": result.get('release_type'),
|
||||
"seeders": result.get('seeders'),
|
||||
"channel": result.get('channel'),
|
||||
"view_count": result.get('view_count'),
|
||||
"duration_seconds": result.get('duration'),
|
||||
"target_file": result.get('target_file'),
|
||||
"target_file_size": result.get('target_file_size'),
|
||||
"track_pattern": result.get('track_pattern'),
|
||||
"match_score": result.get('match_score', 0.0),
|
||||
"quality": quality_data,
|
||||
# Pre-computed fields
|
||||
"formatted_size": result.get('formatted_size'),
|
||||
"formatted_duration": result.get('formatted_duration'),
|
||||
"formatted_views": result.get('formatted_views'),
|
||||
"is_lossless": result.get('is_lossless', False),
|
||||
"quality_str": result.get('quality_str') or result.get('quality', ''),
|
||||
}
|
||||
|
||||
def handle_audio_selection(self, job_id: str) -> None:
|
||||
"""Handle audio source selection interaction (Batch 5)."""
|
||||
self.logger.info("=" * 60)
|
||||
@@ -1527,6 +1618,7 @@ class JobMonitor:
|
||||
# Get search results
|
||||
results_data = self.client.get_audio_search_results(job_id)
|
||||
results = results_data.get('results', [])
|
||||
artist = results_data.get('artist', 'Unknown')
|
||||
|
||||
if not results:
|
||||
self.logger.error("No search results available")
|
||||
@@ -1537,42 +1629,27 @@ class JobMonitor:
|
||||
self.logger.info("Non-interactive mode: Auto-selecting first result")
|
||||
selection_index = 0
|
||||
else:
|
||||
self.logger.info("")
|
||||
self.logger.info("Choose which audio source to download:")
|
||||
self.logger.info("")
|
||||
# Convert API results to Release-compatible dicts for flacfetch display
|
||||
# This gives us the same rich, colorized output as the local CLI
|
||||
release_dicts = [self._convert_api_result_to_release_dict(r) for r in results]
|
||||
|
||||
for result in results:
|
||||
index = result.get('index', 0)
|
||||
provider = result.get('provider', 'Unknown')
|
||||
artist = result.get('artist', 'Unknown')
|
||||
title = result.get('title', 'Unknown')
|
||||
quality = result.get('quality', '')
|
||||
duration = result.get('duration')
|
||||
|
||||
# Format duration if available
|
||||
duration_str = ""
|
||||
if duration:
|
||||
minutes = duration // 60
|
||||
seconds = duration % 60
|
||||
duration_str = f" [{minutes}:{seconds:02d}]"
|
||||
|
||||
quality_str = f" ({quality})" if quality else ""
|
||||
|
||||
self.logger.info(f" {index + 1}) [{provider}] {artist} - {title}{quality_str}{duration_str}")
|
||||
|
||||
self.logger.info("")
|
||||
# Use flacfetch's shared display function
|
||||
print_releases(release_dicts, target_artist=artist, use_colors=True)
|
||||
|
||||
selection_index = -1
|
||||
while selection_index < 0:
|
||||
try:
|
||||
choice = input(f"Enter your choice (1-{len(results)}): ").strip()
|
||||
choice = input(f"\nSelect a release (1-{len(results)}, 0 to cancel): ").strip()
|
||||
if choice == "0":
|
||||
self.logger.info("Selection cancelled by user")
|
||||
raise KeyboardInterrupt
|
||||
choice_num = int(choice)
|
||||
if 1 <= choice_num <= len(results):
|
||||
selection_index = choice_num - 1
|
||||
else:
|
||||
self.logger.error(f"Please enter a number between 1 and {len(results)}")
|
||||
print(f"Please enter a number between 0 and {len(results)}")
|
||||
except ValueError:
|
||||
self.logger.error("Please enter a valid number")
|
||||
print("Please enter a valid number")
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
raise
|
||||
@@ -2038,9 +2115,15 @@ class JobMonitor:
|
||||
self._polls_without_updates = 0
|
||||
else:
|
||||
self._polls_without_updates += 1
|
||||
if self._polls_without_updates >= self._heartbeat_interval:
|
||||
description = self._get_status_description(status)
|
||||
self.logger.info(f" [Still processing: {description}]")
|
||||
# More frequent updates during audio download (every poll)
|
||||
heartbeat_threshold = 1 if status == 'downloading_audio' else self._heartbeat_interval
|
||||
if self._polls_without_updates >= heartbeat_threshold:
|
||||
if status == 'downloading_audio':
|
||||
# Show detailed download progress including transmission status
|
||||
self._show_download_progress(job_data)
|
||||
else:
|
||||
description = self._get_status_description(status)
|
||||
self.logger.info(f" [Still processing: {description}]")
|
||||
self._polls_without_updates = 0
|
||||
|
||||
# Handle human interaction points
|
||||
|
||||
Generated
+429
-418
File diff suppressed because it is too large.
Load diff
+2
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.71.69"
|
||||
version = "0.75.2"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
@@ -57,6 +57,7 @@ syllables = ">=1"
|
||||
fonttools = ">=4.55"
|
||||
ollama = ">=0.4.7"
|
||||
shortuuid = ">=1.0.13"
|
||||
nest-asyncio = ">=1.5"
|
||||
openai = ">=1.63.2"
|
||||
langgraph = ">=0.2.0"
|
||||
langchain = ">=0.3.0"
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -435,6 +435,266 @@ class TestExistingInstrumentalPadding:
|
||||
assert '3.0' in call_args or 3.0 in call_args
|
||||
|
||||
|
||||
class TestDetectCountdownPaddingFromLRC:
|
||||
"""Tests for detecting countdown padding from existing LRC files."""
|
||||
|
||||
def test_detect_padding_from_lrc_with_padding(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that countdown padding is detected when first lyric is after 3 seconds."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at 3 seconds (padding was applied)
|
||||
lrc_content = """[ti:Test Song]
|
||||
[ar:Test Artist]
|
||||
[00:03.50]First line of lyrics
|
||||
[00:06.20]Second line of lyrics
|
||||
[00:10.00]Third line of lyrics
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is True
|
||||
assert padding_seconds == 3.0
|
||||
|
||||
def test_detect_padding_from_lrc_without_padding(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that no padding is detected when first lyric is before 3 seconds."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at 0.5 seconds (no padding)
|
||||
lrc_content = """[ti:Test Song]
|
||||
[ar:Test Artist]
|
||||
[00:00.50]First line of lyrics
|
||||
[00:03.20]Second line of lyrics
|
||||
[00:06.00]Third line of lyrics
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_edge_case(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test detection at the 2.5 second boundary."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at exactly 2.5 seconds (boundary)
|
||||
lrc_content = """[00:02.50]First line at boundary
|
||||
[00:05.00]Second line
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
# 2.5 is the boundary - should be detected as padding
|
||||
assert padding_added is True
|
||||
assert padding_seconds == 3.0
|
||||
|
||||
def test_detect_padding_from_lrc_just_below_boundary(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test detection just below the 2.5 second boundary."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at 2.4 seconds (just below boundary)
|
||||
lrc_content = """[00:02.40]First line just below boundary
|
||||
[00:05.00]Second line
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
# 2.4 is below boundary - should NOT be detected as padding
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_empty_file(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test handling of empty LRC file."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write('')
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_no_timestamps(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test handling of LRC file with no valid timestamps."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
lrc_content = """[ti:Test Song]
|
||||
[ar:Test Artist]
|
||||
Just text without timestamps
|
||||
More text
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_missing_file(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test handling of missing LRC file."""
|
||||
lrc_file = os.path.join(temp_dir, 'nonexistent.lrc')
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_three_digit_milliseconds(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test parsing LRC files with three-digit milliseconds."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Some LRC files use [mm:ss.xxx] format
|
||||
lrc_content = """[00:03.500]First line at 3.5 seconds
|
||||
[00:06.200]Second line
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is True
|
||||
assert padding_seconds == 3.0
|
||||
|
||||
|
||||
class TestExistingFilesWithPaddingDetection:
|
||||
"""Tests for transcribe_lyrics returning padding info from existing files."""
|
||||
|
||||
def test_transcribe_lyrics_existing_files_returns_padding_info(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that transcribe_lyrics returns padding info when using existing files."""
|
||||
track_output_dir = temp_dir
|
||||
artist = "Test Artist"
|
||||
title = "Test Title"
|
||||
|
||||
# Create existing files
|
||||
video_path = os.path.join(track_output_dir, f"{artist} - {title} (With Vocals).mkv")
|
||||
lrc_path = os.path.join(track_output_dir, f"{artist} - {title} (Karaoke).lrc")
|
||||
|
||||
# Create the video file
|
||||
with open(video_path, 'w') as f:
|
||||
f.write('mock video')
|
||||
|
||||
# Create LRC file with countdown padding (first lyric at 3.5s)
|
||||
lrc_content = """[00:03.50]First line with padding
|
||||
[00:06.00]Second line
|
||||
"""
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
# Call transcribe_lyrics - it should detect existing files and return padding info
|
||||
result = basic_karaoke_gen.lyrics_processor.transcribe_lyrics(
|
||||
input_audio_wav=None, # Not needed when existing files found
|
||||
artist=artist,
|
||||
title=title,
|
||||
track_output_dir=track_output_dir,
|
||||
)
|
||||
|
||||
# Should return padding info from LRC detection
|
||||
assert result['countdown_padding_added'] is True
|
||||
assert result['countdown_padding_seconds'] == 3.0
|
||||
assert result['padded_audio_filepath'] is None # Original padded audio not available
|
||||
|
||||
def test_transcribe_lyrics_existing_files_no_padding(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test existing files case when there's no countdown padding."""
|
||||
track_output_dir = temp_dir
|
||||
artist = "Test Artist"
|
||||
title = "Test Title"
|
||||
|
||||
# Create existing files
|
||||
video_path = os.path.join(track_output_dir, f"{artist} - {title} (With Vocals).mkv")
|
||||
lrc_path = os.path.join(track_output_dir, f"{artist} - {title} (Karaoke).lrc")
|
||||
|
||||
# Create the video file
|
||||
with open(video_path, 'w') as f:
|
||||
f.write('mock video')
|
||||
|
||||
# Create LRC file WITHOUT countdown padding (first lyric at 0.5s)
|
||||
lrc_content = """[00:00.50]First line starts immediately
|
||||
[00:03.00]Second line
|
||||
"""
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
result = basic_karaoke_gen.lyrics_processor.transcribe_lyrics(
|
||||
input_audio_wav=None,
|
||||
artist=artist,
|
||||
title=title,
|
||||
track_output_dir=track_output_dir,
|
||||
)
|
||||
|
||||
# Should return no padding info
|
||||
assert result['countdown_padding_added'] is False
|
||||
assert result['countdown_padding_seconds'] == 0.0
|
||||
|
||||
|
||||
class TestScanDirectoryForInstrumentals:
|
||||
"""Tests for scanning directory for existing instrumentals."""
|
||||
|
||||
def test_scan_directory_finds_clean_instrumental(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that _scan_directory_for_instrumentals finds clean instrumentals."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
# Create a clean instrumental file
|
||||
instrumental_file = os.path.join(temp_dir, f"{artist_title} (Instrumental Clean).flac")
|
||||
with open(instrumental_file, 'w') as f:
|
||||
f.write('mock instrumental')
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
assert result['clean_instrumental']['instrumental'] == instrumental_file
|
||||
|
||||
def test_scan_directory_finds_combined_instrumentals(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that _scan_directory_for_instrumentals finds combined instrumentals."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
# Create combined instrumental files
|
||||
bv_file = os.path.join(temp_dir, f"{artist_title} (Instrumental +BV HTDemucs).flac")
|
||||
with open(bv_file, 'w') as f:
|
||||
f.write('mock instrumental with backing')
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
assert 'HTDemucs' in result['combined_instrumentals']
|
||||
assert result['combined_instrumentals']['HTDemucs'] == bv_file
|
||||
|
||||
def test_scan_directory_skips_padded_files(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that _scan_directory_for_instrumentals skips already-padded files."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
# Create both original and padded files
|
||||
original_file = os.path.join(temp_dir, f"{artist_title} (Instrumental Clean).flac")
|
||||
padded_file = os.path.join(temp_dir, f"{artist_title} (Instrumental Clean) (Padded).flac")
|
||||
|
||||
with open(original_file, 'w') as f:
|
||||
f.write('mock original')
|
||||
with open(padded_file, 'w') as f:
|
||||
f.write('mock padded')
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
# Should return original, not padded
|
||||
assert result['clean_instrumental']['instrumental'] == original_file
|
||||
assert 'Padded' not in result['clean_instrumental']['instrumental']
|
||||
|
||||
def test_scan_directory_empty_directory(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test _scan_directory_for_instrumentals on empty directory."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
# Should return empty structure
|
||||
assert result['clean_instrumental'].get('instrumental') is None
|
||||
assert len(result['combined_instrumentals']) == 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
|
||||
@@ -357,6 +357,95 @@ class TestServerStop:
|
||||
assert server._shutdown_event.is_set()
|
||||
|
||||
|
||||
class TestPortFinding:
|
||||
"""Test automatic port finding for concurrent CLI instances."""
|
||||
|
||||
def test_is_port_available_returns_true_for_free_port(self):
|
||||
"""Test _is_port_available returns True for an available port."""
|
||||
import socket
|
||||
# Find a free port by binding to 0
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(('127.0.0.1', 0))
|
||||
free_port = sock.getsockname()[1]
|
||||
|
||||
# The port should now be free
|
||||
assert InstrumentalReviewServer._is_port_available('127.0.0.1', free_port) is True
|
||||
|
||||
def test_is_port_available_returns_false_for_used_port(self):
|
||||
"""Test _is_port_available returns False when port is in use."""
|
||||
import socket
|
||||
# Bind to a port to make it unavailable
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.bind(('127.0.0.1', 0))
|
||||
used_port = sock.getsockname()[1]
|
||||
sock.listen(1)
|
||||
|
||||
# The port should be unavailable while socket is bound
|
||||
assert InstrumentalReviewServer._is_port_available('127.0.0.1', used_port) is False
|
||||
|
||||
def test_find_available_port_returns_preferred_if_free(self):
|
||||
"""Test _find_available_port returns preferred port when available."""
|
||||
import socket
|
||||
# Find a free port
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(('127.0.0.1', 0))
|
||||
free_port = sock.getsockname()[1]
|
||||
|
||||
# Should return the same port
|
||||
result = InstrumentalReviewServer._find_available_port('127.0.0.1', free_port)
|
||||
assert result == free_port
|
||||
|
||||
def test_find_available_port_tries_next_ports_when_preferred_busy(self):
|
||||
"""Test _find_available_port tries subsequent ports when preferred is busy."""
|
||||
import socket
|
||||
# Bind to a port to make it unavailable
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.bind(('127.0.0.1', 0))
|
||||
used_port = sock.getsockname()[1]
|
||||
sock.listen(1)
|
||||
|
||||
# Should return a different port (likely used_port + 1 if available)
|
||||
result = InstrumentalReviewServer._find_available_port('127.0.0.1', used_port)
|
||||
assert result != used_port
|
||||
assert result > used_port # Should be one of the next ports
|
||||
|
||||
def test_find_available_port_skips_multiple_busy_ports(self):
|
||||
"""Test _find_available_port skips multiple busy ports."""
|
||||
import socket
|
||||
sockets = []
|
||||
try:
|
||||
# Bind to 3 consecutive ports
|
||||
base_port = 49000 # Use a high port range
|
||||
for offset in range(3):
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind(('127.0.0.1', base_port + offset))
|
||||
sock.listen(1)
|
||||
sockets.append(sock)
|
||||
except OSError:
|
||||
sock.close()
|
||||
# Port might already be in use, that's fine for this test
|
||||
|
||||
if len(sockets) >= 2:
|
||||
# Should find a port beyond the bound ones
|
||||
result = InstrumentalReviewServer._find_available_port('127.0.0.1', base_port)
|
||||
assert result >= base_port + len(sockets)
|
||||
finally:
|
||||
for sock in sockets:
|
||||
sock.close()
|
||||
|
||||
def test_find_available_port_falls_back_to_os_assigned(self):
|
||||
"""Test _find_available_port falls back to OS-assigned port if needed."""
|
||||
# This should always work since we pass max_attempts=1 and use an invalid port
|
||||
# We can't easily test the fallback without binding to 100 consecutive ports,
|
||||
# but we can verify the method doesn't raise and returns a valid port
|
||||
result = InstrumentalReviewServer._find_available_port('127.0.0.1', 8765, max_attempts=100)
|
||||
assert 1024 <= result <= 65535
|
||||
|
||||
|
||||
class TestServerAPIEndpoints:
|
||||
"""Test FastAPI endpoint handlers using TestClient."""
|
||||
|
||||
|
||||
@@ -2038,3 +2038,201 @@ class TestSignedUrlUploadFlow:
|
||||
assert hasattr(client, '_get_content_type')
|
||||
assert callable(client._upload_file_to_signed_url)
|
||||
assert callable(client._get_content_type)
|
||||
|
||||
|
||||
class TestJobMonitorDownloadProgress:
|
||||
"""Tests for JobMonitor download progress display."""
|
||||
|
||||
@pytest.fixture
|
||||
def config(self):
|
||||
return Config(
|
||||
service_url="https://api.example.com",
|
||||
review_ui_url="https://review.example.com",
|
||||
poll_interval=1,
|
||||
output_dir="/tmp/output"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def logger(self):
|
||||
return logging.getLogger("test")
|
||||
|
||||
@pytest.fixture
|
||||
def monitor(self, config, logger):
|
||||
client = MagicMock()
|
||||
return JobMonitor(client, config, logger)
|
||||
|
||||
def test_show_download_progress_youtube(self, monitor, caplog):
|
||||
"""Test that YouTube downloads show simple message."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'YouTube'
|
||||
}
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading from YouTube" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_torrent_with_active_torrents(self, mock_get, monitor, caplog):
|
||||
"""Test that active torrents show progress details."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': True,
|
||||
'torrents': [{
|
||||
'progress': 45.5,
|
||||
'peers': 3,
|
||||
'download_speed': 1250.0,
|
||||
'stalled': False
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading from Redacted" in caplog.text
|
||||
assert "45.5%" in caplog.text
|
||||
assert "1250.0 KB/s" in caplog.text
|
||||
assert "3 peers" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_torrent_stalled(self, mock_get, monitor, caplog):
|
||||
"""Test that stalled torrents show warning."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'OPS'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': True,
|
||||
'torrents': [{
|
||||
'progress': 0.0,
|
||||
'peers': 0,
|
||||
'download_speed': 0,
|
||||
'stalled': True
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "STALLED" in caplog.text
|
||||
assert "no peers" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_no_torrents(self, mock_get, monitor, caplog):
|
||||
"""Test message when no active torrents."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': True,
|
||||
'torrents': []
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Starting download" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_transmission_unavailable(self, mock_get, monitor, caplog):
|
||||
"""Test warning when Transmission not available."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': False
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "not available" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_request_error(self, mock_get, monitor, caplog):
|
||||
"""Test fallback message on request error."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_get.side_effect = Exception("Connection error")
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading audio" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_http_error(self, mock_get, monitor, caplog):
|
||||
"""Test fallback on non-200 HTTP response."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading from Redacted" in caplog.text
|
||||
|
||||
def test_show_download_progress_missing_provider(self, monitor, caplog):
|
||||
"""Test handling of missing provider in state_data."""
|
||||
job_data = {
|
||||
'state_data': {}
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
# Should use 'unknown' as default
|
||||
assert "unknown" in caplog.text.lower() or "Downloading" in caplog.text
|
||||
Reference in new issue
Block a user